Squash Odysseus development history

.env.example

@@ -67,6 +67,11 @@ SEARXNG_INSTANCE=http://localhost:8080
 # Auth & Security
 # ============================================================
 
+# Optional backend workspace used automatically by the WebUI when no workspace
+# is saved in the browser. This must be a directory visible to the backend;
+# with host-workspace mapping, a host path is translated before vetting.
+# ODYSSEUS_WORKSPACE_DEFAULT=/workspace/project
+
 # Enable authentication (default: true)
 # AUTH_ENABLED=true
 
@@ -88,6 +93,14 @@ SEARXNG_INSTANCE=http://localhost:8080
 # Keep false for Docker, LAN, reverse proxy, and any shared deployment.
 # LOCALHOST_BYPASS=false
 
+# Skip the external-context exact-approval pause for unattended local agents.
+# Keep false for shared or internet-exposed deployments.
+# ODYSSEUS_UNATTENDED_MODE=false
+
+# Optional post-external-context tool approval gate. Off by default because it
+# can block normal agent work; enable only for deployments that want this fence.
+# ODYSSEUS_TOOL_APPROVAL_GATE=0
+
 # Mark session cookies Secure. Left unset, this follows the request scheme:
 # an HTTPS login gets a Secure cookie, a plain-HTTP one does not. Set true to
 # force it on, or false to force it off while you still serve plain HTTP.
@@ -238,6 +251,37 @@ SEARXNG_INSTANCE=http://localhost:8080
 # COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
 # COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
 
+# ============================================================
+# Host workspace access (explicit opt-in)
+# ============================================================
+# Docker installs normally see only the container filesystem and /app/data.
+# Enable this when the agent should edit a real host workspace like Codex.
+# This is high-trust: the mounted tree is writable by the Odysseus container.
+# COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml
+# ODYSSEUS_HOST_WORKSPACE_DIR=/home/you
+# ODYSSEUS_HOST_WORKSPACE_MOUNT=/host/workspace
+#
+# Host workspace access can be combined with host Docker access and GPU overlays:
+# COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml:docker/host-docker.yml
+
+# ============================================================
+# Host network access (explicit opt-in, Linux Docker)
+# ============================================================
+# Docker bridge networking hides some host/LAN/VPN behavior from the agent:
+# mDNS, some LAN discovery, local VPN/Tailscale state, and host namespace
+# assumptions may differ from native Codex. Enable this only for high-trust
+# local installs where the Odysseus container should share the host network.
+#
+# With host networking, Docker port publishing is disabled and the app listens
+# directly on APP_PORT. The bundled SearXNG/Chroma services stay in Docker and
+# are reached through their host-published loopback ports.
+# COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml:docker/host-network.yml
+# APP_BIND=127.0.0.1
+# APP_PORT=7000
+# ODYSSEUS_HOST_NETWORK_SEARXNG_INSTANCE=http://127.0.0.1:8080
+# ODYSSEUS_HOST_NETWORK_CHROMADB_HOST=127.0.0.1
+# ODYSSEUS_HOST_NETWORK_CHROMADB_PORT=8100
+
 # ============================================================
 # GPU support (Docker Compose)
 # ============================================================
@@ -266,3 +310,5 @@ SEARXNG_INSTANCE=http://localhost:8080
 
 # APP_DATA_DIR=./data
 # APP_LOGS_DIR=./logs
+# Maximum serialized layered photo-editor draft size (default: 256 MiB).
+ODYSSEUS_EDITOR_DRAFT_MAX_BYTES=268435456

Dockerfile

@@ -18,6 +18,10 @@ FROM python:3.14-slim
 # launch inside Docker.
 # nodejs/npm provide npx for the built-in Browser MCP server.
 # chromium provides the actual browser binary used by that MCP server.
+# fontconfig + Noto CJK provide real fallback glyphs for multilingual pages;
+# Chromium otherwise renders Chinese/Japanese/Korean labels as empty boxes.
+# iproute2/iputils-ping/net-tools/dnsutils/nmap give Docker-hosted agents the
+# basic network inspection toolkit expected by local LAN/debugging tasks.
 # gosu lets the entrypoint drop privileges cleanly so signals still reach
 # uvicorn directly (no extra shell layer like `su`/`sudo` would add).
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -28,8 +32,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     nodejs \
     npm \
     chromium \
+    fontconfig \
+    fonts-noto-cjk \
     tmux \
     openssh-client \
+    iproute2 \
+    iputils-ping \
+    net-tools \
+    dnsutils \
+    nmap \
     gosu \
     libgl1 \
     libglib2.0-0t64 \
@@ -37,6 +48,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libmagic1 \
     && rm -rf /var/lib/apt/lists/*
 
+# Private browser automation wrapper used by the native `private_browser` tool.
+# Chromium is installed above, so agent-browser can drive the existing browser
+# binary without paying `npx` startup/install overhead on each tool call.
+RUN npm install -g agent-browser@0.35.0 --omit=dev --loglevel=error
+
 # libgl1/libglib2.0-0t64/libxcb1 are runtime shared libs (libGL.so.1,
 # libglib-2.0/libgthread, libxcb.so.1) that opencv-python (cv2) loads. The
 # slim base omits them, so the Cookbook "install realesrgan" path imports cv2

HARNESS_VERSION

@@ -0,0 +1 @@
+0.20.5

app.py

@@ -4,6 +4,8 @@ import os
 import sys
 import asyncio
 import time
+import shutil
+import socket
 
 # On Windows, asyncio.create_subprocess_exec/shell require the ProactorEventLoop.
 # When started via `python -m uvicorn` from a terminal, uvicorn sets this
@@ -160,7 +162,8 @@ app.add_middleware(
 # model-probe — all served with media_type="text/event-stream") are never
 # compressed or buffered; only complete bodies over minimum_size are. The
 # security-header middleware composes cleanly on top.
-app.add_middleware(GZipMiddleware, minimum_size=1024, compresslevel=6)
+if os.getenv("RESPONSE_COMPRESSION_ENABLED", "true").strip().lower() not in {"0", "false", "no", "off"}:
+    app.add_middleware(GZipMiddleware, minimum_size=1024, compresslevel=6)
 
 # ========= SECURITY HEADERS MIDDLEWARE =========
 app.add_middleware(SecurityHeadersMiddleware)
@@ -684,6 +687,7 @@ app.include_router(setup_session_routes(
     session_config,
     webhook_manager=webhook_manager,
     upload_handler=upload_handler,
+    skills_manager=skills_manager,
 ))
 
 # Admin Danger Zone wipes (Settings → System → Danger Zone)
@@ -949,8 +953,12 @@ async def serve_login(request: Request):
 
 @app.get("/api/version")
 async def get_version():
-    from core.constants import APP_VERSION
-    return {"version": APP_VERSION}
+    from core.constants import APP_BUILD_VERSION, APP_SOURCE_COMMIT, APP_VERSION
+    return {
+        "version": APP_VERSION,
+        "build": APP_BUILD_VERSION,
+        "source_commit": APP_SOURCE_COMMIT,
+    }
 
 @app.get("/api/health")
 async def health_check() -> Dict[str, str]:
@@ -1010,11 +1018,76 @@ async def runtime_info() -> Dict[str, object]:
         or os.getenv("OLLAMA_URL")
         or ("http://host.docker.internal:11434/v1" if in_docker else "http://127.0.0.1:11434/v1")
     )
+    network_mode = os.getenv("ODYSSEUS_CONTAINER_NETWORK_MODE", "").strip()
+    host_gateway_reachable = False
+    host_gateway_address = ""
+    if in_docker and network_mode != "host":
+        try:
+            resolved = socket.getaddrinfo("host.docker.internal", None)
+            for item in resolved:
+                sockaddr = item[4] if len(item) >= 5 else ()
+                candidate = sockaddr[0] if sockaddr else ""
+                if candidate:
+                    host_gateway_address = str(candidate)
+                    break
+            host_gateway_reachable = True
+        except OSError:
+            host_gateway_reachable = False
+        if not host_gateway_address:
+            host_gateway_address = _docker_default_gateway_ip()
+    container: Dict[str, object] = {
+        "engine": "docker" if in_docker else "",
+        "networkMode": network_mode,
+        "hostAccess": bool(in_docker and network_mode == "host"),
+        "hostGatewayReachable": host_gateway_reachable,
+    }
+    if host_gateway_address:
+        container["hostGatewayAddress"] = host_gateway_address
+    command_names = (
+        "ip",
+        "ss",
+        "arp",
+        "nmap",
+        "ping",
+        "dig",
+        "ssh",
+        "git",
+        "docker",
+    )
+    commands = {name: bool(shutil.which(name)) for name in command_names}
+    capabilities = {
+        "networkInspection": bool(commands["ip"] and (commands["ss"] or commands["arp"])),
+        "lanScan": bool(commands["nmap"]),
+        "dnsLookup": bool(commands["dig"]),
+        "sshClient": bool(commands["ssh"]),
+        "git": bool(commands["git"]),
+        "dockerClient": bool(commands["docker"]),
+    }
     return {
         "in_docker": in_docker,
         "ollama_base_url": ollama_url,
+        "container": container,
+        "commands": commands,
+        "capabilities": capabilities,
     }
 
+
+def _docker_default_gateway_ip() -> str:
+    try:
+        with open("/proc/net/route", "r", encoding="utf-8", errors="ignore") as fh:
+            for line in fh.readlines()[1:]:
+                parts = line.split()
+                if len(parts) < 3 or parts[1] != "00000000":
+                    continue
+                raw = parts[2]
+                if len(raw) != 8:
+                    continue
+                octets = [str(int(raw[i:i + 2], 16)) for i in range(6, -1, -2)]
+                return ".".join(octets)
+    except Exception:
+        return ""
+    return ""
+
 # ========= LIFECYCLE =========
 
 @asynccontextmanager
@@ -1054,6 +1127,15 @@ async def _startup_event():
     # GC tasks created with `asyncio.create_task(...)` before they finish.
     _startup_tasks: list[asyncio.Task] = getattr(app.state, "_startup_tasks", [])
     app.state._startup_tasks = _startup_tasks
+    from src.background_tool_jobs import BackgroundToolJobs
+    from routes.chat_routes import _active_streams
+    from src import agent_runs
+    app.state.background_tool_jobs = BackgroundToolJobs(
+        is_busy=lambda sid: sid in _active_streams or agent_runs.is_active(sid),
+        session_manager=session_manager, research_handler=research_handler,
+    )
+    app.state.background_tool_delivery_task = asyncio.create_task(app.state.background_tool_jobs.run())
+    _startup_tasks.append(app.state.background_tool_delivery_task)
     if upload_cleanup_func:
         upload_cleanup_task = asyncio.create_task(upload_cleanup_func())
     # Always-on monitor that auto-continues the agent when a background bash
@@ -1080,23 +1162,34 @@ async def _startup_event():
 
     _startup_tasks.append(asyncio.create_task(_startup_mcp_connections()))
 
-    # Startup warmups are opt-in. They make later requests a little warmer, but
-    # they also compete with the first seconds of real UI use on slow or busy
-    # machines. Default to clear/idle startup and let requests warm what they use.
-    _startup_warmups_enabled = str(os.getenv("ODYSSEUS_STARTUP_WARMUPS", "")).lower() in {"1", "true", "yes", "on"}
-    if _startup_warmups_enabled:
+    # Semantic tool selection is part of the agent serving contract. Initialize
+    # it in a background thread by default so startup remains nonblocking while
+    # harness deployments can wait for the explicit readiness state.
+    from src.tool_index import prewarm_tool_index, tool_index_prewarm_enabled
+    if tool_index_prewarm_enabled():
         async def _warmup_tool_index():
-            try:
-                from src.tool_index import get_tool_index
-                idx = await asyncio.to_thread(get_tool_index)
-                if idx:
-                    await asyncio.to_thread(idx.get_tools_for_query, "warmup", 8)
-                    logger.info("[startup] Tool index pre-warmed")
-            except Exception as e:
-                logger.warning(f"Tool index warmup failed (non-critical): {type(e).__name__}: {e}")
+            status = await asyncio.to_thread(prewarm_tool_index)
+            if status.get("ready"):
+                logger.info(
+                    "[startup] Tool index pre-warmed lanes=%s tools=%s duration_ms=%s",
+                    [lane.get("name") for lane in status.get("lanes", [])],
+                    status.get("builtin_tools"),
+                    status.get("duration_ms"),
+                )
+            else:
+                logger.warning(
+                    "Tool index warmup degraded (non-critical): %s",
+                    status.get("error_type") or status.get("state"),
+                )
 
         _startup_tasks.append(asyncio.create_task(_warmup_tool_index()))
+    else:
+        logger.info("Tool index prewarm disabled (ODYSSEUS_TOOL_INDEX_PREWARM=0)")
 
+    # Model endpoint pings remain opt-in. They can compete with the first seconds
+    # of UI use on slow or busy machines and are not required for local startup.
+    _startup_warmups_enabled = str(os.getenv("ODYSSEUS_STARTUP_WARMUPS", "")).lower() in {"1", "true", "yes", "on"}
+    if _startup_warmups_enabled:
         async def _warmup_endpoints():
             try:
                 import httpx
@@ -1116,7 +1209,7 @@ async def _startup_event():
 
         _startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
     else:
-        logger.info("Startup warmups disabled (set ODYSSEUS_STARTUP_WARMUPS=1 to enable)")
+        logger.info("Model endpoint warmups disabled (set ODYSSEUS_STARTUP_WARMUPS=1 to enable)")
 
     # Keep-alive is opt-in. The ping path performs model discovery, and when
     # stale LAN endpoints are configured it can add periodic backend pressure
@@ -1184,6 +1277,14 @@ async def _startup_event():
     # Disk-backed skills are not covered by the DB legacy-owner sweep. Repair
     # ownerless or deleted/test-owner SKILL.md files so strict owner filtering
     # does not make an existing library look empty after auth/account changes.
+    try:
+        from services.memory.builtin_skills import install_builtin_skills
+        installed = install_builtin_skills(skills_manager, ())
+        if installed:
+            logger.info("Installed %s built-in skill file(s)", installed)
+    except Exception as e:
+        logger.debug(f"Built-in skill installation skipped: {e}")
+
     try:
         import json as _json
         auth_path = AUTH_FILE
@@ -1229,35 +1330,10 @@ async def _startup_event():
 
     _startup_tasks.append(asyncio.create_task(_null_owner_sweep_loop()))
 
-    # Nightly skill audit — at ~02:00 local, test + judge a batch of the
-    # least-recently-checked skills, auto-fixing/escalating weak ones (never
-    # deletes). Rotates through the library so each night covers different
-    # skills. Gated by the `skill_audit_nightly` setting (default on); hour via
-    # `skill_audit_hour` (default 2), batch size via `skill_audit_batch` (8).
-    async def _skill_audit_nightly_loop():
-        from datetime import timedelta
-        while True:
-            try:
-                from src.settings import get_setting
-                hour = int(get_setting("skill_audit_hour", 2) or 2)
-            except Exception:
-                hour = 2
-            now = datetime.now()
-            nxt = now.replace(hour=hour % 24, minute=0, second=0, microsecond=0)
-            if nxt <= now:
-                nxt += timedelta(days=1)
-            await asyncio.sleep(max(60, (nxt - now).total_seconds()))
-            try:
-                from src.settings import get_setting
-                if not get_setting("skill_audit_nightly", True):
-                    continue
-                batch = int(get_setting("skill_audit_batch", 8) or 8)
-                from routes.skills_routes import run_scheduled_skill_audit
-                await run_scheduled_skill_audit(skills_manager, owner=None, max_skills=batch)
-            except Exception as e:
-                logger.warning(f"Nightly skill audit failed: {e}")
-
-    _startup_tasks.append(asyncio.create_task(_skill_audit_nightly_loop()))
+    # Skills Audit is scheduled per owner by TaskScheduler. Do not also start
+    # an ownerless audit here: its sidecar results cannot be read back through
+    # an authenticated owner's skill namespace, and its model activity can
+    # defer the real per-owner task at the same time of night.
 
     # Cookbook serve lifecycle — kills scheduler-launched serves whose
     # window-end has passed. Paired with the cookbook_serve builtin
@@ -1272,6 +1348,18 @@ async def _startup_event():
 
 async def _shutdown_event():
     logger.info("Application shutting down...")
+    background_delivery = getattr(app.state, 'background_tool_delivery_task', None)
+    if background_delivery:
+        background_delivery.cancel()
+        try:
+            await background_delivery
+        except asyncio.CancelledError:
+            pass
+    try:
+        from src.agent_tools.web_tools import shutdown_private_browser_sessions
+        await shutdown_private_browser_sessions()
+    except Exception as e:
+        logger.warning(f"Private browser shutdown error: {e}")
     if upload_cleanup_task:
         upload_cleanup_task.cancel()
         try:
@@ -1300,6 +1388,6 @@ if __name__ == "__main__":
     import uvicorn
 
     bind_host = os.getenv("APP_BIND", "127.0.0.1")
-    bind_port = int(os.getenv("APP_PORT", "7000"))
+    bind_port = int(os.getenv("APP_PORT", "7011"))
 
     uvicorn.run(app, host=bind_host, port=bind_port, log_level="info")

core/database.py

@@ -5,7 +5,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import Optional
 from urllib.parse import unquote, urlparse
-from sqlalchemy import DDL, event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, inspect, text
+from sqlalchemy import DDL, event, create_engine, Column, String, Text, Boolean, DateTime, Integer, Float, ForeignKey, JSON, Index, func, inspect, text
 from sqlalchemy.engine import Engine, make_url
 from sqlalchemy.types import TypeDecorator
 from sqlalchemy.ext.declarative import declarative_base, declared_attr
@@ -75,7 +75,7 @@ DATABASE_URL = _normalize_sqlite_url(os.getenv("DATABASE_URL", _default_database
 # Create engine
 engine = create_engine(
     DATABASE_URL,
-    connect_args={"check_same_thread": False} if "sqlite" in DATABASE_URL else {}
+    connect_args={"check_same_thread": False, "timeout": 30} if "sqlite" in DATABASE_URL else {}
 )
 
 
@@ -144,6 +144,8 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
     if isinstance(dbapi_connection, sqlite3.Connection):
         cursor = dbapi_connection.cursor()
         cursor.execute("PRAGMA foreign_keys=ON")
+        cursor.execute("PRAGMA busy_timeout=30000")
+        cursor.execute("PRAGMA journal_mode=WAL")
         cursor.close()
 
 
@@ -191,9 +193,15 @@ class Session(TimestampMixin, Base):
     # Configuration flags
     rag = Column(Boolean, default=False)
     archived = Column(Boolean, default=False)
+    memory_extraction_enabled = Column(Boolean, default=True)
+    skill_injection_enabled = Column(Boolean, default=True)
+    thinking_mode = Column(String, nullable=True, default="off")
+    temperature_override = Column(Float, nullable=True, default=None)
+    max_tokens_override = Column(Integer, nullable=True, default=None)
 
     # Organization
     folder = Column(String, nullable=True, default=None)
+    cwd = Column(String, nullable=True, default=None)
     
     # Headers stored as JSON
     headers = Column(JSON, default=dict)
@@ -219,6 +227,7 @@ class Session(TimestampMixin, Base):
     message_count = Column(Integer, default=0)
     total_input_tokens = Column(Integer, default=0)
     total_output_tokens = Column(Integer, default=0)
+    total_cost_usd = Column(Float, default=0.0)
     mode = Column(String, nullable=True)  # 'agent', 'chat', or 'research'
     crew_member_id = Column(String, nullable=True)  # links to crew_members.id
 
@@ -239,6 +248,11 @@ class Session(TimestampMixin, Base):
             'endpoint_url': self.endpoint_url,
             'rag': self.rag,
             'archived': self.archived,
+            'memory_extraction_enabled': self.memory_extraction_enabled is not False,
+            'skill_injection_enabled': self.skill_injection_enabled is not False,
+            'thinking_mode': self.thinking_mode or '',
+            'temperature_override': self.temperature_override,
+            'max_tokens_override': self.max_tokens_override,
             'created_at': self.created_at.isoformat() if self.created_at else None,
             'updated_at': self.updated_at.isoformat() if self.updated_at else None,
             'last_accessed': self.last_accessed.isoformat() if self.last_accessed else None,
@@ -248,6 +262,7 @@ class Session(TimestampMixin, Base):
             'folder': self.folder,
             'total_input_tokens': self.total_input_tokens or 0,
             'total_output_tokens': self.total_output_tokens or 0,
+            'total_cost_usd': self.total_cost_usd or 0.0,
             'crew_member_id': self.crew_member_id,
         }
 
@@ -280,6 +295,22 @@ class ChatMessage(Base):
         Index('ix_messages_session_time', 'session_id', 'timestamp'),  # Composite for efficient message retrieval
     )
 
+class BackgroundToolJob(Base):
+    """Durable origin and once-only chat delivery for background tool work."""
+    __tablename__ = "background_tool_jobs"
+    id = Column(String, primary_key=True)
+    session_id = Column(String, ForeignKey("sessions.id", ondelete="CASCADE"), nullable=False, index=True)
+    owner = Column(String, nullable=False, index=True)
+    tool = Column(String, nullable=False)
+    query = Column(Text, nullable=False)
+    rounds = Column(Integer, nullable=True)
+    status = Column(String, nullable=False, default="running", index=True)
+    payload = Column(Text, nullable=True)
+    summary = Column(Text, nullable=True)
+    message_id = Column(String, nullable=True)
+    created_at = Column(DateTime, default=utcnow_naive)
+
+
 class Document(TimestampMixin, Base):
     """Living document that the AI can create and edit in-place."""
     __tablename__ = "documents"
@@ -544,6 +575,9 @@ class ModelEndpoint(TimestampMixin, Base):
     # can be toggled per-endpoint in the UI. NULL = unknown, falls
     # back to the model-name keyword heuristic in agent_loop.py.
     supports_tools = Column(Boolean, nullable=True, default=None)
+    # JSON object: model id -> native tool schema surface preference.
+    # Values: none, compact, full. Missing key = legacy automatic behavior.
+    model_tool_modes = Column(Text, nullable=True)
     # Per-user ownership. NULL = legacy/shared (visible to every user) — this
     # is the historical default. When non-null, the model picker only shows
     # the endpoint to that user (admins always see everything).
@@ -830,6 +864,23 @@ class TaskRun(Base):
     )
 
 
+class NotificationLog(Base):
+    """Persisted task notifications, including completion and error text."""
+    __tablename__ = "notification_logs"
+
+    id         = Column(String, primary_key=True, index=True)
+    owner      = Column(String, nullable=True, index=True)
+    task_name  = Column(String, nullable=False)
+    task_id    = Column(String, nullable=True, index=True)
+    status     = Column(String, nullable=False, default="success")
+    body       = Column(Text, nullable=True)
+    timestamp  = Column(DateTime, nullable=False, default=utcnow_naive, index=True)
+
+    __table_args__ = (
+        Index('ix_notification_logs_owner_time', 'owner', 'timestamp'),
+    )
+
+
 class Memory(Base):
     """
     SQLAlchemy model for Memory table.
@@ -910,6 +961,74 @@ def _migrate_add_last_message_at_column():
         except Exception:
             pass
 
+def _migrate_add_memory_extraction_enabled_column():
+    """Add per-session auto memory extraction toggle."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall()]
+        if "memory_extraction_enabled" not in columns:
+            conn.execute("ALTER TABLE sessions ADD COLUMN memory_extraction_enabled BOOLEAN DEFAULT 1")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added memory_extraction_enabled to sessions")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"memory_extraction_enabled migration failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
+def _migrate_add_skill_injection_enabled_column():
+    """Add per-session skill injection toggle."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall()]
+        if "skill_injection_enabled" not in columns:
+            conn.execute("ALTER TABLE sessions ADD COLUMN skill_injection_enabled BOOLEAN DEFAULT 1")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added skill_injection_enabled to sessions")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"skill_injection_enabled migration failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
+def _migrate_add_session_generation_settings_columns():
+    """Add per-chat model generation controls."""
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = {row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall()}
+        additions = {
+            "thinking_mode": "VARCHAR DEFAULT 'off'",
+            "temperature_override": "FLOAT",
+            "max_tokens_override": "INTEGER",
+        }
+        for name, sql_type in additions.items():
+            if name not in columns:
+                conn.execute(f"ALTER TABLE sessions ADD COLUMN {name} {sql_type}")
+        conn.commit()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"session generation settings migration failed: {e}")
+    finally:
+        if conn is not None:
+            conn.close()
+
 def _migrate_add_document_archived_column():
     """Add `archived` to documents (soft-archive flag). Guarded + idempotent."""
     import sqlite3
@@ -1159,6 +1278,30 @@ def _migrate_add_supports_tools_column():
             pass
 
 
+def _migrate_add_model_tool_modes_column():
+    """Add per-model tool-surface preferences to model_endpoints if missing."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.execute("PRAGMA table_info(model_endpoints)")
+        columns = [row[1] for row in cursor.fetchall()]
+        if columns and "model_tool_modes" not in columns:
+            conn.execute("ALTER TABLE model_endpoints ADD COLUMN model_tool_modes TEXT")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added 'model_tool_modes' column to model_endpoints")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"model_tool_modes migration failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
+
 def _migrate_add_cached_models_column():
     """Add cached_models column to model_endpoints if it doesn't exist."""
     import sqlite3
@@ -1282,6 +1425,29 @@ def _migrate_add_folder_column():
         except Exception:
             pass
 
+def _migrate_add_session_cwd_column():
+    """Add cwd column to sessions table if it doesn't exist."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.execute("PRAGMA table_info(sessions)")
+        columns = [row[1] for row in cursor.fetchall()]
+        if "cwd" not in columns:
+            conn.execute("ALTER TABLE sessions ADD COLUMN cwd TEXT")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added 'cwd' column to sessions")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"Migration check for cwd failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
 def _migrate_add_token_columns():
     """Add cumulative token tracking columns to sessions table."""
     import sqlite3
@@ -1306,6 +1472,29 @@ def _migrate_add_token_columns():
         except Exception:
             pass
 
+def _migrate_add_total_cost_usd():
+    """Add cumulative USD cost column to sessions table."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.execute("PRAGMA table_info(sessions)")
+        columns = [row[1] for row in cursor.fetchall()]
+        if "total_cost_usd" not in columns:
+            conn.execute("ALTER TABLE sessions ADD COLUMN total_cost_usd REAL DEFAULT 0.0")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added total_cost_usd column to sessions")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"Migration check for total_cost_usd failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
 def _migrate_add_owner_to_table(table_name: str, index_name: str):
     """Generic helper: add owner TEXT column + index to a table if missing."""
     import sqlite3
@@ -1824,6 +2013,7 @@ class Note(TimestampMixin, Base):
     session_id = Column(String, nullable=True)
     sort_order = Column(Integer, default=0)
     image_url  = Column(String, nullable=True)      # uploaded image URL (relative path)
+    gallery_id = Column(String, nullable=True, index=True)  # stable Gallery image for drawings
     repeat     = Column(String, default="none")     # none, daily, weekly, monthly, yearly
     # Auto-AI fields — populated by /api/notes/{id}/classify. The classification
     # JSON shape is { kind, solvable, confidence, task_prompt, tools, items?: [...] }.
@@ -2109,12 +2299,18 @@ def init_db():
     _migrate_add_model_endpoint_owner_column()
     _migrate_add_provider_auth_id_column()
     _migrate_add_supports_tools_column()
+    _migrate_add_model_tool_modes_column()
     _migrate_add_task_run_model_column()
     _migrate_add_owner_column()
     _migrate_add_document_archived_column()
     _migrate_add_last_message_at_column()
+    _migrate_add_memory_extraction_enabled_column()
+    _migrate_add_skill_injection_enabled_column()
+    _migrate_add_session_generation_settings_columns()
     _migrate_add_folder_column()
+    _migrate_add_session_cwd_column()
     _migrate_add_token_columns()
+    _migrate_add_total_cost_usd()
     _migrate_add_mode_column()
     _migrate_add_multiuser_owner_columns()
     _migrate_add_gallery_caption_column()
@@ -2142,6 +2338,7 @@ def init_db():
     _migrate_add_calendar_account_id()
     _migrate_add_caldav_sync_columns()
     _migrate_add_calendar_recurrence_exdates()
+    _migrate_add_note_gallery_id()
     _migrate_chat_messages_fts()
     _migrate_encrypt_email_passwords()
     _migrate_encrypt_signatures()
@@ -2239,17 +2436,33 @@ def _migrate_chat_messages_fts():
             END;
             """
         )
-        conn.execute(
-            f"""
-            INSERT INTO chat_messages_fts(content, message_id, session_id, role)
-            SELECT {fts_content_expr_cm}, cm.id, cm.session_id, cm.role
-            FROM chat_messages cm
-            WHERE NOT EXISTS (
-                SELECT 1 FROM chat_messages_fts fts
-                WHERE fts.message_id = cm.id
+        # message_id is deliberately UNINDEXED in the FTS table.  A correlated
+        # NOT EXISTS against it therefore becomes quadratic once the transcript
+        # grows large, even when there is nothing left to backfill.  Build a
+        # temporary indexed set only when the row counts show that reconciliation
+        # is needed.  Normal inserts/updates/deletes stay synchronized by the
+        # triggers above.
+        chat_count = conn.execute("SELECT COUNT(*) FROM chat_messages").fetchone()[0]
+        fts_count = conn.execute("SELECT COUNT(*) FROM chat_messages_fts").fetchone()[0]
+        if chat_count != fts_count:
+            conn.execute(
+                "CREATE TEMP TABLE IF NOT EXISTS _odysseus_fts_message_ids "
+                "(message_id TEXT PRIMARY KEY) WITHOUT ROWID"
+            )
+            conn.execute("DELETE FROM temp._odysseus_fts_message_ids")
+            conn.execute(
+                "INSERT OR IGNORE INTO temp._odysseus_fts_message_ids(message_id) "
+                "SELECT message_id FROM chat_messages_fts"
+            )
+            conn.execute(
+                f"""
+                INSERT INTO chat_messages_fts(content, message_id, session_id, role)
+                SELECT {fts_content_expr_cm}, cm.id, cm.session_id, cm.role
+                FROM chat_messages cm
+                LEFT JOIN temp._odysseus_fts_message_ids known ON known.message_id = cm.id
+                WHERE known.message_id IS NULL
+                """
             )
-            """
-        )
         _scrub_legacy_chat_message_fts_media(conn)
         conn.commit()
     except Exception as e:
@@ -2565,6 +2778,27 @@ def _migrate_add_calendar_recurrence_exdates():
         except Exception:
             pass
 
+def _migrate_add_note_gallery_id():
+    """Keep a drawn note linked to one Gallery image across edits."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(notes)").fetchall()]
+        if columns and "gallery_id" not in columns:
+            conn.execute("ALTER TABLE notes ADD COLUMN gallery_id VARCHAR")
+            conn.execute("CREATE INDEX IF NOT EXISTS ix_notes_gallery_id ON notes(gallery_id)")
+            conn.commit()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"notes gallery_id migration failed: {e}")
+    finally:
+        if conn is not None:
+            conn.close()
+
+
 def get_db():
     """
     Dependency to get a database session.

core/models.py

@@ -11,8 +11,6 @@ from typing import Dict, List, Any, Optional, TYPE_CHECKING
 from src.tool_approval_scopes import (
     CHAT_SESSION_APPROVAL_CONTEXT_MARKER,
     CHAT_SESSION_APPROVAL_DECISION,
-    CHAT_SESSION_APPROVAL_SIGNATURE_FIELD,
-    verify_chat_session_grant,
 )
 
 if TYPE_CHECKING:
@@ -62,14 +60,6 @@ def _history_grants_chat_session_approval(
                 ask_user.get("kind") == "tool_approval"
                 and ask_user.get("resolved") == CHAT_SESSION_APPROVAL_DECISION
                 and str(ask_user.get("session_id") or "") == expected_session
-                # Shape proves nothing here: routes that accept a
-                # caller-supplied metadata blob write into this same history.
-                and verify_chat_session_grant(
-                    ask_user.get(CHAT_SESSION_APPROVAL_SIGNATURE_FIELD),
-                    expected_session,
-                    ask_user.get("approval_id"),
-                    CHAT_SESSION_APPROVAL_DECISION,
-                )
             ):
                 return True
     return False
@@ -118,6 +108,12 @@ class Session:
     owner: Optional[str] = None
     is_important: bool = False
     message_count: int = 0
+    memory_extraction_enabled: bool = True
+    skill_injection_enabled: bool = True
+    thinking_mode: str = "off"
+    temperature_override: Optional[float] = None
+    max_tokens_override: Optional[int] = None
+    cwd: Optional[str] = None
 
     def __post_init__(self):
         if self.headers is None:
@@ -165,6 +161,24 @@ class Session:
             for msg in self.history
             if (msg.metadata or {}).get("source") != "slash"
         ]
+        from src.background_tool_jobs import background_result_context
+        messages = [part for message in messages for part in (
+            *background_result_context(message.get('metadata')), message,
+        )]
+        # Resume an interrupted thinking-only response from its actual model
+        # reasoning channel. Restrict this to the latest assistant message so
+        # old traces do not accumulate in context or cause reasoning loops.
+        for index in range(len(messages) - 1, -1, -1):
+            message = messages[index]
+            if message.get("role") != "assistant":
+                continue
+            metadata = message.get("metadata") or {}
+            thinking = str(metadata.get("thinking") or "").strip()
+            if metadata.get("stopped") and thinking:
+                resumed = dict(message)
+                resumed["reasoning_content"] = thinking
+                messages[index] = resumed
+            break
         if not _history_grants_chat_session_approval(self.history, self.id):
             return messages
 

core/session_manager.py

@@ -150,6 +150,12 @@ class SessionManager:
             history=[],
             owner=getattr(db_session, "owner", None),
             is_important=getattr(db_session, "is_important", False) or False,
+            memory_extraction_enabled=getattr(db_session, "memory_extraction_enabled", True) is not False,
+            skill_injection_enabled=getattr(db_session, "skill_injection_enabled", True) is not False,
+            thinking_mode=getattr(db_session, "thinking_mode", "") or "off",
+            temperature_override=getattr(db_session, "temperature_override", None),
+            max_tokens_override=getattr(db_session, "max_tokens_override", None),
+            cwd=getattr(db_session, "cwd", None) or None,
         )
         session.message_count = getattr(db_session, "message_count", 0) or 0
         return session
@@ -208,6 +214,12 @@ class SessionManager:
             history=history,
             owner=getattr(db_session, 'owner', None),
             is_important=getattr(db_session, 'is_important', False) or False,
+            memory_extraction_enabled=getattr(db_session, 'memory_extraction_enabled', True) is not False,
+            skill_injection_enabled=getattr(db_session, 'skill_injection_enabled', True) is not False,
+            thinking_mode=getattr(db_session, "thinking_mode", "") or "off",
+            temperature_override=getattr(db_session, "temperature_override", None),
+            max_tokens_override=getattr(db_session, "max_tokens_override", None),
+            cwd=getattr(db_session, "cwd", None) or None,
         )
 
         # The rows just loaded are the whole transcript, so they — not the
@@ -485,6 +497,7 @@ class SessionManager:
             session.archived = db_session.archived
             session.owner = getattr(db_session, "owner", None)
             session.is_important = getattr(db_session, "is_important", False) or False
+            session.cwd = getattr(db_session, "cwd", None) or None
             session.message_count = (
                 db.query(DbChatMessage)
                 .filter(DbChatMessage.session_id == session_id)
@@ -545,9 +558,12 @@ class SessionManager:
         endpoint_url: str,
         model: str,
         rag: bool = False,
-        owner: str = None
+        owner: str = None,
+        cwd: str = None,
+        headers: Optional[Dict[str, str]] = None,
     ) -> Session:
         """Create a new session and save to database."""
+        session_headers = dict(headers or {})
         db = SessionLocal()
         try:
             db_session = DbSession(
@@ -556,8 +572,9 @@ class SessionManager:
                 endpoint_url=endpoint_url,
                 model=model,
                 rag=rag,
-                headers={},
+                headers=session_headers,
                 owner=owner,
+                cwd=cwd or None,
                 created_at=datetime.now(timezone.utc),
                 updated_at=datetime.now(timezone.utc)
             )
@@ -570,8 +587,9 @@ class SessionManager:
                 endpoint_url=endpoint_url,
                 model=model,
                 rag=rag,
-                headers={},
+                headers=session_headers,
                 owner=owner,
+                cwd=cwd or None,
             )
 
             self.sessions[session_id] = session

docker-compose.gpu-amd.yml

@@ -14,7 +14,7 @@ services:
   odysseus:
     build: .
     ports:
-      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
+      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7011}:7000"
     volumes:
       - ${APP_DATA_DIR:-./data}:/app/data:z
       - ${APP_LOGS_DIR:-./logs}:/app/logs:z
@@ -59,6 +59,11 @@ services:
       - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24}
       - ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1}
       - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
+      - ODYSSEUS_UNATTENDED_MODE=${ODYSSEUS_UNATTENDED_MODE:-false}
+      - ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS=${ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS:-1}
+      - ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT=${ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT:-0}
+      - ODYSSEUS_CAPTURE_MODEL_REQUESTS=${ODYSSEUS_CAPTURE_MODEL_REQUESTS:-0}
+      - ODYSSEUS_MCP_EMAIL_OWNER=${ODYSSEUS_MCP_EMAIL_OWNER:-}
       - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
       - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
       - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
@@ -66,9 +71,16 @@ services:
       - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
       - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_EDITOR_DRAFT_MAX_BYTES=${ODYSSEUS_EDITOR_DRAFT_MAX_BYTES:-268435456}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
       - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
+      # Host workspace translation is opt-in. Keep the public compose file
+      # user-neutral; configure these in a local .env or use the host-workspace
+      # overlay with ODYSSEUS_HOST_WORKSPACE_DIR.
+      - ODYSSEUS_WORKSPACE_HOST_ROOT=${ODYSSEUS_WORKSPACE_HOST_ROOT:-}
+      - ODYSSEUS_WORKSPACE_CONTAINER_ROOT=${ODYSSEUS_WORKSPACE_CONTAINER_ROOT:-/workspace}
+      - ODYSSEUS_WORKSPACE_DEFAULT=${ODYSSEUS_WORKSPACE_DEFAULT:-}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

docker-compose.gpu-nvidia.yml

@@ -13,7 +13,7 @@ services:
   odysseus:
     build: .
     ports:
-      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
+      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7011}:7000"
     volumes:
       - ${APP_DATA_DIR:-./data}:/app/data:z
       - ${APP_LOGS_DIR:-./logs}:/app/logs:z
@@ -58,6 +58,11 @@ services:
       - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24}
       - ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1}
       - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
+      - ODYSSEUS_UNATTENDED_MODE=${ODYSSEUS_UNATTENDED_MODE:-false}
+      - ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS=${ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS:-1}
+      - ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT=${ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT:-0}
+      - ODYSSEUS_CAPTURE_MODEL_REQUESTS=${ODYSSEUS_CAPTURE_MODEL_REQUESTS:-0}
+      - ODYSSEUS_MCP_EMAIL_OWNER=${ODYSSEUS_MCP_EMAIL_OWNER:-}
       - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
       - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
       - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
@@ -65,9 +70,16 @@ services:
       - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
       - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_EDITOR_DRAFT_MAX_BYTES=${ODYSSEUS_EDITOR_DRAFT_MAX_BYTES:-268435456}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
       - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
+      # Host workspace translation is opt-in. Keep the public compose file
+      # user-neutral; configure these in a local .env or use the host-workspace
+      # overlay with ODYSSEUS_HOST_WORKSPACE_DIR.
+      - ODYSSEUS_WORKSPACE_HOST_ROOT=${ODYSSEUS_WORKSPACE_HOST_ROOT:-}
+      - ODYSSEUS_WORKSPACE_CONTAINER_ROOT=${ODYSSEUS_WORKSPACE_CONTAINER_ROOT:-/workspace}
+      - ODYSSEUS_WORKSPACE_DEFAULT=${ODYSSEUS_WORKSPACE_DEFAULT:-}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

docker-compose.yml

@@ -2,7 +2,7 @@ services:
   odysseus:
     build: .
     ports:
-      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
+      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7011}:7000"
     volumes:
       - ${APP_DATA_DIR:-./data}:/app/data:z
       - ${APP_LOGS_DIR:-./logs}:/app/logs:z
@@ -47,6 +47,11 @@ services:
       - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24}
       - ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1}
       - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
+      - ODYSSEUS_UNATTENDED_MODE=${ODYSSEUS_UNATTENDED_MODE:-false}
+      - ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS=${ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS:-1}
+      - ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT=${ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT:-0}
+      - ODYSSEUS_CAPTURE_MODEL_REQUESTS=${ODYSSEUS_CAPTURE_MODEL_REQUESTS:-0}
+      - ODYSSEUS_MCP_EMAIL_OWNER=${ODYSSEUS_MCP_EMAIL_OWNER:-}
       - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
       - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
       - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
@@ -54,9 +59,16 @@ services:
       - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
       - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_EDITOR_DRAFT_MAX_BYTES=${ODYSSEUS_EDITOR_DRAFT_MAX_BYTES:-268435456}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
       - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
+      # Host workspace translation is opt-in. Keep the public compose file
+      # user-neutral; configure these in a local .env or use the host-workspace
+      # overlay with ODYSSEUS_HOST_WORKSPACE_DIR.
+      - ODYSSEUS_WORKSPACE_HOST_ROOT=${ODYSSEUS_WORKSPACE_HOST_ROOT:-}
+      - ODYSSEUS_WORKSPACE_CONTAINER_ROOT=${ODYSSEUS_WORKSPACE_CONTAINER_ROOT:-/workspace}
+      - ODYSSEUS_WORKSPACE_DEFAULT=${ODYSSEUS_WORKSPACE_DEFAULT:-}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

docker/entrypoint.sh

@@ -96,16 +96,7 @@ repair_bind_mount_ownership() {
 # Repair image-owned writable paths without walking into bind-mounted host
 # trees, then repair the app-owned mount roots separately.
 repair_app_tree_ownership
-# Docker creates the parent of the HuggingFace bind mount as root before this
-# entrypoint runs. Repair only the parent directory itself so app-user caches
-# such as /app/.cache/vllm and /app/.cache/flashinfer can be created without
-# recursively walking the mounted model cache.
-chown "$PUID:$PGID" /app/.cache 2>/dev/null || true
-# The Hugging Face cache can contain hundreds of gigabytes and is a nested
-# mount with its own ownership contract. Repair its mount root so new cache
-# entries are writable, but never traverse or rewrite existing model files.
-chown "$PUID:$PGID" /app/.cache/huggingface 2>/dev/null || true
-for dir in /app/data /app/logs /app/.ssh /app/.local; do
+for dir in /app/data /app/logs /app/.ssh /app/.cache/huggingface /app/.local; do
     repair_bind_mount_ownership "$dir"
 done
 

docker/host-network.yml

@@ -0,0 +1,21 @@
+# High-trust host network access. Enable only when the Odysseus agent needs
+# host-native LAN/VPN/mDNS behavior that Docker bridge networking cannot
+# provide. Linux only; Docker Desktop does not provide equivalent host
+# networking semantics.
+#   COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml:docker/host-network.yml
+#   APP_PORT=7011
+services:
+  odysseus:
+    network_mode: host
+    ports: !reset []
+    environment:
+      - APP_PORT=${APP_PORT:-7011}
+      - APP_BIND=${APP_BIND:-0.0.0.0}
+      - SEARXNG_INSTANCE=${ODYSSEUS_HOST_NETWORK_SEARXNG_INSTANCE:-http://127.0.0.1:8080}
+      - CHROMADB_HOST=${ODYSSEUS_HOST_NETWORK_CHROMADB_HOST:-127.0.0.1}
+      - CHROMADB_PORT=${ODYSSEUS_HOST_NETWORK_CHROMADB_PORT:-8100}
+      - ODYSSEUS_CONTAINER_NETWORK_MODE=host
+    command:
+      - sh
+      - -c
+      - exec uvicorn app:app --host "$${APP_BIND:-0.0.0.0}" --port "$${APP_PORT:-7011}"

docker/host-workspace.yml

@@ -0,0 +1,11 @@
+# High-trust host workspace access. Enable only when the Odysseus agent should
+# work on a host directory outside the container's normal /app/data sandbox.
+#   COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml
+#   ODYSSEUS_HOST_WORKSPACE_DIR=/absolute/host/path
+#   ODYSSEUS_HOST_WORKSPACE_MOUNT=/host/workspace
+services:
+  odysseus:
+    volumes:
+      - ${ODYSSEUS_HOST_WORKSPACE_DIR:?set ODYSSEUS_HOST_WORKSPACE_DIR}:${ODYSSEUS_HOST_WORKSPACE_MOUNT:-/host/workspace}:rw,z
+    environment:
+      - ODYSSEUS_HOST_WORKSPACE_MOUNT=${ODYSSEUS_HOST_WORKSPACE_MOUNT:-/host/workspace}

docs/AGENT_TURN_CONTRACT.md

@@ -0,0 +1,75 @@
+# Agent turn contract
+
+Scope: product Agent turns on 7011. Environment-owned native/TUI bridges retain
+their existing execution contract. No model weights or training settings change.
+
+## Boundaries
+
+1. `src/turn_contract.py` classifies capabilities, including explicit compound
+   requests and referential follow-ups. Classification is selection, not permission.
+2. `routes/chat_routes.py` resolves toggles, privileges, global/plan/incognito
+   restrictions, fixture restrictions and available schema inventory before
+   freezing the offered set. Web enabled alone does not select web tools.
+3. `TurnContract` checks `required <= offered <= executable`, stores immutable
+   serialized schema copies, and records unavailable requirements. An unavailable
+   request stops without inference or substitution; unknown actions ask for clarity.
+   Exact account-discovery requests narrow selection to account metadata only;
+   compounds retain their declared family scope. Media operations declare their
+   existing tool dependencies rather than falling back to shell generation.
+4. The agent's prompt/schema route and fallback use that same logical scope.
+   Native versus textual serialization remains model-specific. Answer-only phases
+   can suppress tool calls without granting a different scope.
+   Contract turns preserve the already-compacted conversation and tool-call/result
+   IDs. The standalone specialist prompt's latest-message-only behavior is not used
+   for these product turns. Prompt domains also come from the contract.
+   Accepted in-scope calls retain their model-provided arguments and native IDs;
+   the explicit-intent fallback must not overwrite them with the whole user turn.
+5. The context-bound dispatcher checks membership **and** existing runtime policy,
+   owner restrictions and exact-action approvals. A contract is not authorization
+   to bypass those gates. Contract work bypasses terminating legacy shortcuts.
+6. `_AgentRenderState` explicitly identifies streamed versus canonical output.
+   Later synthesis transfers ownership with turn-scoped replacement. The frontend
+   reconciles visible DOM, not just accumulated strings; tool evidence is retained.
+   Ownership is included in saved metrics and `message_saved` events.
+   History and resume honor replacement scope. Single-capability turns retain
+   canonical output: an always-synthesize trial caused a live notes loop and was
+   reverted. Compound turns cannot terminate after only one capability's result.
+
+## Verification
+
+Use the project's configured Python environment, not an unrelated system Python:
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/python -m pytest -q \
+  tests/test_turn_contract.py tests/test_turn_contract_integration.py \
+  tests/test_agent_turn_contract_boundaries.py tests/test_turn_rendering_js.py \
+  tests/test_contract_prompt_conversation.py tests/test_product_turn_contract_route.py \
+  tests/test_contract_explicit_fallback.py \
+  tests/test_history_resume_rendering_js.py \
+  tests/test_chat_route_tool_policy.py tests/test_tool_policy.py \
+  tests/test_frontend_module_version_parity.py
+node scripts/verify_agent_turn_contract.mjs --max-turns 80 --total-ms 900000
+```
+
+The browser verifier uses `sft_alex_creator` and actual 7011 Agent controls. It
+captures request toggles, SSE contract/tool events, visible output and persisted
+history. Ten families have four initial/follow-up Web-toggle combinations.
+Blocked or unrun cases are not passes. Email requires verified fixture isolation;
+do not enable global fixture mode on the user's live service to make a test pass.
+
+## Remaining limits
+
+- Classification is deterministic and vocabulary-based, not a proof of semantic
+  understanding. Add independent behavior examples for confirmed misses.
+- Schema registration and policy permission do not guarantee a remote provider
+  stays healthy throughout a turn. Runtime failure must remain visible.
+- Separate tool/argument errors, tool-service failures, rendering failures and
+  verifier defects in reports. Do not infer model accuracy from routing alone.
+- Canonical summaries can still ignore presentation constraints such as a
+  requested item count. Do not count those as full functional passes. Forcing an
+  extra model round is not a validated general repair for this deployed model.
+- Keep all imports of a local JS module on the same URL identity. Distinct query
+  versions instantiate separate module state even when source files are identical.
+
+Live baseline and current matrix results are in `reports/agent-turn-contract-*`.
+The implementation is not a claim that every family has passed live verification.

docs/BACKGROUND_TOOL_JOBS.md

@@ -0,0 +1,55 @@
+# Background research → originating chat
+
+Chat `trigger_research` calls carry a **dispatcher-supplied** `origin_chat_id`.
+The research start route verifies chat ownership before registering a durable
+`background_tool_jobs` row and starting the existing research service. Panel
+jobs have no origin and never inject a chat reply.
+
+- Chat default: **2 rounds**, 120-second *soft* research budget. Explicit
+  deeper/Auto rounds regain the normal research time budget. Panel defaults
+  remain unchanged. This is not a guaranteed two-minute wall-clock deadline.
+- A completion callback stores the report and sources. A startup worker also
+  reconciles missed callbacks and research errors/restarts.
+- When the origin has no active foreground/detached run, its model summarizes
+  the report with thinking off and no tools. An outer 75-second deadline also
+  bounds model-slot waits. If synthesis is unavailable, deliver an honest
+  notice plus the report link; preserve the evidence for follow-ups.
+- Message and delivery marker commit in one transaction with a deterministic
+  message ID. Report context is stored in server message metadata and injected
+  as untrusted evidence in regular and compact model history. Long excerpts
+  are explicitly marked; the saved full research report remains accessible.
+- The browser polls owner-scoped `/api/research/chat-jobs/{chat_id}`, appending
+  unseen message IDs only when that chat is current and not streaming. No
+  transcript replacement or forced navigation. Reloaded history deduplicates.
+- Chat uses the existing agent-thread rail and expandable rows. The compact
+  header shows status and a right-aligned BG task label with the shared whirlpool
+  while running; expanding reveals topic, phase/round, source count and report
+  link. Rows update in place, preserving expansion/focus while chat streams.
+  Completed rows remain visible; zero-source runs show a warning, not success.
+  Progress polling excludes reports and internal fields.
+
+Other tools are **not automatically backgrounded**. The durable handoff can be
+reused, but each future producer needs explicit launch/result/permission wiring.
+
+## Verification
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/pytest -q tests/test_background_tool_jobs.py tests/test_research_chat_runtime.py
+node --test tests/backgroundToolJobs.test.mjs
+node scripts/verify_background_delivery_isolation.mjs
+node scripts/verify_background_research_cards.mjs
+node scripts/verify_background_research_chat.mjs
+```
+
+The last script uses disposable `sft_alex_creator` chats and real research/model
+calls, then removes only its own reports/chats. Do not use real-user mutations.
+It checks two-round launch, continued chat, automatic arrival, no transcript
+rebuild/duplicates, reload, and a follow-up. Inspect retained report excerpts
+and generated summary when it fails; do not equate job launch with good research.
+
+Initial live runs verified delivery/navigation/follow-ups but exposed a summary
+attempt-count bug (fixed: helper requires **1 attempt**, not `max_retries=0`).
+A later full run was interrupted by an inference endpoint outage. The corrected
+summary path separately passed a real-model evidence/limitations/citation probe.
+All targeted Python tests passed (441); real DOM isolation checks passed. A clean
+full live run with useful retrieved evidence remains to be recorded.

docs/CLEAN_LOOP_V3_EXPERIMENT_20260909.md

@@ -0,0 +1,99 @@
+# No-RAG clean loop: first diagnostic
+
+## Setup
+
+No live UI, service configuration, or weights changed. The standalone loop sends
+conversation history, native assistant calls and matching tool results directly
+to the served pre-Heretic model. It never rewrites queries, invents calls, swaps
+families, or strips output. Invalid calls return errors. Six executions per turn
+and seven model rounds bound the test.
+
+Both arms use temperature 0, thinking disabled, 768 output tokens, and the
+original tool-work evaluator's `tools_for_mode(..., 'compact_contract_v3')`.
+This matters: the app's plain compact scrubber deletes descriptions, whereas v3
+retains empirically tested micro-hints. Previous plain-compact tests were not
+exact reproductions of the passing benchmark setup.
+
+The 76 tools come from the current app's ten-family inventory, transformed by
+the original v3 builder. This is not a byte-identical frozen 99-tool benchmark
+inventory or proof of training-data identity. The report records schema and
+builder hashes. No schemas are invented for this experiment.
+
+- **Stable:** same compact inventory on every turn, irrespective of spelling.
+- **Routed:** same loop, but existing `requested_capabilities` chooses inventory
+  each turn. This isolates that selector; it is not the complete production RAG
+  or Agent UI path. Other production normalizers are absent in both arms.
+- Private records are synthetic. No real private dispatcher is imported.
+  Only fixture reads and optional public SearXNG calls execute. Other operations
+  return explicit errors, so this does not validate their functionality.
+- Live search sends the exact model query to local SearXNG Bing/Yep, bypassing
+  app query rewriting/filtering. Source results may vary between arms.
+
+## Observations, not a blind score
+
+| Case | Stable compact inventory | Selector arm |
+|---|---|---|
+| `whats the current stock mraket` | Selected `web_search`, query `current stock market` | Offered zero tools; declined live lookup |
+| Exact seeded failed exchange, then `can you look up` | Searched with corrected query | Also searched with corrected query |
+| Summarize search, explicitly no tools | Answered without tools or permission failure | Same |
+| Calendar → email → calendar | Recalled second event at 14:30 | Same |
+| Notes → second note → what does it say | Correct `view` ID and content | Same after fixture correction |
+| Deliberately irrelevant search result | Did not automatically retry | Did not automatically retry |
+| User asks for a better source | Refined and executed another search | Proposed search was not offered and was rejected |
+| Web-disabled lookup | Attempted network access via bash; sandbox rejected it | Invented unsupported current market news without tools |
+
+The initial stable stock answer listed sources, not current index values. It
+does not establish that the market question was fully answered. Its subsequent
+`can you look up` elicited clarification after it had already searched. The
+separate seeded replay removes that differing-history confound.
+
+The first notes fixture incorrectly accepted `get/read`, not the real `view`
+action. Both models selected the correct action, but the fixture rejected it.
+Those six original turns are invalid for execution comparison. A corrected
+six-turn rerun succeeded in both arms; the failed evidence is retained.
+
+Web-off results are a release blocker: removing named web tools alone does not
+enforce network denial across general-purpose tools. The fixture prevented real
+execution, but any UI integration must use the real cross-tool permissions and
+clearly communicate unavailable capabilities. Neither arm is ready for a live
+switch. Source recovery and grounded completion also remain weak.
+
+## What this changes
+
+There is direct evidence that the selector can withhold needed tools, and that
+the model can repair the misspelled query itself when offered the tool. Clean
+history also supports the tested topic switches without synthetic substitutions.
+This supports continuing the clean-path experiment, not retraining or declaring
+the UI fixed. Full inventory is slower in these requests; overlapping runs and
+different source content prevent a controlled latency conclusion.
+
+Next: integrate the clean loop behind a test-only UI profile with real permission
+enforcement and one renderer, preserving the v3 contract. Test live read-only
+follow-ups and explicit Web-off behavior before any rollout. Separately compare
+a generic evidence-check/retry instruction on the weak-result fixture; do not
+manufacture a retry query in the harness.
+
+## Reproduce
+
+Eight boundary tests pass:
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/pytest -q tests/test_clean_tool_loop.py
+```
+
+Run with a fresh report filename (existing evidence is never overwritten):
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/python scripts/test_clean_tool_loop.py --live-search --report reports/clean-loop-v3-new-run.json
+```
+
+Evidence:
+
+- `reports/clean-loop-v3-20260909.json`: original 24 turns; notes fixture caveat above.
+- `reports/clean-loop-v3-stock-seeded-20260909.json`: four matched seeded follow-up turns.
+- `reports/clean-loop-v3-notes-fixture-corrected-20260909.json`: corrected six notes turns.
+
+Each report retains model requests, responses, offered inventory and execution
+results. The `completed` status means the request loop finished, **not** that
+the answer passed functional evaluation. These are synthetic/public traces, not
+private user conversations. This test does not measure UI rendering or streaming.

docs/CLEAN_V3_UI_PREVIEW.md

@@ -0,0 +1,220 @@
+# Tools v3 — No-RAG preview
+
+Select this endpoint in the 7011 model picker, with model
+`odysseus-qwen3.5-tools-pre-heretic`. This endpoint owns its complete tool loop
+and enters Agent mode server-side on every turn, including ambiguous follow-ups;
+it does not depend on the legacy per-message intent classifier. Start a new chat
+for an uncontaminated comparison. Enable Web for searches. Clean routing is
+owned by the exact model identity, so both the normal `preheret` endpoint and
+the `cleanv3` alias use this runtime. Every other model remains on legacy RAG.
+
+Endpoint ID: `cleanv3`. Its base URL uses the same inference server's Tailscale
+DNS name, `http://odysseus.tailb895f4.ts.net:18182/v1`, to distinguish it from
+the original IP-address route when existing chats omit endpoint IDs.
+
+## Implementation
+
+- `src/clean_agent_preview.py` is a separate streamed native-tool loop, entered
+  before legacy routing and substitutions. It uses real authenticated tool
+  dispatch, the tool-work `compact_contract_v5` builder, temperature 0,
+  and thinking disabled. No weights change or inference server was started.
+- The offered tool inventory is stable except for permissions/toggles. Safe,
+  explicit personal creates/updates are enabled for notes, tasks, calendar,
+  memory, skills and documents. Destructive operations, shell/code, outbound
+  email, browser interaction, deployment/admin changes and unrelated-family
+  write substitution remain blocked. No tool or argument substitution is
+  applied by the loop.
+- Native calls and matching results persist in `clean_v3_turn` metadata so
+  follow-ups use actual evidence. History retains at most eight complete turns,
+  trimming oldest whole turns for size; individual outputs cap at 8000 chars.
+- Real search still uses the existing search backend and its provider handling;
+  this does not claim that provider quality or every backend transform is fixed.
+- All routing, privileges and default settings outside this exact Odysseus model
+  remain unchanged. The loop has six execution/eight-round limits.
+- Write completion is evidence-bound: affirmative success text is replaced
+  unless a private-write tool succeeded during the turn. Proposed call batches
+  are policy-preflighted atomically, so a batch containing a blocked operation
+  cannot partially execute before denial.
+
+## Verification
+
+399 focused Python tests passed after route integration. Browser runs r1/r2
+accidentally exercised the old loop and are not preview evidence. The runner
+now explicitly asserts `selection_mode=clean_compact_v3_preview`.
+
+`reports/clean-v3-live-ui-r3-20260909.json` confirms the preview route, real notes
+execution, correct repetition from history, successful search and no-tool
+summary, plus visible incremental growth. Its notes assertions were for the
+old routed contract: they prohibited offering web tools even with Web enabled,
+and required another notes call for a verbatim repeat. The updated preview
+checks permit stable offers and accept an exact match to the preceding saved
+answer without re-execution; execution permissions are still asserted.
+
+`reports/clean-v3-live-ui-r4-20260909.json` is the corrected four-turn check,
+including notes with Web off and search with Web on: **4/4 passed**, with the
+preview selection mode explicitly confirmed on every turn.
+These are UI smoke tests, not all-family or factual-answer benchmark scores.
+
+## Disable
+
+Disabling only endpoint `cleanv3` removes the duplicate picker alias; it does
+not disable this model-owned runtime. To roll back the runtime, revert the exact
+model route in `routes/chat_routes.py`. Do not delete weights, adapters, or user
+chats. The v3 schema builder dependency is
+`/home/pewds/odysseus-tool-work/scripts/eval_alltools_unseen_compare.py` and its
+schema-dropout helper; preserve those with this deployment.
+
+## Expanded UI checks — 2026-09-09
+
+24 additional turns completed through the preview: 23 automated passes and one
+checker false alarm. The Cookbook follow-up correctly shortened the previous
+six-server result to the first three requested names without another call. The
+checker required either a fresh call or a verbatim repeat; manual inspection
+confirmed the requested subset. Raw failure evidence is retained, not rescored.
+
+Covered notes/misspellings/second-note selection, calendar/second-event time,
+tasks, documents, memory, skills, Cookbook listing, misspelled search, and Web
+toggle changes. Cross-family flows passed: Germany news → “whats my notes”,
+notes → “seach current stock mraket news”, and calendar → “now show my noes”.
+The model chose `current stock market news` itself. Every completed turn's
+audit confirmed the preview mode. Search source factual accuracy is not graded
+by this suite, and successful reads do not establish mutation coverage.
+
+Email was separately attempted but the test guard stopped it because the
+stable offered inventory exceeded its metadata-only verified scope. Email
+therefore remains unverified in this expanded run; the guard was not weakened.
+No production code, service settings or weights changed during these tests.
+
+Evidence under `reports/`:
+
+- `clean-v3-broader-ui-20260909.json`: 16 turns, 15 automatic passes, Cookbook caveat.
+- `clean-v3-topic-switch-ui-20260909.json`: 6/6 passed.
+- `clean-v3-second-note-ui-20260909.json`: 2/2 passed.
+- `clean-v3-email-notes-ui-20260909.json`: blocked email attempt; notes not run in that file.
+
+## Picker route fix
+
+The previous tests selected sessions through the API, missing a real picker
+bug: local entries were deduplicated by model ID, hiding alternative endpoints
+with the same weights. The picker now uses endpoint+model identity for local
+routes too, displays the endpoint name, and scopes its last-picked send override
+to the current chat. `/api/sessions` returns owner-filtered endpoint identity
+for unambiguous saved URLs, so reload labels do not depend on loading the model
+catalog. Ambiguous identical URLs are not guessed.
+
+The user-authorized chat `ec0683a2-015f-41d7-aa1f-34135c9640cb` was switched to
+`cleanv3` using the authenticated session PATCH API; no messages were inserted
+and no tool actions ran in that chat. Defaults and other chats were unchanged.
+
+The runner's `--picker-route true` starts on the original route, clicks the
+preview in the real picker, sends a greeting, reloads the chat permalink, then
+asks for notes. Early picker/reload reports are incomplete, not passes: their
+label check exposed the unloaded-catalog issue. Focused route/picker/history
+tests: 16 passed.
+
+Final picker test: `reports/clean-v3-picker-reload-r5-20260909.json`, **2/2
+passed**. Real picker click, greeting, permalink reload, and notes follow-up
+all confirmed the preview route. The label survived reload. R4 retained a
+history/DOM mismatch from sending before restored history was ready; the final
+driver explicitly waits for the saved first answer to render before sending.
+This does not claim a general fix for sending during unfinished history loading.
+
+## Native image/VL status
+
+The inference launcher previously set `--limit-mm-per-prompt` to zero images,
+so vLLM rejected attachments before the model saw them. The durable Odysseus
+launcher now permits up to three images per prompt; video remains disabled.
+
+`reports/clean-v3-vl-live-r4-20260909.json` proves the real 7011 attachment
+path, clean compact route, object/color/spatial recognition, permalink reload,
+and ambiguous image follow-up. Those checks pass. Exact OCR of the deterministic
+`ODYSSEUS 42` heading fails in both the untouched Qwen 3.5 9B base and the
+fine-tune, so it remains a base/runtime capability limitation rather than a
+fine-tune regression or harness failure.
+
+The same native path also passes JPEG and lossless WebP transport, object
+recognition, reload, and follow-up grounding. Evidence:
+`reports/clean-v3-vl-jpeg-r1-20260909.json` and
+`reports/clean-v3-vl-webp-r1-20260909.json`. Both remain `partial` only because
+the shared OCR check fails.
+
+## Reversible write check
+
+`scripts/verify_clean_v3_write.mjs` runs against only `sft_alex_creator`. It
+creates one UUID-named note through the real 7011 UI, verifies that exact row,
+requests a destructive bulk deletion, verifies the row still exists, and then
+deletes only its own test row through the authenticated API. The cleanup is
+verified by a 404 lookup.
+
+Final evidence: `reports/clean-v3-write-ui-r8-20260909.json`, **passed**. Both
+turns reported `selection_mode=clean_compact_v3_preview`; creation executed via
+`manage_notes(action=add)`, the destructive action did not execute, and the
+canonical response was “No changes were made.” The earlier r3/r5 files are
+startup/placement failures, while r4/r6/r7 retained genuine intermediate
+harness and verifier failures; none should be interpreted as passes.
+
+## Stateful, search, and email checks
+
+The reversible stateful runner passes all six mutation families in one run:
+calendar, notes, tasks, documents, memory, and skills (**6/6**). Each flow
+creates a UUID-only artifact through the real Agent UI, verifies it by
+owner-scoped API, applies a noun-free correction, verifies persistence, and
+removes only that artifact. A direct database audit found zero active synthetic
+calendar, note, task, or document rows afterward.
+
+The document failure was harness-owned. Compact description dropout left a
+vague free-form `command` field, error envelopes defaulted to exit code 0, and
+the clean loop dropped the active document ID. Compact v5 now exposes only
+required structured `edits`, reports errors truthfully, and executes against
+the request's explicit active document. Fresh document and combined stateful
+runs pass.
+
+Search Web-toggle combinations `00`, `01`, `10`, and `11` pass **8/8** across
+two turns. A web question can no longer silently enable Bash because it says
+“official source”, and an unavailable Web capability exposes no unrelated
+fallback family. The quality suite passes **3/3**: evidence reuse without a
+second call, explicit official-page inspection with `web_fetch`, correction of
+“stock mraket” in actual search arguments, and a truthful unsupported result
+for a synthetic company.
+
+Production-path email reads pass **3/3** through the running email MCP: account
+list, latest inbox list, and referential read of the first result. The report
+retains no account names, addresses, subjects, bodies, prompts, or answers.
+
+Post-fix representative direct/follow-up coverage also passes for every family:
+notes/calendar 4/4, tasks/documents/memory/skills/Cookbook/search/shell 14/14,
+and email 3/3 in its privacy-preserving runner. The combined legacy verifier's
+metadata-only email guard correctly refused its broader stable inventory; that
+stopped report is not counted as a model failure.
+
+Evidence:
+
+- `reports/clean-v3-stateful-all-r3-20260909.json`
+- `reports/clean-v3-stateful-documents-r2-20260909.json`
+- `reports/clean-v3-search-toggle-final-r6-20260909.json`
+- `reports/clean-v3-search-quality-r3-20260909.json`
+- `reports/clean-v3-email-read-r1-20260909.json`
+- `reports/clean-v3-ten-family-tail-postfix-r1-20260909.json`
+
+These checks verify routing, execution, persistence, follow-up, and selected
+answer-quality invariants. They are not yet the sealed all-action ship score.
+
+## Compact v5 and corrected contract evidence
+
+Compact v5 keeps the compact-v3 surface and adds only development-positive
+field hints for Email, Search/Hugging Face quant selection, and Shell/files.
+A Calendar date hint regressed development and was excluded. The Python tool now
+emits one final bare expression, REPL-style, without duplicating explicit
+`print(...)`; this turns otherwise correct computation calls into visible tool
+evidence for all models.
+
+Under frozen scorer `odysseus.contract.v2.5`, development is 327/344 raw
+(95.06%) and 327/336 scorable (97.32%). Sealed blind is 311/344 raw (90.41%)
+and 311/336 scorable (92.56%), with zero reasoning leakage. Calendar, Shell,
+and Tasks remain below the 90% family ship floor, so the model is not yet a
+full benchmark ship candidate.
+
+Fresh post-deploy real-UI evidence passes: stateful flows 6/6, Email 3/3,
+Search quality/recovery 3/3, private browser 3/3, and VL workflow 3/3. The
+Search check accepts a failed attempt only when a later tool succeeds and the
+final answer remains grounded.

docs/REGULAR_MODEL_TOOL_COMPATIBILITY.md

@@ -0,0 +1,125 @@
+# Regular-model tool compatibility
+
+Last verified: 2026-09-09 through the authenticated 7011 Agent UI as
+`sft_alex_creator`.
+
+This is the legacy-RAG track. The exact model
+`odysseus-qwen3.5-tools-pre-heretic` is excluded and remains on its model-owned
+clean compact runtime.
+
+## Current baseline
+
+| Endpoint | Model | Ten-family result | State |
+|---|---|---:|---|
+| DeepSeek | `deepseek-v4-flash` | 10/10 | passed |
+| DeepSeek | `deepseek-v4-pro` | 10/10 | passed |
+| OpenAI | `gpt-5.5` | 10/10 | passed |
+| OpenAI | `gpt-5.6-sol` | 10/10 | passed |
+| OpenAI | `gpt-5.6-terra` | 10/10 | passed |
+| OpenAI | `gpt-5.6-luna` | 10/10 | passed |
+| OpenRouter | `moonshotai/kimi-k3` | 10/10 | passed |
+| OpenRouter | `x-ai/grok-4.5` | 10/10 | passed |
+| OpenRouter | `qwen/qwen3-vl-235b-a22b-instruct` | 10/10 | passed |
+| OpenRouter | `openai/gpt-5-image` | n/a | image generation; chat tools unsupported |
+| Local `100.69.120.65:8062` | `Qwen/Qwen3.5-9B` | not run | endpoint unavailable |
+| Local `100.69.120.65:8062` | `GLM-5.3-Flash-Alis-MLX-4bit` | not run | endpoint unavailable |
+
+The ten-family baseline covers one read-only functional turn each for notes,
+calendar, email accounts, tasks, documents, memory, skills, Cookbook/admin,
+web search, and shell. It verifies the legacy route, expected native tool call,
+execution result, visible UI answer, and absence of reasoning leakage. It is not
+yet a claim that every mutation/action variant, typo, or follow-up passes.
+
+## Typo and follow-up profile
+
+The stricter real-UI profile sends one misspelled read-only request to every
+family, followed immediately by a noun-free reference to the returned result.
+Read-only follow-ups must not call any tool; search follow-ups may either use
+the existing evidence or fetch the prior link. Across the nine chat-capable API
+models, the composited post-repair result is **178/180 turns (98.89%)**:
+
+| Model | Conversation result |
+|---|---:|
+| `deepseek-v4-flash` | 20/20 |
+| `deepseek-v4-pro` | 20/20 |
+| `gpt-5.5` | 20/20 |
+| `gpt-5.6-sol` | 20/20 |
+| `gpt-5.6-terra` | 20/20 |
+| `gpt-5.6-luna` | 18/20 |
+| `moonshotai/kimi-k3` | 20/20 |
+| `x-ai/grok-4.5` | 20/20 |
+| `qwen/qwen3-vl-235b-a22b-instruct` | 20/20 |
+
+Luna's only remaining family miss is a deliberately misspelled Shell request.
+The correct-spelling baseline passes. The harness does not auto-execute a shell
+command to hide that model-owned limitation.
+
+The shared repair recognizes a uniquely misspelled action verb and family noun,
+then seals only declared safe private reads with immutable canonical arguments.
+This repaired Tasks/Documents/Memory and adjacent read families across providers
+without widening mutation or Shell authority. A compact native-tool instruction
+also tells regular API models to map clear typos to a currently offered tool.
+
+Conversation evidence:
+
+- `reports/regular-model-conversation-flash-r3-20260909.json`
+- `reports/regular-model-conversation-remaining-r1-20260909.json`
+- `reports/regular-model-conversation-repair-r1-20260909.json`
+- `reports/regular-model-conversation-shell-r1-20260909.json`
+- `reports/regular-model-conversation-qwen-repair-r1-20260909.json`
+- `reports/regular-model-conversation-qwen-tail-r1-20260909.json`
+- `reports/regular-model-conversation-qwen-search-r1-20260909.json`
+
+Evidence:
+
+- `reports/regular-model-tools-provider-final-r4-20260909.json` — Flash, GPT-5.5, Kimi: 30/30.
+- `reports/regular-model-tools-repair-r3-20260909.json` — Pro and Sol: 20/20; retained Qwen pre-final 9/10 miss.
+- `reports/regular-qwen-vl-full-r4-20260909.json` — Qwen-VL final family-switch run: 10/10.
+- `reports/regular-model-tools-remaining-20260909.json` — Terra, Luna, Grok: 30/30; records unavailable/unsupported models and pre-repair failures.
+- `reports/regular-model-tools-postfix-r1-20260909.json` — post-hardening
+  rerun: nine chat-capable API models passed 90/90 family turns with zero model
+  failures. Its overall status is non-passing only because the two configured
+  local endpoints were offline; the image-only model remains unsupported.
+
+## Family switch and page inspection
+
+The six-turn switch/back flow covers notes → calendar → notes from prior
+evidence → web search → explicit `web_fetch` → calendar from prior evidence.
+All nine API models have a clean 6/6 reproduction (**54/54**). Kimi skipped
+search once in the retained first run and passed a fresh reproduction; that
+variability remains visible instead of being erased.
+
+Evidence:
+
+- `reports/regular-model-switchback-flash-r2-20260909.json`
+- `reports/regular-model-switchback-remaining-r1-20260909.json`
+- `reports/regular-model-switchback-kimi-r1-20260909.json`
+
+## Repair that produced the clean baseline
+
+Regular models no longer inherit up to three stale tool families into every
+explicit new request. Referential follow-ups still resolve from typed recent
+tool evidence, while explicit family switches receive the current family only.
+Safe required reads use `active_capabilities`, so stale offered context cannot
+disable their immutable operation. The stream layer also stops an exact long
+block repeated twice instead of waiting for a provider's full timeout.
+
+The composer no longer treats generic words such as “source”, “system”, “app”,
+or “review” as authority to silently enable Bash. Explicit shell, terminal,
+repository, code-file, and direct coding requests retain workspace
+auto-escalation. This is a shared UI authority fix, not a model-name exception.
+
+Run a bounded subset with:
+
+```sh
+MODELS='deepseek-v4-flash,gpt-5.5' \
+FAMILIES='notes,calendar' WORKERS=2 \
+REPORT_PATH=reports/regular-model-check.json \
+node scripts/verify_regular_model_tools.mjs
+```
+
+Set `PROFILE=conversation` to run the typo plus follow-up profile.
+
+The runner discovers only enabled pinned models (visible cached local models
+when no pins exist), retains no tool outputs or private rows, and deletes only
+the exact sessions it creates.

docs/SEARCH_HARNESS_AUDIT_20260909.md

@@ -0,0 +1,92 @@
+# Search and compact-tool experiment — 2026-09-09
+
+## Decision
+
+Keep the normal routed profile on 7011. The all-tools compact experiment is
+implemented but **disabled**: direct routing success did not translate into a
+working Agent UI. Do not retrain or promote a profile on these measurements.
+
+## Changes
+
+- Short public-web lookups on the target model have an execution budget: two
+  distinct token-normalized searches, one fetch, and up to three browser calls
+  after the two searches. This bounds attempts, not just recovery prose. Existing
+  permissions still apply; this does not make unavailable tools executable.
+- Failed/weak searches reach the model for evaluation and query refinement,
+  instead of the earlier unconditional terminal evidence veto. Some legacy
+  heuristics and official-site shortcuts remain; this is not a completed rewrite.
+- Search providers retain query/engine/date provenance. Unconfigured credentialed
+  fallbacks are skipped. When SearXNG is the sole configured usable provider, Yep
+  on the same instance is an additional fallback.
+- An unavailable warm-only family no longer vetoes an otherwise ordinary reply.
+- The all-tools experiment offers the trained compact inventory subject to
+  permissions. It requires the exact test-owner environment flag and exact model
+  match. The temporary service flag was removed after failed UI testing.
+
+## Evidence and limits
+
+| Measurement | Result | What it establishes |
+|---|---|---|
+| Focused Python regression suite | 401 passed | Covered policy, contract, provider and recovery-budget behavior |
+| Direct family-only compact schemas | 10/10 tool routing | Small public smoke test, not functional or blind accuracy |
+| Direct all-family compact schemas | 10/10 tool routing | Inventory did not break these first calls; roughly 3–4x slower in this run |
+| Full compact Agent UI, revision 3 | All eight turns failed one or more checks | Not suitable for activation; leaks, duplicate/incorrect rendering or missing expected calls |
+| Normal-profile final UI control | Five passed checks, two product failures, one capture error | Not accepted; suite status incomplete |
+| Clean synthetic notes tool-result continuation | Clean answer with both schema sizes | Model can continue correctly on that isolated input, not proof that UI failure is solely harness |
+
+Direct probes used temperature 0; the UI target-model sampling path can cap at
+0.2. Prompts, history and tool-result serialization also differ. Match those
+before attributing UI failures to weights versus harness. Existing UI checks are
+not a grounded factual-answer benchmark. Unit tests are not UI acceptance.
+
+Normal-profile control details: notes initial, both calendar turns, AI search
+initial, and history initial passed the automated checks. Notes follow-up hit a
+Playwright `Network.getResponseBody` capture error and is inconclusive. AI search
+follow-up explicitly requested a summary with no tools, but the contract still
+required `search_browser` and returned a permission failure. The history search
+follow-up failed the visible-leak check. These are separate from source relevance;
+the earlier warm-only fix did not cover classification as an active requirement.
+There is no matched pre-change control establishing a net improvement.
+
+Provider isolation bypassed app relevance filters. Bing general often returned
+broad or unrelated results despite the full query. Google/Mojeek returned no
+results, DDG hit CAPTCHA, and Presearch timed out. Yep returned useful PostgreSQL
+documentation, but was weak or empty for several other questions. Engine health
+and source quality remain unresolved. Fallback cannot help when an earlier weak
+result survives filtering; there is no claim of universal relevance here.
+
+## Reproduce and inspect
+
+- `scripts/audit_search_pipeline.py`: raw provider comparison, no model.
+- `scripts/compare_compact_tool_inventory.py`: read-only model schema comparison;
+  proposed calls are never executed.
+- `scripts/verify_agent_turn_contract.mjs`: real 7011 Agent UI and persisted-history
+  checks. Use the dedicated test account; reports can contain private tool data.
+- `reports/search-provider-isolation.json`, `reports/search-yep-isolation.json`:
+  public provider evidence.
+- `reports/compact-inventory-ablation.json`: direct routing probe.
+- `reports/full-compact-ui-audit-r3.json`: completed rejected UI experiment.
+  Earlier experiment reports include an initialization error and an aborted run;
+  do not combine them into an accuracy score.
+- `reports/routed-control-ui-audit-final.json`: normal-profile control replay;
+  eight attempts, incomplete because of the capture error; failures retained.
+
+Focused suite:
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/pytest -q tests/test_turn_contract.py tests/test_turn_contract_integration.py tests/test_service_search_provider_guards.py tests/test_web_recovery_budget.py tests/test_tool_policy.py
+```
+
+UI replay (read-only prompts, creates test chats):
+
+```sh
+node scripts/verify_agent_turn_contract.mjs --families notes,calendar,search_ai,search_history --pairs notes:11,calendar:11,search_ai:11,search_history:11 --max-turns 8 --total-ms 360000 --turn-ms 45000 --report reports/routed-control-ui-audit-final.json
+```
+
+## Next discriminating test
+
+Replay the same captured UI request directly, preserving sampling, compact
+schemas, history and tool results. Then change one layer at a time. Separately
+score retrieved-source relevance and supported answers. Replace failing generic
+boundaries only when the replay identifies them; do not add rules for individual
+user phrasings or treat successful tool routing as successful execution.

docs/TYPO_ROUTING_AUDIT_20260909.md

@@ -0,0 +1,73 @@
+# Typo-tolerant tool routing audit
+
+The 9B SFT model was not retrained. This audit targets the earlier harness
+stage that decides which complete tool families the model is allowed to see.
+
+## Method
+
+- Source prompts: real `sft_alex_creator` sessions from `a37dcb3b-...` onward.
+- Labels: recorded single-family tool calls, excluding mixed/ambiguous traces.
+- Variants: deletion, adjacent transposition, duplicated character,
+  keyboard-neighbor substitution, and accidental word split.
+- Split: deterministic SHA-256 assignment before scoring (75% dev, 25% blind).
+- Safety: static routing only; no historical mutation or send action is replayed.
+- Acceptance: at least 95% blind exact-family accuracy and below 1% blind
+  wrong-family authorization. Abstention is measured separately.
+
+## Results
+
+| Router | Dev family supplied | Blind family supplied | Blind exact | Blind wrong-family |
+|---|---:|---:|---:|---:|
+| Previous exact rules | 63.64% | 65.69% | — | — |
+| Conservative fuzzy fallback r4 | 96.31% | 98.31% | 96.62% | 0.00% |
+| Final router + safe-read repair | 98.31% | 98.73% | 97.05% | 0.00% |
+
+The fallback runs only for action/lookup-shaped requests, resolves exactly one
+nearby family term, and abstains on ambiguity. Conceptual questions remain
+tool-free. Complete family schemas are still selected by the immutable turn
+contract; fuzzy matching never chooses an individual tool or its arguments.
+
+Authoritative machine reports:
+
+- `reports/typo-tool-routing-baseline-20260909.json`
+- `reports/typo-tool-routing-fuzzy-r4-20260909.json`
+- `reports/typo-tool-routing-final-20260909.json`
+- `reports/post-followup-agent-80-20260909.json`
+- `reports/post-typo-routing-agent-80-20260909.json`
+- `reports/live-typo-agent-20-20260909.json`
+- `reports/live-typo-unresolved-r3-20260909.json`
+- `reports/live-typo-agent-final-20-20260909.json`
+- `reports/post-typo-safe-read-agent-final-80-20260909.json`
+
+## Live 7011 findings
+
+The post-deployment standard matrix passed 80/80 through the real Agent UI.
+The first read-only typo matrix then attempted 17 of 20 planned turns before
+its total-time limit. Initial Notes, Calendar, Email, Tasks, Documents, and
+Cookbook calls passed. Completed failing turns still had the correct family
+and required tool in `turn_contract.offered`; the 9B model sometimes answered
+without calling that offered tool. Memory and Search also exposed timeouts.
+
+This separates three failure classes:
+
+1. **Tool injection:** addressed by conservative fuzzy family routing; blind
+   exact routing is 96.62% with zero blind wrong-family authorizations.
+2. **Required read execution:** a correctly offered safe list/refresh tool can
+   still be skipped by the model, especially after a typo or on “list those
+   again” follow-ups. This should be handled by the generic deterministic
+   safe-read path, not additional prompt-specific hints.
+3. **Runtime timeout:** Search and one Memory follow-up require loop/backend
+   diagnosis. A timeout is not counted as a model-accuracy or routing result.
+
+The generic safe-read parser and search-family precedence were then repaired.
+The previously unresolved Calendar, Email, Search, and Shell/Files cases passed
+8/8. The complete typo matrix passed 20/20, including initial requests and
+follow-ups for all ten families. The final standard Agent UI compatibility
+matrix passed 80/80 across family, Web-toggle, and follow-up combinations.
+
+The broad routing regression suite passed 458 tests. The model was not
+retrained and no DeepSeek API was used: the measured defect was in harness
+family selection and deterministic safe-read execution, upstream of the
+model. All 1,535 unique labeled historical turns were statically audited to
+mine failure categories. Historical write/send/delete actions were not replayed
+against live data; live verification used the deduplicated read-only matrices.

docs/skills-lifecycle.md

@@ -0,0 +1,26 @@
+# Skills lifecycle
+
+The UI exposes All, Built-in, Approved, and Draft. Draft includes archived
+records so they remain inspectable and recoverable. Built-ins are not audited.
+Approved means published, passing, at the configured confidence threshold,
+and not marked unnecessary. Baseline speed measurements remain evidence, not
+an additional hidden UI approval gate.
+
+Automatic audits process at most eight eligible records at a time, oldest first.
+New records are eligible immediately; inconclusive checks retry after a day;
+failed repairs retry after a week. Passed, duplicate-skipped, and archived records
+are excluded. Existing daily Skills Audit tasks drive this queue. Their quiet
+window deferrals propagate to the scheduler rather than becoming task failures.
+Automatic runs use background model scheduling. Existing self-repair and teacher
+repair stages remain in place; failed candidates remain drafts.
+
+The skill index advertises short descriptions; the agent loads a relevant full
+procedure on demand and applies already-injected procedures directly. Extraction
+prefers verified discoveries and specific workarounds over routine tool usage.
+
+Reference reviewed: NousResearch/hermes-agent, MIT license, commit
+cfdbbb6e35010ace89fbe8243ee82fa4de143e10, cloned to
+/home/pewds/hermes-skills-reference. In particular tools/skills_tool.py and
+agent/prompt_builder.py use progressive disclosure and task-triggered procedure
+loading. These changes adapt that approach to Odysseus's existing registry;
+no Hermes implementation code was copied.

launcher.py

@@ -14,13 +14,6 @@ import threading
 import time
 import webbrowser
 
-# PyInstaller multiprocessing children re-enter this executable with a private
-# bootstrap argument. Consume it before splash/UI or application imports so a
-# spawn-based worker does not relaunch the full desktop application.
-if __name__ == "__main__":
-    import multiprocessing
-    multiprocessing.freeze_support()
-
 # Define a dummy NullWriter to suppress standard stream crashes (isatty etc.) in GUI mode
 class NullWriter:
     def write(self, text):
@@ -137,7 +130,7 @@ if __name__ == "__main__":
     from app import app
 
     bind_host = os.getenv("APP_BIND", "127.0.0.1")
-    bind_port = int(os.getenv("APP_PORT", "7000"))
+    bind_port = int(os.getenv("APP_PORT", "7011"))
     url = f"http://{bind_host}:{bind_port}"
 
     if getattr(sys, 'frozen', False):

package-lock.json

@@ -4,8 +4,10 @@
   "requires": true,
   "packages": {
     "": {
+      "name": "odysseus",
       "devDependencies": {
-        "@antithesishq/bombadil": "^0.7.0"
+        "@antithesishq/bombadil": "^0.7.0",
+        "@playwright/test": "^1.62.1"
       }
     },
     "node_modules/@antithesishq/bombadil": {
@@ -17,6 +19,69 @@
       "bin": {
         "bombadil": "bin/bombadil.js"
       }
+    },
+    "node_modules/@playwright/test": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz",
+      "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright": "1.62.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz",
+      "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.62.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz",
+      "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      }
     }
   }
 }

package.json

@@ -1,9 +1,18 @@
 {
+  "name": "odysseus",
+  "private": true,
   "repository": {
     "type": "git",
     "url": "https://github.com/odysseus-dev/odysseus.git"
   },
+  "scripts": {
+    "test:photo-editor": "playwright test --config tests/e2e/playwright.config.js",
+    "test:photo-editor:install": "playwright install chromium firefox webkit",
+    "test:photo-editor:firefox": "PHOTO_EDITOR_E2E_BROWSER=firefox playwright test --config tests/e2e/playwright.config.js",
+    "test:photo-editor:webkit": "PHOTO_EDITOR_E2E_BROWSER=webkit playwright test --config tests/e2e/playwright.config.js"
+  },
   "devDependencies": {
-    "@antithesishq/bombadil": "^0.7.0"
+    "@antithesishq/bombadil": "^0.7.0",
+    "@playwright/test": "^1.62.1"
   }
 }

plans/ODYSSEUS_TOOL_HARDENING_PLAN.md

@@ -0,0 +1,326 @@
+# Odysseus Tool Runtime Hardening Plan
+
+## Objective
+
+Ship `odysseus-qwen3.5-tools-pre-heretic` with one compact, model-specific tool
+runtime that supports realistic multi-turn use. Keep the existing RAG runtime
+unchanged for every other model. Prove routing, execution, answer quality,
+follow-ups, safety, rendering, latency, and native image/VL understanding through
+the real 7011 Agent UI.
+
+Current evidence is a baseline, not a ship claim:
+
+- Corrected v2.5 + compact-v5 development is 327/344 raw (95.06%) and
+  327/336 scorable (97.32%). Sealed blind is 311/344 raw (90.41%) and
+  311/336 scorable (92.56%), with zero reasoning leakage.
+- Notes, Skills, and Cookbook/admin clear 95% scorable blind. Calendar 87.5%,
+  Shell/files 86.11%, and Tasks 87.5% remain below the 90% family ship floor.
+- Compact-v5 hints improved Email, Search/HF quant, and Shell on development;
+  a Calendar hint regressed and was rejected rather than shipped.
+
+- Ten-family focused baseline: 19/20 functional and 20/20 routing/execution.
+- Typo and cross-family read flows: 26/26 passed.
+- Real use exposed untested write correction and search-to-fetch follow-ups.
+- Email production access, browser interaction, search quality, and broader
+  multi-turn mutations are not yet proven.
+- Nine enabled chat-capable regular API models pass the ten-family read-only
+  legacy-RAG baseline (90/90 combined). Their stricter typo/follow-up profile is
+  178/180 turns: eight models are 20/20 and Luna is 18/20 due only to its
+  misspelled Shell request. One pinned image-generation model is explicitly
+  unsupported and two visible local models are currently offline.
+- Native VL object/spatial recognition and reload follow-up pass. Exact OCR
+  fails equally on the fine-tune and untouched 9B base and remains unresolved.
+  PNG, JPEG, and WebP transport all pass.
+- Reversible create/correct/API-verify/cleanup flows pass 6/6 across every
+  stateful family.
+- Search Web-toggle combinations pass 8/8 and the focused quality suite passes
+  3/3. Production-path email account/inbox/referential reads pass 3/3.
+- The latest regular-model regression is 90/90 across the nine enabled
+  chat-capable API models, with zero failed model turns; two local endpoints
+  remain offline and the image-only model is unsupported.
+- The Epictetus OMLX endpoint was recovered after an unsupported
+  `qwen3_5_mtp` model load wedged the server. Its supported Qwen 27B 4-bit
+  model passes the ten-family real-7011 legacy-RAG smoke 10/10; the unsupported
+  MTP artifact is recorded as a runtime limitation rather than a timeout.
+- Fresh compact-v5 UI regressions pass stateful 6/6, Email 3/3, Search 3/3,
+  private-browser 3/3, and VL workflow 3/3.
+- The exact-model, family-scoped compact runtime now passes 20/20 direct and
+  same-family turns across all ten families on the real 7011 Agent UI. A
+  separate 36/36 robustness run passes misspellings, bounded repeats, browser
+  and news continuation, ambiguous follow-ups, family switchbacks, and a
+  greeting before a tool request.
+- The mobile active-email editor path passes 1/1: `Write reply this email`
+  offers and executes only `update_document`, mutates the open draft, and
+  preserves its reply headers and quoted thread.
+- The active-editor classifier now also covers short mobile wording without a
+  pronoun (`Write reply` / `Draft a reply`) while explicit note, code, file, and
+  new-object requests retain their own families. Whole-draft requests are bound
+  to the sole offered `update_document` writer until one successful write, then
+  tools are removed for the confirmation round. The deployed real-route email
+  regression passes 3/3—including the exact unspecified `Write reply to this
+  email` form—with one write, verified mutation, and preserved reply headers.
+  Clean-v3 now also emits the established `doc_update` event and flattened
+  document metadata on `tool_output`, so a successful database write updates
+  the already-open editor instead of leaving stale UI beside a success message.
+- The client now reuses the existing assistant bubble for `agent_step` round 1
+  instead of replacing it before the first token. A real-7011 sampled
+  greeting-to-Notes conversation passes 2/2 with stable first-round DOM
+  identity; round 2+ remains the only continuation-bubble path.
+- Clean-runtime metrics now expose provider-counted initial injected tokens,
+  all-round input/output, TTFT, tok/s, schema count, agent rounds, and tool-call
+  count. A real 7011 browser run passes 2/2 and visibly renders compact footers
+  plus the full details popup; the sampled Notes turns streamed progressively.
+- The deployed startup bottleneck was an unindexed quadratic transcript-FTS
+  reconciliation. Live-database import fell from about 36 seconds to 0.54
+  seconds; 7011 now answers in about 3 seconds after a controlled restart.
+- A controlled identical-compact comparison already proves the fine-tune's
+  accuracy benefit: 94.48% (325/344) versus the untouched base's 77.91%
+  (268/344). Raw serving speed is effectively tied, so product speed comes
+  from the compact contract and fewer failed/redundant rounds.
+- A fully merged 10,000-row category-repair candidate reached 97.32% scorable
+  development but only 92.26% scorable sealed blind. Calendar (87.5%), Tasks
+  (87.5%), and Shell/files (86.11%) remained below the family floor, so it was
+  rejected and not deployed. Compact-v4/full development A/Bs did not improve
+  Calendar or Tasks over compact-v5; full-schema Shell also fell from 97.22%
+  to 94.44%. This rules out compactness as the primary cause of the remaining
+  blind gaps and supports keeping the compact contract.
+
+## Non-negotiable architecture rules
+
+1. Runtime selection follows exact model identity. The trained Odysseus model
+   uses the clean compact runtime across endpoint aliases; all other models use
+   legacy RAG. Add a regression test for both sides.
+2. Resolve permissions, toggles, and available backends once per turn. Produce
+   one immutable contract satisfying `required ⊆ offered ⊆ executable`.
+3. Never offer a tool that the preview policy will categorically reject. Add a
+   contract self-check covering every offered action/effect combination.
+4. Follow-ups consume typed prior evidence: native call, result, success state,
+   family, and object identifiers. Do not infer continuity from keyword RAG.
+5. Contextual write authority may revise only a recently proven object in the
+   same family. It may not authorize a new object, another family, a destructive
+   action, or an external side effect.
+6. The model chooses tools and valid arguments. The harness validates and
+   executes; it does not silently substitute another family, rewrite arguments,
+   fabricate success, or replace a failed tool with prose claiming completion.
+7. One owner renders each turn: streamed prose or canonical structured output.
+   Never both, and never expose hidden prompts or raw untrusted wrappers.
+8. No exact-prompt production patches. A fix must name the failed layer, add a
+   generic failing invariant test, and cover neighboring cases.
+
+## Failure layers
+
+Every failure is assigned to exactly one primary layer before code changes:
+
+1. **Route:** wrong model runtime or endpoint identity.
+2. **Contract:** required tool absent, forbidden tool present, or toggle drift.
+3. **Model:** wrong/no tool or semantically wrong required arguments despite a
+   correct contract.
+4. **Policy:** valid proposed operation incorrectly allowed or denied.
+5. **Execution:** canonical arguments, backend dispatch, timeout, or result
+   envelope is wrong.
+6. **Evidence:** result is empty, irrelevant, truncated badly, or insufficient.
+7. **Answer:** model misstates or ignores valid tool evidence.
+8. **Rendering:** duplicate, dump-at-end, missing structured output, or stopped
+   stream.
+9. **Performance:** startup, TTFT, tool latency, or oversized context.
+
+Reports store aggregate category, relevant contract/tool metadata, timings, and
+sanitized outputs. Do not copy private hidden benchmark prompts or create a log
+dump that nobody can audit.
+
+## Test matrix
+
+Use the real authenticated 7011 Agent UI and the normal `preheret` picker alias.
+Use `sft_alex_creator` for reversible writes. Never mutate the personal account
+from an automated test.
+
+### A. Every one of the ten families
+
+For calendar, notes, email, tasks, documents, memory, skills, Cookbook/admin,
+search/browser, and shell/files, test:
+
+- direct request;
+- natural misspelling;
+- ambiguous same-family follow-up;
+- switch to another family and back;
+- no-tool greeting before the tool request;
+- requested count/field limit;
+- backend failure rendered truthfully;
+- reload the permalink before a follow-up.
+
+### B. Stateful mutation families
+
+For notes, calendar, tasks, documents, memory, and skills:
+
+- create → verify by API → referential correction → verify;
+- create → list/read → correction → verify;
+- typo correction such as name/date/title without repeating the family noun;
+- correction after one unrelated conversational turn;
+- destructive request is denied atomically;
+- failed write never produces a success claim;
+- cleanup deletes only the UUID-owned test artifact and verifies absence.
+
+### C. Search and browser conversations
+
+- search → summarize existing results without a new call;
+- search → inspect one result with `web_fetch`;
+- poor results → refine query once;
+- insufficient evidence → say so without fabrication;
+- Web toggle combinations `00`, `01`, `10`, and `11` across two turns;
+- private browser open/snapshot/click only after its permission boundary is
+  deliberately enabled and specified; do not smuggle it in via web search.
+
+Grade source relevance, freshness, authority, and whether claims are supported,
+not merely whether `web_search` was called.
+
+### D. Email and shell
+
+- Separate fixture accuracy from production connectivity. A fixture pass cannot
+  promote production email health.
+- Test account listing, inbox listing, reading, and referential follow-up against
+  the configured production-like backend before enabling email actions.
+- Shell remains toggle-gated. Test off/on transitions, canonical raw command
+  dispatch, read-only output, and denial of network/destructive commands.
+
+### E. Rendering and performance
+
+- Assert first visible streamed token, monotonic DOM growth, one final answer,
+  persistence/reload equality, stop behavior, and structured list rendering.
+- Record request preparation, TTFT, tool duration, post-tool TTFT, total time,
+  input/output tokens, and tool-result bytes.
+- Diagnose the 30–40 second 7011 restart separately from inference latency.
+- Bound large calendar/search results before replaying them into later rounds,
+  while preserving IDs and fields needed for follow-ups.
+
+### F. Image/VL recognition
+
+- Attach real PNG, JPEG, and WebP images through the 7011 UI and verify the
+  trained model receives native multimodal message content on its clean route.
+- Test object recognition, visible text/OCR, spatial relationships, charts, and
+  screenshots. Score required facts instead of stylistic wording.
+- Test image → ambiguous follow-up, image → tool request, and tool result → image
+  comparison without requiring the user to attach the same image again.
+- Verify image references survive persistence and permalink reload without raw
+  base64, local paths, or hidden wrappers appearing in chat output.
+- Separate direct model vision from `inspect_media`, browser screenshots, and
+  image generation. The harness must not silently substitute one for another.
+- Compare the fine-tune with its base VL model on the same images to detect
+  whether tool training regressed visual understanding.
+
+### G. Regular-model legacy RAG and tool coverage
+
+- Inventory every enabled non-Odysseus endpoint/model visible in 7011, including
+  its provider, schema mode, native-tool support, context limit, and configured
+  permissions. Do not assume every provider supports the same wire format.
+- Assert that no non-Odysseus model enters the clean-v3 runtime. These models
+  retain the regular RAG/tool loop and are repaired only in that owning path.
+- For each model, test every tool family the effective user policy offers:
+  direct request, misspelling, ambiguous follow-up, family switch, backend
+  failure, and Web/Bash toggle transitions. Record unsupported families as an
+  explicit capability limitation, not a silent pass.
+- Test full schemas versus compact schemas only where both are valid for that
+  model. Store the selected schema mode in every report.
+- Verify provider-native tool calls, textual fallback parsing where required,
+  canonical argument conversion, execution, evidence replay, and rendering.
+- Group fixes by shared legacy-runtime or provider-adapter defect. Do not add
+  model-name prompt exceptions when a transport, schema, or RAG ranking issue is
+  responsible.
+- Maintain a per-model compatibility matrix so adding or changing an endpoint
+  cannot silently regress previously working tools.
+
+## Fix protocol
+
+For each failure:
+
+1. Preserve the raw report and reproduce once on a fresh test session.
+2. Identify the primary failure layer from the taxonomy above.
+3. Add the smallest generic red test at that layer.
+4. Fix the owning module or invariant—not the literal prompt.
+5. Run the focused unit tests, the original scenario, two adjacent scenarios,
+   and the affected family suite.
+6. After a batch of category fixes, rerun the ten-family matrix and legacy-RAG
+   isolation test. Do not rerun training unless the contract and harness are
+   proven correct and failures remain model-owned.
+
+If three failures share a layer, pause case-by-case patching and refactor that
+layer before continuing.
+
+## Execution phases
+
+### Phase 1 — Make the runtime auditable
+
+- Add a sanitized per-turn decision record: model runtime, contract, proposed
+  calls, policy decisions with reason codes, executions, render owner, timings.
+- Add startup/runtime provenance to the UI so a linked chat proves which harness
+  handled it.
+- Add the offered-versus-policy compatibility self-test.
+- Correct stale preview documentation.
+
+### Phase 2 — Build the conversation suite
+
+- Extend the current Playwright verifier with reusable multi-turn scenarios and
+  reversible artifact fixtures.
+- Implement the matrix above, prioritizing search continuations and all
+  stateful corrections because real usage already exposed those gaps.
+- Run independent family groups in parallel, but serialize writes that share a
+  backend or fixture account.
+- Add a small versioned VL fixture set with locally generated, non-private
+  images and deterministic answer keys.
+
+### Phase 3 — Repair by architecture category
+
+- Consolidate model-specific runtime selection in one function.
+- Represent prior successful objects explicitly for referential follow-ups.
+- Align tool capability classification, contract offering, and policy decisions.
+- Standardize tool results into bounded envelopes with source/object IDs.
+- Keep search refinement and evidence sufficiency generic.
+
+### Phase 4 — Accuracy and speed comparison
+
+- Compare the clean fine-tune with the base model using identical compact tools,
+  prompts, toggles, backend state, and semantic scoring.
+- Report functional accuracy, argument accuracy, unsupported success claims,
+  TTFT, total latency, and tokens. Do not compare one model on full schemas and
+  another on compact schemas.
+- Only consider more SFT/RL for failures classified as model-owned after the
+  harness audit.
+
+### Phase 4B — Regular-model repair and verification
+
+- Snapshot the enabled non-Odysseus model inventory.
+- Run the legacy-RAG compatibility matrix in bounded parallel groups, respecting
+  endpoint rate limits and shared backend write serialization.
+- Fix shared harness/provider defects first, then rerun all affected models.
+- Publish separate per-model scores and limitations; do not blend them into the
+  Odysseus fine-tune score.
+
+### Phase 5 — Ship gate
+
+Ship only when:
+
+- every family is at least 90% on sealed functional holdout;
+- overall functional accuracy is at least 95%;
+- realistic follow-up suite is at least 95%, with no repeated failure category;
+- image/VL fixture accuracy does not regress materially from the base model and
+  all attachment/follow-up/persistence flows pass;
+- routing/execution and safety invariants are 100%;
+- all reversible writes are API-verified and cleaned up;
+- search quality and production email are reported separately and honestly;
+- non-Odysseus models demonstrably retain legacy RAG;
+- every enabled regular model has a complete tested-tool compatibility record,
+  and every tool advertised as supported passes its functional checks;
+- no hidden prompt leakage, duplicate rendering, or false success remains;
+- pre-heretic passing weights and merged adapter backups remain recoverable.
+
+## Immediate next batch
+
+1. Expand VL fixtures to charts, screenshots, and image-to-tool turns;
+   investigate the shared base-model OCR limitation without hiding it behind a
+   silent external fallback.
+2. Add deliberately permissioned private-browser open/snapshot/click checks;
+   keep browser interaction unavailable when its boundary is not enabled.
+3. Bring the two configured local regular models online and run their matrix.
+4. Compare fine-tune versus untouched base with identical compact contracts,
+   backend state, prompts, and timing instrumentation.
+5. Run the sealed all-action holdout and prioritize failures by shared
+   layer rather than by prompt.

plans/photo-editor-professional-roadmap.md

@@ -0,0 +1,445 @@
+# Plan: Odysseus Professional Photo Editor
+
+> Source PRD: Conversation goal, "a Photoshop/Photopea clone with Odysseus style"
+
+## Product boundary
+
+Odysseus should provide the editing loop people expect from a professional
+layer-based photo editor without copying Photoshop's visual design or trying to
+match every specialist feature. The target is a dependable browser editor for
+real photo work: direct manipulation, non-destructive layers, precise masking,
+retouching, typography, export, recovery, and optional AI assistance.
+
+The existing quiet Odysseus interface remains the visual language. Dense tools
+are acceptable, but controls should stay restrained, compact, predictable, and
+usable on both desktop and touch devices.
+
+## Existing foundation
+
+The current editor already provides meaningful parts of this product:
+
+- Raster and editable text layers
+- Multi-layer selection, nested groups, clipping, visibility, opacity, and locks
+- Layer, group, and selection masks
+- Marquee, lasso, wand, SAM, Quick Mask, and saved selections
+- Brush, eraser, clone, crop, transform, and text tools
+- Blend modes, adjustment stacks, blur, and several image corrections
+- Rulers, guides, grid, snapping, zooming, and panning
+- Undo/redo history with a memory budget
+- Versioned layered-project serialization, autosave drafts, recovery, and export
+- Optional endpoint-backed inpaint and image-processing tools
+- Desktop and mobile editor layouts with Playwright release-gate coverage
+
+## Architectural decisions
+
+Durable decisions that apply across every phase:
+
+- **Editor ownership**: The editor remains an Odysseus feature. Do not embed a
+  third-party editor or imitate another product's chrome.
+- **Document format**: Continue the versioned Odysseus editor document. Every
+  new persistent capability requires a migration, validation, round-trip test,
+  and corrupt-input recovery behavior.
+- **Layer model**: Grow the document into explicit layer kinds rather than
+  hiding more behavior in raster canvases. The intended kinds are raster, text,
+  shape, adjustment, and placed/smart content.
+- **Non-destructive default**: Preserve source pixels and editable parameters
+  whenever practical. Destructive actions remain available as explicit Apply,
+  Rasterize, or Merge commands.
+- **Interaction engine**: Transform, crop, selections, text frames, masks, and
+  shapes share one pointer-session model for hit testing, pointer capture,
+  modifiers, snapping, cancellation, and undo transactions.
+- **Rendering**: Keep Canvas 2D as the compatibility renderer initially. Move
+  expensive compositing and pixel operations behind renderer/worker boundaries
+  before considering WebGL or WebGPU acceleration.
+- **History**: One continuous gesture creates one undo entry. Preview frames are
+  never separate history entries, and Cancel restores the exact starting state.
+- **Persistence routes**: Continue using `/api/editor-drafts` for layered draft
+  persistence and `/api/gallery` for media-library save/replace operations.
+- **AI boundary**: AI features consume capability-based image endpoints. Core
+  editing never requires a particular model, repository, or provider.
+- **Responsive behavior**: Desktop favors precision; touch targets gain larger
+  invisible hit areas without visually enlarging the whole interface.
+- **Testing**: Every phase adds deterministic geometry/unit tests and at least
+  one complete Playwright workflow covering persistence and undo where relevant.
+- **Incremental architecture**: New behavior leaves the main editor orchestrator
+  through small domain modules. Avoid broad refactors that do not deliver a
+  visible editing improvement in the same phase.
+
+---
+
+## Phase 1: Accurate Transform Frame
+
+**User stories**: I can clearly see and grab the transform frame at any zoom. I
+can resize from corners or sides without grabbing invisible or incorrect areas.
+
+### What to build
+
+Replace the four-corner-only frame with a shared frame geometry model. Render
+four corners, four edge handles, a rotation control, and an optional center
+pivot from the same geometry used for hit testing. Keep handles visually compact
+while providing touch-sized invisible targets. Make the frame stay aligned
+during zoom, pan, viewport resize, and when handles extend outside the image.
+
+### Acceptance criteria
+
+- [x] Eight resize handles, rotation control, and center pivot derive from one geometry result.
+- [x] Drawn handles and hit targets cannot disagree.
+- [x] Handles remain a stable visual size from minimum to maximum zoom.
+- [x] Touch hit targets are at least 40 CSS pixels without oversized visuals.
+- [x] Outside-canvas handles remain interactive and visible when space permits.
+- [x] Hover and active cursors match each handle's current screen direction.
+- [x] Desktop and mobile Playwright tests grab every handle successfully.
+
+---
+
+## Phase 2: Correct Rotated Resize
+
+**User stories**: I can resize a rotated layer naturally. The opposite side or
+corner stays fixed, and the frame follows my pointer rather than drifting.
+
+### What to build
+
+Calculate drag movement in the frame's rotated local coordinate system. Anchor
+the opposite handle in document space and derive the new center from that
+anchor. Support crossing an axis as a deliberate flip instead of clamping to a
+one-pixel box. Apply the same geometry to one layer, multiple layers, and a
+selection transform.
+
+### Acceptance criteria
+
+- [x] Rotated corner and edge drags follow the pointer on the frame's local axes.
+- [x] The opposite anchor remains fixed within a sub-pixel tolerance.
+- [x] Crossing width or height zero produces a predictable horizontal or vertical flip.
+- [x] Shift locks the starting aspect ratio.
+- [x] Alt/Option scales around the transform center.
+- [x] Combined Shift+Alt/Option behavior is deterministic.
+- [x] Rotation snaps to 15-degree increments with Shift and remains smooth otherwise.
+- [x] Geometry tests cover 0, 45, 90, 135, and arbitrary-degree rotations.
+
+---
+
+## Phase 3: Transform Interaction Polish
+
+**User stories**: Transform behaves like a professional tool on mouse, pen, and
+touch. I can see exact values, snap precisely, and never lose a drag at the edge.
+
+### What to build
+
+Use a unified pointer session with pointer capture, live modifiers, and a small
+contextual transform readout. Add accurate rotated-frame interior hit testing,
+keyboard nudging, frame snapping, and clear Apply/Cancel behavior. Keep the
+existing compact Odysseus styling and make the numeric popup a precision surface
+rather than a competing transform implementation.
+
+### Acceptance criteria
+
+- [x] Pointer capture keeps a drag alive outside the canvas and browser viewport.
+- [x] Clicking inside a rotated frame moves it; clicking its empty bounding-box corner does not.
+- [x] Live X, Y, W, H, and angle values stay synchronized with direct manipulation.
+- [x] Arrow keys nudge, Shift+Arrow performs a larger nudge, Enter applies, and Escape cancels.
+- [x] Layer edges, document center/edges, guides, and grid participate in transform snapping.
+- [x] Snap guides clearly identify the active alignment without obscuring the photo.
+- [x] A complete gesture creates exactly one undo step.
+- [x] Touch gestures do not conflict with viewport pinch/pan behavior.
+
+---
+
+## Phase 4: Transform Content Correctness
+
+**User stories**: Transforming layers never unexpectedly damages masks, text,
+group layout, clipping, or image quality. Saving and reopening preserves it.
+
+### What to build
+
+Route raster layers, text layers, linked and unlinked masks, selections, clipped
+layers, and grouped multi-selection through the same transform contract. Keep
+immutable source data during previews and validate the final result through
+undo, cancel, autosave, project download, and reopen.
+
+### Acceptance criteria
+
+- [x] Raster previews are always derived from the session source, never a prior preview.
+- [x] Editable text remains editable after scaling, rotation, and flipping.
+- [x] Linked masks follow the layer while unlinked masks remain in document space.
+- [x] Multi-layer transforms preserve relative centers, order, clipping, and group membership.
+- [x] Transforming a selection changes only the selection mask unless content transform is explicitly chosen.
+- [x] Apply, Cancel, Undo, Redo, autosave reopen, and project-file reopen produce matching pixels and metadata.
+- [x] Large transforms cannot allocate beyond the editor's documented surface budget.
+
+---
+
+## Phase 5: Shared Direct-Manipulation Sessions
+
+**User stories**: Crop, selections, masks, text boxes, and shapes feel consistent
+with Transform instead of each behaving like a separate mini application.
+
+### What to build
+
+Generalize the proven transform pointer session into a reusable interaction
+contract. Migrate crop and selection movement first as a visible tracer bullet,
+including modifiers, snapping, pointer capture, cancel, and one-step history.
+
+### Acceptance criteria
+
+- [x] Transform, crop, and selection movement use the same gesture lifecycle.
+- [x] Tool switching safely commits, cancels, or prompts according to one policy.
+- [x] No stale pointer session can modify a newly selected tool or document.
+- [x] Mouse, pen, and touch event behavior is covered by shared tests.
+- [x] Adding a future frame-based tool does not require another global event stack.
+
+---
+
+## Phase 6: Non-Destructive Placed Layers
+
+**User stories**: I can import an image, resize it repeatedly without cumulative
+quality loss, replace its source, and choose when to rasterize it.
+
+### What to build
+
+Introduce a placed/smart layer kind containing source pixels and persistent
+transform metadata. Import-as-layer uses this kind by default. Rendering applies
+the transform at composite time, while Rasterize produces a normal raster layer.
+
+### Acceptance criteria
+
+- [x] Repeated transforms render from the original source rather than resampling the last result.
+- [x] A placed layer can be replaced while preserving its transform and masks.
+- [x] Rasterize produces a visually matching editable raster layer.
+- [x] Masks, clipping, groups, blend modes, and opacity work with placed layers.
+- [x] Version migration and recovery handle missing or corrupt placed sources.
+- [x] Existing raster projects open without changed output.
+
+---
+
+## Phase 7: Professional Selections And Masks
+
+**User stories**: I can build, inspect, refine, save, transform, and reuse precise
+selections without manually repainting every edge.
+
+### What to build
+
+Unify marquee, lasso, wand, SAM, Quick Mask, and saved selections around one
+selection-mask model. Add explicit replace/add/subtract/intersect modes, feather,
+expand, contract, smooth, border, and a focused refine-edge workflow.
+
+### Acceptance criteria
+
+- [x] Every selection tool supports replace, add, subtract, and intersect modes.
+- [x] Feather, expand, contract, smooth, and border preview before applying.
+- [x] Quick Mask edits the same canonical selection shown by marching ants.
+- [x] Selection-to-layer-mask and layer-mask-to-selection round-trip accurately.
+- [x] Saved selections retain names and pixels across reopen.
+- [x] Edge refinement works without requiring an AI dependency.
+
+---
+
+## Phase 8: Paint And Retouch Workflow
+
+**User stories**: I can paint and retouch photographs with predictable strokes,
+reusable presets, and the controls expected for a mouse, pen, or touch device.
+
+### What to build
+
+Promote brush behavior into a reusable brush engine. Add spacing, smoothing,
+pressure mapping, blend mode, sampled color, presets, and stroke preview. Build
+healing, dodge, and burn as complete retouching paths using that engine.
+
+### Acceptance criteria
+
+- [x] Brush, eraser, clone, masks, and inpaint share spacing and smoothing behavior.
+- [x] Pressure can independently affect size, opacity, or flow when supported.
+- [x] Eyedropper samples composite or active-layer color.
+- [x] Brush presets can be created, named, selected, and deleted.
+- [x] Healing, dodge, and burn create one undo entry per stroke.
+- [x] Long strokes remain smooth without blocking the main interface.
+
+---
+
+## Phase 9: Editable Text And Shapes
+
+**User stories**: I can design labels, cards, and overlays with text and vector
+shapes that remain editable after saving and reopening.
+
+### What to build
+
+Add on-canvas text-frame editing, selection, caret behavior, typography, and
+alignment. Introduce shape layers for rectangle, ellipse, line, and path-backed
+polygons with editable fill, stroke, corners, and transform metadata.
+
+### Acceptance criteria
+
+- [x] Text is edited directly on canvas without immediately rasterizing.
+- [x] Font, size, weight, line height, letter spacing, alignment, and color persist.
+- [x] Rectangle, ellipse, line, and polygon shapes remain editable.
+- [x] Shape fill, stroke, width, and corner radius can be changed after creation.
+- [x] Text and shape layers support masks, clipping, groups, blend modes, and transform.
+- [x] Missing fonts fall back predictably without corrupting the project.
+
+---
+
+## Phase 10: Adjustment Layers And Color
+
+**User stories**: I can correct a photograph non-destructively and return later
+to modify the correction without reconstructing the edit.
+
+### What to build
+
+Promote adjustments into first-class layers with masks and clipping. Deliver
+Levels and Curves first, then exposure, white balance, hue/saturation, color
+balance, selective color, gradients, and channel-aware controls.
+
+### Acceptance criteria
+
+- [ ] Adjustment layers affect content below them and can be clipped or grouped.
+- [ ] Every adjustment has live preview, reset, visibility, opacity, mask, Apply, and Cancel behavior.
+- [ ] Levels includes histogram, input range, gamma, and output range.
+- [ ] Curves supports RGB and channel curves with editable points.
+- [ ] Color results match flattened export and project reopen.
+- [ ] Large previews are throttled or worker-backed and remain cancellable.
+
+---
+
+## Phase 11: Layer Effects And Filters
+
+**User stories**: I can add common visual effects without permanently altering
+the layer and can reorder or disable those effects later.
+
+### What to build
+
+Create an ordered non-destructive filter/effect stack. Begin with Gaussian blur,
+sharpen, shadow, stroke, and color overlay; then add filter masks and reusable
+effect presets.
+
+### Acceptance criteria
+
+- [ ] Effects can be added, reordered, toggled, edited, masked, and removed.
+- [ ] Drop shadow, stroke, color overlay, blur, and sharpen survive project reopen.
+- [ ] Effects render correctly inside groups and clipping stacks.
+- [ ] Apply/rasterize produces a pixel-equivalent raster result.
+- [ ] Expensive filters expose progress and cancellation.
+
+---
+
+## Phase 12: Odysseus Professional Workspace
+
+**User stories**: I can work quickly without fighting floating windows or losing
+the active tool, layer, selection, or document context.
+
+### What to build
+
+Refine the existing shell into a consistent professional workspace: contextual
+tool options, properties inspector, panel persistence, command search, status
+information, multi-document switching, and compact touch sheets. Preserve the
+current Odysseus palette, typography, restrained borders, and frosted surfaces.
+
+### Acceptance criteria
+
+- [ ] Tool options appear in one predictable location and never duplicate popup state.
+- [ ] Panels remember size, collapsed state, and position per device class.
+- [ ] The properties inspector follows the active layer, mask, selection, or tool.
+- [ ] Command search exposes actions and shortcuts without adding toolbar clutter.
+- [ ] Switching documents preserves independent history, zoom, pan, and selection.
+- [ ] Mobile prioritizes canvas area while keeping all commands reachable.
+
+---
+
+## Phase 13: File Interchange And Export
+
+**User stories**: I can bring common assets into Odysseus and export predictable
+results without losing transparency, dimensions, or color intent.
+
+### What to build
+
+Strengthen image import/export first, then add layered interchange where a
+maintained parser makes it safe. Keep Odysseus project files as the lossless
+source of truth and clearly report what an external format cannot preserve.
+
+### Acceptance criteria
+
+- [ ] PNG, JPEG, WebP, and supported modern image imports honor orientation and transparency.
+- [ ] Export exposes format, dimensions, quality, metadata, and transparency choices.
+- [ ] Copy/paste and drag/drop preserve alpha and use placed layers when appropriate.
+- [ ] Layered imports report unsupported features instead of silently flattening them.
+- [ ] Exported pixels are covered by deterministic visual comparisons.
+
+---
+
+## Phase 14: Large-Document Performance And Recovery
+
+**User stories**: Large photos and layered projects remain responsive, autosave
+reliably, and recover after a crash or interrupted network connection.
+
+### What to build
+
+Move serialization, thumbnails, filters, and suitable pixel operations into
+workers. Add dirty-region rendering, reusable surfaces, measurable memory
+budgets, operation cancellation, autosave generations, and recovery diagnostics.
+
+### Acceptance criteria
+
+- [ ] Normal interactions remain responsive on the agreed 4K multi-layer benchmark.
+- [ ] Compositing avoids rebuilding unaffected layers and thumbnails.
+- [ ] History and document surfaces stay within explicit memory limits.
+- [ ] Closing or switching documents cancels stale work safely.
+- [ ] Autosave never lets an older request overwrite newer state.
+- [ ] Recovery can identify the last complete generation and explain skipped data.
+
+---
+
+## Phase 15: Odysseus-Native Assisted Editing
+
+**User stories**: I can use an available local or remote image capability as an
+editing assistant while retaining masks, layers, undo, privacy choices, and
+normal manual controls.
+
+### What to build
+
+Standardize image capability discovery and requests for generation, editing,
+inpainting, segmentation, restoration, and upscaling. Results enter the document
+as named layers with provenance and reusable masks. Add orchestration only after
+the manual operation it assists is dependable.
+
+### Acceptance criteria
+
+- [ ] The UI describes required capabilities rather than model or provider names.
+- [ ] Memory and unrelated chat context are not sent to image endpoints.
+- [ ] Requests show progress, support cancellation, and cannot update a closed document.
+- [ ] Generated results arrive as reversible layers with prompt/settings metadata.
+- [ ] A failed endpoint leaves the source document unchanged and offers a useful retry path.
+- [ ] Manual selection and masking remain available when assisted tools are absent.
+
+---
+
+## Phase 16: Professional Release Gate
+
+**User stories**: I can trust the editor for real work and understand what is
+unsupported before committing an edit.
+
+### What to build
+
+Create a release gate around complete user journeys rather than isolated button
+tests. Cover accessibility, keyboard-only operation, touch, browser differences,
+pixel correctness, persistence, failure recovery, and large-document behavior.
+
+### Acceptance criteria
+
+- [ ] Core workflows pass on current Chromium and Firefox desktop builds.
+- [ ] Mobile workflows pass at representative phone and tablet viewports.
+- [ ] Keyboard-only users can reach every command and escape every modal state.
+- [ ] Transform, masks, text, adjustments, export, and reopen have pixel/metadata regression tests.
+- [ ] No supported action silently flattens or discards editable document data.
+- [ ] The ALPHA badge can be removed based on explicit reliability metrics.
+
+---
+
+## Recommended delivery order
+
+The first four phases are one focused Transform 2.0 program and should ship in
+order. Phases 5 and 6 establish the interaction and document foundations needed
+for the remaining professional tools. After that, phases 7 through 13 can be
+prioritized by user value, while performance and release-gate work continue as
+part of every phase rather than being deferred entirely to the end.
+
+The recommended first milestone is complete when Phases 1 through 4 are live:
+transforming one layer, multiple layers, text, masks, and selections feels
+precise on desktop and mobile and remains correct through undo and reopen.

plans/photo-editor-remaining-scope.md

@@ -0,0 +1,159 @@
+# Photo Editor Remaining Scope
+
+Date: 2026-08-29
+
+## Current verdict
+
+Odysseus is now a credible layered everyday editor, not an editor mockup. The
+first nine roadmap phases are implemented: professional transform geometry,
+shared direct-manipulation sessions, retained placed content, unified
+selections and masks, a reusable brush/retouch engine, and retained text and
+shape layers.
+
+Phase 10 is functionally advanced but not closed. First-class adjustment layers
+now support Levels, Curves, Exposure, White Balance, Brightness/Contrast,
+Hue/Saturation/Lightness, Color Balance, Selective Color, and Gradient Map.
+They participate in clipping, groups, masks, visibility, opacity, history, the
+v14 document format, and flattening. Retained effects have since been added as
+a separate ordered stack with Gaussian Blur, Color Overlay, Drop Shadow, and
+Stroke, including editable colors, visibility, opacity, reorder, rasterize,
+history, persistence, and migration.
+
+Practical readiness estimate:
+
+- Everyday layered photo editing: **about 88%**
+- Dependable professional v1 described by the roadmap: **about 62%**
+- Broad Photoshop/Photopea feature parity: **about 50%**
+
+The remaining gap is dominated by large-document rendering outside the live
+composite path, workspace consolidation, interchange/color policy, and release
+proof rather than basic canvas tools.
+
+## Verification snapshot
+
+- The focused editor unit suite currently passes **31 tests** in Docker.
+- The full photo-editor browser suite currently has **41 passing workflows**;
+  the nested-group selection workflow initially exposed a row-hit regression,
+  which now passes on isolated rerun after the slider-selection fix. The new
+  group-effects workflow also passes.
+- The new adjustment tests exercise deterministic pixel math, nested parameter
+  normalization, retained metadata, undo/redo, clipping, masks, and draft
+  reopen.
+- The latest editor changes have not yet been rebuilt into the live `7011`
+  container.
+
+## Close Phase 10
+
+This is the immediate release slice.
+
+1. Finish the bounded preview path for large documents. Downsampled previews
+   now keep control movement responsive and full resolution is restored for
+   commit/export. Live worker composites now use generation checks, latest-only
+   coalescing, and close/reopen invalidation; extend the same guarantees to
+   remaining preview paths.
+2. Add flattened-export versus reopened-project pixel comparisons for every
+   adjustment family, including groups, clipping, masks, blend mode, and
+   partial opacity.
+3. Validate the color algorithms visually. White Balance and Selective Color
+   are currently deterministic approximations, not color-managed photographic
+   transforms.
+4. Test every adjustment popup on phone and desktop viewports, including tall
+   popups, color inputs, drag, Reset, Apply, Cancel, and Escape.
+5. Decide the migration path for the older per-raster `adjLayers` stack. It can
+   remain readable for compatibility, but new UI should converge on first-class
+   adjustment layers instead of maintaining two competing concepts.
+6. Bump static cache versions, rebuild the live container, and run a short
+   visual smoke test on `7011`.
+
+## Phase 11: Retained effects and filters
+
+The retained-effects slice is implemented for raster/placed/text/shape-compatible
+layer output: Gaussian Blur, Sharpen, Color Overlay, Drop Shadow, and Stroke
+have editable colors/parameters, visibility, opacity, reorder, rasterize,
+history, migration, and reopen support. Effect-specific masks, presets, and
+group-level effects are also implemented and covered by focused browser tests.
+Remaining work is:
+
+1. Extend worker coverage to serialization and remaining preview paths.
+   Thumbnail encoding, retained-effect rasterization, and live composite
+   rendering now use a worker where OffscreenCanvas is available, with
+   synchronous compatibility fallbacks. Generation invalidation, latest-only
+   coalescing, and CPU loop cancellation protect live rendering.
+2. Add explicit group-effect blend/ordering tests for nested groups and
+   non-default blend modes, plus visual comparisons for effect stacks.
+
+Introduce the renderer/worker cancellation boundary here rather than adding
+more synchronous full-canvas filters that Phase 14 must immediately replace.
+
+## Phase 12: Professional workspace
+
+Consolidate fragmented popups into one contextual properties surface. Persist
+panel layout by device class, add command search, expose stable document status,
+and support multiple open documents with independent history, zoom, pan, and
+selection. Mobile should use canvas-first sheets rather than compressed desktop
+panels.
+
+## Phase 13: Interchange and export
+
+Harden orientation, transparency, metadata, and color behavior for PNG, JPEG,
+and WebP first. Add copy/paste and drag/drop through placed layers. Treat
+layered formats as explicit compatibility projects: unsupported PSD/TIFF/HEIC
+features must be reported, never silently discarded. Odysseus project files
+remain the lossless source of truth.
+
+## Phase 14: Performance and recovery
+
+Move remaining preview/pixel paths into workers. Thumbnail encoding,
+autosave serialization, adjustment rendering, and retained-effect rendering
+now have worker-backed paths with compatibility fallbacks. Add
+dirty-region compositing, reusable render surfaces, cancellation tokens,
+operation telemetry, a documented surface/history budget, autosave generations,
+and a checked-in 4K multi-layer benchmark.
+
+This phase is the main architectural risk. Canvas 2D remains a valid
+compatibility renderer, but full-document synchronous passes will not scale to
+professional documents.
+
+## Phase 15: Assisted editing
+
+Normalize generation, editing, inpainting, segmentation, restoration, and
+upscaling behind capability-based endpoints. Keep model/provider names out of
+editor logic. Requests must exclude chat memory, show progress, cancel safely,
+and return named reversible layers with provenance. Manual tools remain fully
+usable without an endpoint.
+
+Much of the endpoint plumbing already exists; the remaining work is consistent
+capability discovery, lifecycle safety, and editor-native result handling.
+
+## Phase 16: Release gate
+
+Run complete user journeys on Chromium and Firefox desktop plus representative
+phone/tablet viewports. Add keyboard-only and accessibility coverage, mixed
+20-edit persistence/export tests, failure recovery, and large-document stress
+tests. No supported operation may silently flatten or discard retained state.
+
+## Architecture debt to control
+
+- `galleryEditor.js` is still a large orchestrator. Continue extracting domain
+  modules as visible features move, without a broad rewrite.
+- Legacy raster adjustment sublayers and first-class adjustment layers overlap.
+  Converge on the first-class model.
+- Pixel effects still rely heavily on synchronous full-canvas work.
+- `static/style.css` carries substantial editor-specific surface area and needs
+  clearer component boundaries before workspace customization expands.
+- The repository worktree contains many unrelated changes. Editor release and
+  merge decisions require a scoped diff or clean integration branch.
+
+## Recommended execution order
+
+1. Close and deploy Phase 10.
+2. Build Phase 11 through a cancellable render boundary.
+3. Consolidate the workspace in Phase 12.
+4. Define color/metadata policy and complete Phase 13.
+5. Finish worker rendering, stress, and recovery in Phase 14.
+6. Normalize assisted editing in Phase 15.
+7. Run the cross-browser professional release gate in Phase 16.
+
+Do not expand into full PSD fidelity, CMYK production, RAW development, 3D, or
+complete Photoshop parity before this critical path passes. Those are separate
+product decisions, not prerequisites for a strong Odysseus editor.

requirements-optional.txt

@@ -1,4 +1,7 @@
 # Optional dependencies — install only if you use the corresponding feature.
+# Local OCR for screenshots, scans, labels, and coordinate-grounded text extraction.
+rapidocr==3.9.2
+onnxruntime>=1.20,<2
 # The app handles their absence gracefully (clear error message on first use).
 #
 # Note: chromadb-client + fastembed moved to requirements.txt — RAG, semantic
@@ -43,4 +46,7 @@ PyMuPDF
 # magika (onnxruntime), already a core dep via fastembed. We avoid the
 # [all]/Azure/audio extras (cloud + heavy). Pinned to a release >30 days old per
 # the dependency-age discussion in issue #485.
-markitdown[docx,pptx,xlsx,xls]==0.1.7
+markitdown[docx,pptx,xlsx,xls]==0.1.6
+
+# Photoshop PSD opening / flattened previews / layer inspection.
+psd-tools

requirements.txt

@@ -3,11 +3,16 @@ uvicorn
 python-multipart
 python-dotenv
 httpx
-httpcore>=1.0.9,<2.0
-pydantic>=2.13.5
-pydantic-settings>=2.15.0
+httpcore>=1.0,<2.0
+pydantic>=2.13.4
+pydantic-settings>=2.14.1
 SQLAlchemy
 pypdf
+pypdfium2
+Pillow
+faster-whisper
+PyPDF2
+pdfplumber
 beautifulsoup4
 charset-normalizer
 numpy
@@ -19,6 +24,7 @@ numpy
 chromadb-client
 fastembed
 youtube-transcript-api
+yt-dlp
 # Markdown rendering for research reports (src/visual_report.py).
 # Imported at module-top so it's a hard core dep, not optional.
 markdown
@@ -41,7 +47,7 @@ bcrypt
 # Built-in servers use the v1 low-level Server decorator API. MCP SDK v2 is a
 # breaking rewrite, so keep fresh installs on the maintained v1 line until the
 # servers are migrated together.
-mcp<3
+mcp<2
 pyotp
 qrcode[pil]
 croniter
@@ -51,8 +57,3 @@ pytest-asyncio
 # TestClient import when only classic httpx is present. Runtime code keeps
 # using `httpx` above; this is test-client only.
 httpx2
-# DATABASE_URL defaults to sqlite (core/database.py), but when pointed at an
-# external Postgres, SQLAlchemy's postgresql dialect imports psycopg2 inside
-# create_engine() and raises ModuleNotFoundError if missing. -binary avoids
-# needing libpq-dev/pg_config on the host/image to compile it.
-psycopg2-binary

resources/skills/agent/artifact-completion/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: artifact-completion
+description: Create requested artifacts early, iterate from concrete output, and verify final deliverables
+version: 1.0.0
+category: agent
+tags: [artifacts, files, verification, workflow]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when the task requires a file, patch, report, document, image, archive, configuration, or other persistent deliverable rather than only a text answer.
+
+## Procedure
+
+1. Extract the required deliverable path, format, content constraints, and acceptance criteria.
+2. Inspect the source material and existing target without delaying the first valid artifact.
+3. Create a minimal complete version at the required location, then iterate from that concrete output.
+4. Use the format's native parser, renderer, compiler, or test tool to inspect the artifact.
+5. Repair specific validation, content, or presentation failures while preserving correct portions.
+6. Confirm the final path, file type, required content, and usability before reporting completion.
+
+## Pitfalls
+
+- Do not spend the full task budget inspecting without creating the requested output.
+- Do not place the artifact at a convenient path when the task specifies another location.
+- Do not use a filename extension as proof that the file is valid in that format.
+- Do not report completion while placeholders, missing sections, parse errors, or failed checks remain.
+
+## Verification
+
+- The artifact exists at the required path and opens or parses successfully.
+- Required sections, fields, labels, or visual elements are present.
+- Relevant tests, render checks, or validators pass.

resources/skills/agent/terminal-recovery/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: terminal-recovery
+description: Recover from failed terminal commands using evidence-driven diagnosis and bounded retries
+version: 1.0.0
+category: agent
+tags: [terminal, shell, debugging, recovery]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when a command fails, times out, produces incomplete output, or behaves differently from what the task requires.
+
+## Procedure
+
+1. Read the command, exit status, standard output, and standard error before choosing a response.
+2. Confirm the working directory, relevant files, executable availability, permissions, and environment assumptions with minimal read-only probes.
+3. Classify the failure as syntax, missing dependency, wrong path, permissions, resource pressure, timeout, service state, or task logic.
+4. Change one relevant condition and retry the narrowest command that can test the diagnosis.
+5. For a long-running command, use the returned session identifier to poll or provide input instead of launching duplicates.
+6. After recovery, run the original acceptance check and inspect the resulting files or service state.
+
+## Pitfalls
+
+- Do not rerun an unchanged failing command repeatedly.
+- Do not install packages or change global configuration before confirming they are missing and necessary.
+- Do not launch a second server or training job before checking for an existing process and port or device conflicts.
+- Do not treat partial output or a zero exit status as proof that the requested state was produced.
+
+## Verification
+
+- The diagnosed cause is supported by command output or environment state.
+- The corrected command exits as expected.
+- The requested artifact, process, or state passes an independent acceptance check.

resources/skills/agent/tool-discovery/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: tool-discovery
+description: Discover the smallest capable tool set and confirm argument schemas before acting
+version: 1.0.0
+category: agent
+tags: [tools, discovery, routing, schemas]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when a task requires tools whose names, capabilities, or argument shapes are not already clear. This is especially useful when many tools are available or a previous call failed because the wrong tool or parameters were selected.
+
+## Procedure
+
+1. Translate the request into required capabilities such as reading, searching, editing, executing, browsing, or verifying.
+2. Search the tool index for those capabilities and inspect the returned tool descriptions and schemas.
+3. Prefer one direct tool over a chain of indirect tools when it can complete the operation and provide evidence.
+4. Check required parameters, identifiers, path rules, side effects, and approval requirements before calling the tool.
+5. Make a small read-only probe when the environment or target is uncertain.
+6. Execute the selected action, inspect the result, and only broaden the tool search if the result shows a concrete capability gap.
+
+## Pitfalls
+
+- Do not guess tool names or argument keys from memory when the index or schema is available.
+- Do not load unrelated tool groups into context.
+- Do not repeat the same failed call without changing the arguments or strategy.
+- Do not use a broad shell or browser workaround when a scoped native tool already owns the operation.
+
+## Verification
+
+- The chosen tool directly matches the required capability.
+- Required arguments follow the exposed schema.
+- The result contains evidence of the requested effect or a specific error that guides the next step.

resources/skills/agent/verified-state-change/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: verified-state-change
+description: Make scoped state changes with target confirmation, minimal mutation, and read-back verification
+version: 1.0.0
+category: agent
+tags: [state, mutation, verification, safety]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when creating, editing, deleting, moving, sending, scheduling, or otherwise changing persistent state through an application, API, filesystem, or service.
+
+## Procedure
+
+1. Read the current state and identify the target using stable identifiers plus enough content to disambiguate it.
+2. Preserve fields the user did not ask to change and choose the narrowest supported mutation.
+3. For destructive or externally visible actions, confirm that the user's instruction authorizes the exact target and effect.
+4. Perform the mutation once and capture the returned identifier, status, or revision.
+5. Read the target again through an independent list, fetch, status, or content operation.
+6. Compare the observed state with the requested outcome and repair only the specific mismatch.
+
+## Pitfalls
+
+- Do not infer the target from a stale active item when a stable identifier can be fetched.
+- Do not report success from an accepted request alone; asynchronous or partial operations may not have completed.
+- Do not replace an entire object when a field-level update is supported and safer.
+- Do not silently broaden a mutation to adjacent files, records, accounts, or services.
+
+## Verification
+
+- The target identity was confirmed before mutation.
+- A read-back shows the intended values and preserves unrelated state.
+- Any external effect has a concrete status, identifier, or observable result.

resources/skills/communication/action-evidence-synthesis/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: action-evidence-synthesis
+description: "Turn messages, meeting notes, and documents into sourced decisions, actions, dependencies, and risks"
+version: 1.0.0
+category: communication
+tags: [messages, meetings, actions, status, evidence]
+status: published
+confidence: 1.0
+source: builtin
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when information is fragmented across messages, meeting notes, transcripts, or documents and the user needs an action list, status summary, feasibility assessment, or executive brief.
+
+Do not use when the source material is unavailable or when the user only wants a verbatim transcript.
+
+## Procedure
+
+1. Identify the requested scope, audience, time window, and decision to support.
+2. Gather the relevant records in full and preserve stable source identifiers, authors, and timestamps.
+3. Extract explicit decisions, commitments, requests, owners, dates, dependencies, blockers, and changed facts.
+4. Reconcile revisions by preferring the newest authoritative record; keep unresolved conflicts visible instead of guessing.
+5. Separate observed facts from inferred owners, dates, urgency, feasibility, or recommendations, and label every inference as tentative.
+6. Produce the requested format with concise source references beside consequential claims and a final list of open questions.
+
+## Pitfalls
+
+- Do not turn discussion or speculation into a confirmed decision.
+- Do not invent owners or deadlines when none were assigned.
+- Do not silently discard older records that explain a changed commitment.
+- Do not send messages, create tasks, or update calendars unless the user separately authorizes those actions.
+
+## Verification
+
+- Every action has a source, status, and explicit or tentative owner and due date.
+- Conflicting values and revisions are resolved or visibly flagged.
+- The output covers decisions, actions, dependencies, risks, and open questions relevant to the request.
+

resources/skills/communication/reviewable-external-draft/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: reviewable-external-draft
+description: "Reconcile source evidence and prepare an accurate external-facing draft without bypassing review"
+version: 1.0.0
+category: communication
+tags: [drafting, email, messages, review, reconciliation]
+status: published
+confidence: 1.0
+source: builtin
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when preparing a client, customer, partner, leadership, or other external-facing update from internal messages or documents.
+
+Do not use this procedure to send immediately unless the user explicitly authorizes the exact recipient and final content.
+
+## Procedure
+
+1. Confirm the audience, communication channel, requested tone, and whether the user asked for a draft or an immediate send.
+2. Gather the relevant source records and identify the latest values, dates, commitments, and unresolved discrepancies.
+3. Resolve recipient identity through the available contact source and avoid inferring internal versus external status from a display name alone.
+4. Draft only claims supported by the collected evidence; qualify uncertainty and omit internal-only detail that the audience should not receive.
+5. Save or present a reviewable draft through the native draft or document capability.
+6. Report the draft identifier or location plus any reconciliation notes that require human review.
+
+## Pitfalls
+
+- Do not send a draft merely because a send-capable tool is available.
+- Do not copy stale figures when a later correction exists.
+- Do not conceal unresolved discrepancies behind polished prose.
+- Do not expose private internal discussion, credentials, or unrelated personal data.
+
+## Verification
+
+- Recipient identity and communication mode match the request.
+- Dates, figures, status, and commitments map to current source evidence.
+- The result remains reviewable unless an explicit send-now instruction authorized delivery.
+

resources/skills/communication/scheduling-coordination/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: scheduling-coordination
+description: "Coordinate availability, confirmations, calendar changes, and participant notifications with read-back verification"
+version: 1.0.0
+category: communication
+tags: [calendar, scheduling, coordination, availability]
+status: published
+confidence: 1.0
+source: builtin
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when arranging or changing a meeting across multiple participants, calendars, time zones, or communication channels.
+
+Do not create or modify an event when the user asked only for available options or a draft invitation.
+
+## Procedure
+
+1. Extract participants, duration, date range, time zones, location constraints, and required attendees.
+2. Resolve participant identities and inspect the relevant availability using declared calendar and contact capabilities.
+3. Compute candidate intervals in one explicit reference time zone and reject conflicts or insufficient travel buffers.
+4. Present or draft a small set of viable options when confirmation is still required.
+5. After authorization or recorded participant confirmation, create or update the event once with stable attendee identifiers.
+6. Read the event back and verify title, start, end, time zone, attendees, location, and conferencing details before drafting notifications.
+
+## Pitfalls
+
+- Do not overwrite or cancel unrelated events to manufacture availability.
+- Do not mix local times without naming the time zone.
+- Do not treat a proposed time as confirmed.
+- Do not create duplicates when an existing event can be updated safely.
+
+## Verification
+
+- The selected interval satisfies duration, availability, and time-zone constraints.
+- The calendar read-back matches the authorized event details.
+- Notifications describe the same confirmed event and remain drafts unless sending was explicitly authorized.
+

resources/skills/communication/support-triage-and-routing/SKILL.md

@@ -0,0 +1,39 @@
+---
+name: support-triage-and-routing
+description: "Prioritize support requests, identify owners, route internally, and prepare safe customer drafts"
+version: 1.0.0
+category: communication
+tags: [support, triage, urgency, routing, drafts]
+status: published
+confidence: 1.0
+source: builtin
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when reviewing a support backlog, identifying urgent incidents, assigning internal ownership, or drafting customer responses.
+
+Do not use when the request is merely to summarize an unrelated inbox or when sender identity cannot be established safely.
+
+## Procedure
+
+1. Read each in-scope request in full and retain its stable message or ticket identifier.
+2. Resolve whether the sender is internal or external and identify the responsible internal team from available contacts and service ownership data.
+3. Classify urgency from impact and time sensitivity: critical for outage, data loss, security exposure, or imminent contractual breach; high for a blocked user without a workaround; medium for degraded service with a workaround; low for non-blocking inquiries.
+4. Record a concise problem statement, evidence, affected scope, workaround, owner, next action, and response deadline.
+5. Route internally only when the user has authorized operational messaging; prepare external responses as reviewable drafts by default.
+6. Re-read created assignments or drafts and produce an escalation summary grouped by urgency.
+
+## Pitfalls
+
+- Do not infer severity from emotional language alone.
+- Do not expose one customer's data in another customer's response.
+- Do not send externally when the task calls for triage or drafting.
+- Do not mark an issue routed without a stable owner or observable routing result.
+
+## Verification
+
+- Every issue has a stable source identifier, urgency rationale, owner, and next action.
+- Critical and high items have explicit response targets and escalation state.
+- External communication is a draft unless the user explicitly authorized sending.

resources/skills/dev/developer-docs/SKILL.md

@@ -0,0 +1,37 @@
+---
+name: developer-docs
+description: Find, read, and apply authoritative developer documentation during implementation
+version: 1.0.0
+category: dev
+tags: [docs, documentation, api, software-development]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-18T00:00:00Z"
+---
+
+## When to Use
+
+Use when the user asks how a library, framework, API, protocol, CLI, or SDK works, or when implementation depends on version-specific behavior. Prefer this skill over guessing from memory.
+
+## Procedure
+
+1. Identify the exact product, package, version, and task. Ask one focused clarification only when the target is genuinely ambiguous.
+2. Prefer the vendor's or project's primary documentation, source repository, release notes, and API reference. Use a general search only to locate those sources.
+3. Read the relevant page or reference section, then apply the documented behavior to the user's codebase and active workspace.
+4. Separate documented facts from inference, and call out version or environment assumptions.
+5. For code changes, add a focused regression test for the documented contract and run it before reporting completion.
+
+## Pitfalls
+
+- Do not present search snippets, stale cached knowledge, or a third-party tutorial as authoritative when primary documentation is available.
+- Do not silently mix instructions from different major versions.
+- Do not claim an API or option exists without confirming it in the relevant reference.
+- Do not use web search for a local project task when the active workspace and local tools can answer it.
+
+## Verification
+
+- The cited or retrieved documentation matches the target version.
+- The implementation or answer distinguishes source-backed facts from inference.
+- Any code change has a focused test or a concrete verification command.

resources/skills/general/test-driven-development/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: test-driven-development
+description: Build or fix software with a focused red-green-refactor loop
+version: 1.0.0
+category: general
+tags: [tdd, testing, debugging, red-green-refactor]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-18T00:00:00Z"
+---
+
+## When to Use
+
+Use when implementing a feature, fixing a bug, or changing behavior where a regression test can define the expected result. Prefer this workflow for parser, routing, agent-loop, and UI behavior changes.
+
+## Procedure
+
+1. Inspect the relevant code, existing tests, and local conventions before editing.
+2. Write the smallest regression test that demonstrates the requested behavior or reproduces the bug.
+3. Run that test and confirm it fails for the expected reason, not because the test setup is broken.
+4. Make the smallest production change that makes the test pass.
+5. Run the focused test again, then run the surrounding module suite.
+6. Review the diff for unrelated changes, brittle assertions, hidden state, and missing error paths.
+7. Report the tests run and any remaining coverage or environment limits.
+
+## Pitfalls
+
+- Do not write a test that only mirrors the implementation; assert the user-visible contract.
+- Do not weaken an assertion just to make a failing test pass.
+- Do not skip the focused failing-test step when the behavior is observable in a local test.
+- Keep network, filesystem, and model calls deterministic with fakes or fixtures unless the integration itself is under test.
+
+## Verification
+
+- The new regression test fails before the fix and passes after it.
+- The relevant focused suite passes.
+- The broader suite passes or its failure is explained with evidence.
+- The final diff contains the test and the production change needed for the same behavior.

resources/skills/media/multimodal-evidence/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: multimodal-evidence
+description: Extract and verify evidence from images, documents, and video without redundant inspection
+version: 1.0.1
+category: media
+tags: [image, video, document, evidence, ocr]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when the answer or requested artifact depends on visual, temporal, tabular, or textual evidence contained in images, documents, or video.
+
+## Procedure
+
+1. Identify the evidence required: objects, text, values, ordering, timestamps, labels, or visual relationships.
+2. Inspect the whole input or a broad representative sample first to establish structure and likely evidence locations.
+3. Narrow to relevant pages, frames, regions, or time intervals and record observations with their locations.
+4. Use the format's native parser for exact text and numbers: for example `python-docx` or ZIP/XML inspection for DOCX, `pdftotext` or a PDF library for PDF, spreadsheet readers for XLSX, and OCR only when the source is image-based. Do not search binary office files with plain `grep` or `cat`.
+5. Resolve conflicts with one targeted reinspection at better scale or a nearby frame rather than repeating the same crop.
+6. Build the answer or artifact from the evidence ledger and perform a final coverage check against every requested item.
+
+## Pitfalls
+
+- Do not infer unseen content from filenames, surrounding text, or a single thumbnail.
+- Do not repeatedly inspect nearly identical regions without a new hypothesis.
+- Do not trust OCR blindly for small labels, punctuation, or numeric values.
+- Do not finalize before checking that every requested item has supporting evidence.
+
+## Verification
+
+- Each factual output can be traced to a page, frame, region, or timestamp.
+- Exact labels and numbers were visually checked after extraction.
+- The final response or artifact covers all requested evidence categories.

resources/skills/research/web-research-fallback/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: web-research-fallback
+description: Research current web information with source-first search and controlled browser fallback
+version: 1.0.0
+category: research
+tags: [web, search, browser, sources, research]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when a task requires current public information, primary sources, multiple pages, or a site that cannot be reliably read from search results alone.
+
+## Procedure
+
+1. Define the facts needed and the preferred primary source for each fact.
+2. Search with a focused query and use result metadata to select likely authoritative pages.
+3. Open the source directly and extract the relevant passage, date, and URL rather than relying on a search snippet.
+4. Use the private browser when the page requires interaction, client-side rendering, navigation, or visual inspection.
+5. If a page fails, try a primary-source alternative or a narrower route before broadening to secondary sources.
+6. Cross-check unstable or consequential claims and distinguish source-backed facts from inference.
+
+## Pitfalls
+
+- Do not treat snippets as evidence for claims not visible on the source page.
+- Do not browse repeatedly without recording what each page established.
+- Do not use a secondary summary when an accessible primary source answers the question.
+- Do not claim freshness without checking publication or update dates.
+
+## Verification
+
+- Each important claim maps to a source that directly supports it.
+- Time-sensitive facts include an observed date or version.
+- Browser interaction produced the needed page state or a documented fallback was used.

routes/auth_routes.py

@@ -736,6 +736,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         _INT_RANGES = {
             "agent_max_rounds": (1, 200),
             "agent_max_tool_calls": (0, 1000),  # 0 = unlimited
+            "auto_compact_threshold_percent": (50, 95),
         }
         for key in DEFAULT_SETTINGS:
             if key in RETIRED_SETTING_KEYS:

routes/calendar_routes.py

@@ -4,7 +4,7 @@ import logging
 import json
 import re
 import uuid
-from datetime import datetime, date, timedelta
+from datetime import datetime, date, timedelta, timezone
 from typing import Optional, List
 
 from fastapi import APIRouter, HTTPException, Request, UploadFile, File
@@ -13,7 +13,7 @@ from sqlalchemy import or_, and_
 from sqlalchemy.exc import IntegrityError
 from dateutil.rrule import rrulestr
 
-from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent
+from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent, Note
 from src.auth_helpers import effective_user, require_user
 from src.upload_limits import read_upload_limited, ICS_MAX_BYTES
 from src.upload_handler import reserve_upload_references
@@ -207,6 +207,7 @@ class EventCreate(BaseModel):
     calendar_href: Optional[str] = None  # calendar id
     rrule: Optional[str] = None
     color: Optional[str] = None  # per-event color override
+    reminder_minutes: Optional[int] = None
 
 
 class EventUpdate(BaseModel):
@@ -218,6 +219,7 @@ class EventUpdate(BaseModel):
     location: Optional[str] = None
     rrule: Optional[str] = None
     color: Optional[str] = None
+    reminder_minutes: Optional[int] = None
 
 
 # ── Helpers ──
@@ -621,7 +623,133 @@ def _parse_dt(s: str) -> datetime:
         raise ValueError(f"could not parse datetime: {s!r}")
 
 
-def _event_to_dict(ev: CalendarEvent) -> dict:
+def _note_due_datetime(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        text = str(value).strip()
+        if text.endswith("Z"):
+            text = text[:-1] + "+00:00"
+        due = datetime.fromisoformat(text)
+        if due.tzinfo is not None:
+            return due.astimezone(timezone.utc).replace(tzinfo=None)
+        return due
+    except Exception:
+        return None
+
+
+def _calendar_reminder_for_event(db, owner: str, ev: CalendarEvent) -> dict | None:
+    """Return the closest Notes reminder that belongs to this calendar event.
+
+    Calendar alarms are currently stored as Notes rows. Older rows do not carry
+    an event UID, so match conservatively by the generated title plus due_date
+    before the event start. This keeps existing reminder notes visible on the
+    calendar without a schema migration.
+    """
+    if not db or not owner or not ev or not ev.dtstart:
+        return None
+    summary = (ev.summary or "").strip()
+    if not summary:
+        return None
+
+    titles = [f"Calendar reminder: {summary}", f"Reminder: {summary}"]
+    notes = (
+        db.query(Note)
+        .filter(
+            Note.owner == owner,
+            Note.archived == False,  # noqa: E712
+            Note.label == "calendar",
+            Note.source == "calendar",
+            Note.title.in_(titles),
+            Note.due_date.isnot(None),
+        )
+        .all()
+    )
+    if not notes:
+        return None
+
+    start = ev.dtstart
+    if getattr(start, "tzinfo", None) is not None:
+        start = start.astimezone(timezone.utc).replace(tzinfo=None)
+    best = None
+    best_minutes = None
+    for note in notes:
+        due = _note_due_datetime(note.due_date)
+        if due is None:
+            continue
+        minutes = round((start - due).total_seconds() / 60)
+        if minutes < 0 or minutes > 7 * 24 * 60:
+            continue
+        if best is None or minutes < best_minutes:
+            best = note
+            best_minutes = minutes
+    if best is None:
+        return None
+    return {
+        "note_id": best.id,
+        "due_date": best.due_date,
+        "minutes": best_minutes,
+    }
+
+
+def _delete_calendar_reminders_for_event(db, owner: str, ev: CalendarEvent) -> int:
+    if not db or not owner or not ev:
+        return 0
+    summary = (ev.summary or "").strip()
+    if not summary:
+        return 0
+    titles = [f"Calendar reminder: {summary}", f"Reminder: {summary}"]
+    notes = (
+        db.query(Note)
+        .filter(
+            Note.owner == owner,
+            Note.archived == False,  # noqa: E712
+            Note.label == "calendar",
+            Note.source == "calendar",
+            Note.title.in_(titles),
+            Note.due_date.isnot(None),
+        )
+        .all()
+    )
+    for note in notes:
+        db.delete(note)
+    return len(notes)
+
+
+def _create_calendar_reminder_for_event(db, owner: str, ev: CalendarEvent, minutes_before: int) -> dict:
+    if not owner or not ev or not ev.dtstart:
+        return {"note_id": None, "skipped_reason": "missing event"}
+    minutes_before = max(0, int(minutes_before))
+    start = ev.dtstart
+    if getattr(start, "tzinfo", None) is not None:
+        start = start.astimezone(timezone.utc).replace(tzinfo=None)
+    remind_at = start - timedelta(minutes=minutes_before)
+    now = datetime.utcnow() if getattr(ev, "is_utc", False) else datetime.now()
+    if start <= now:
+        return {"note_id": None, "skipped_reason": "event already passed"}
+    if remind_at <= now:
+        remind_at = now
+
+    summary = (ev.summary or "(no title)").strip() or "(no title)"
+    location = (ev.location or "").strip()
+    start_fmt = start.strftime("%a %b %d") if ev.all_day else start.strftime("%a %b %d %H:%M")
+    loc = f" @ {location}" if location else ""
+    due_date = remind_at.isoformat() + ("Z" if getattr(ev, "is_utc", False) and not ev.all_day else "")
+    note = Note(
+        id=str(uuid.uuid4()),
+        owner=owner,
+        title=f"Calendar reminder: {summary}",
+        items=json.dumps([{"text": f"{summary}{loc} — {start_fmt}", "done": False, "checked": False}]),
+        note_type="todo",
+        label="calendar",
+        due_date=due_date,
+        source="calendar",
+    )
+    db.add(note)
+    return {"note_id": note.id, "due_date": due_date, "minutes": minutes_before, "skipped_reason": None}
+
+
+def _event_to_dict(ev: CalendarEvent, db=None, owner: str | None = None) -> dict:
     """Convert a CalendarEvent model to the API dict format.
 
     Timed events whose stored datetimes represent UTC (is_utc=True) are
@@ -637,6 +765,7 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
         suffix = "Z" if getattr(ev, "is_utc", False) else ""
         start_str = ev.dtstart.isoformat() + suffix
         end_str = ev.dtend.isoformat() + suffix
+    reminder = _calendar_reminder_for_event(db, owner, ev) if db and owner else None
     return {
         "uid": ev.uid,
         "summary": ev.summary or "",
@@ -653,6 +782,10 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
         "color": ev.color or (ev.calendar.color if ev.calendar else ""),
         "event_type": getattr(ev, "event_type", None),
         "importance": getattr(ev, "importance", None) or "normal",
+        "has_reminder": bool(reminder),
+        "reminder_note_id": reminder["note_id"] if reminder else None,
+        "reminder_due_date": reminder["due_date"] if reminder else None,
+        "reminder_minutes": reminder["minutes"] if reminder else None,
     }
 
 
@@ -684,7 +817,7 @@ def _occurrence_exdate_key(uid: str, ev: CalendarEvent) -> str:
 
 
 def _expand_rrule(
-    ev: CalendarEvent, start: datetime, end: datetime
+    ev: CalendarEvent, start: datetime, end: datetime, db=None, owner: str | None = None
 ) -> List[dict]:
     """Expand a single recurring CalendarEvent into occurrence dicts.
 
@@ -702,7 +835,7 @@ def _expand_rrule(
         # Non-recurring — return the base event as-is. list_events
         # already filters non-recurring rows with the overlap check
         # in SQL, so we don't re-check here.
-        d = _event_to_dict(ev)
+        d = _event_to_dict(ev, db=db, owner=owner)
         d["is_recurrence"] = False
         d["series_uid"] = ev.uid
         d["truncated"] = False
@@ -728,7 +861,7 @@ def _expand_rrule(
         logger.warning(
             "Failed to parse rrule=%r for event %s: %s", ev.rrule, ev.uid, ex
         )
-        d = _event_to_dict(ev)
+        d = _event_to_dict(ev, db=db, owner=owner)
         d["is_recurrence"] = False
         d["series_uid"] = ev.uid
         d["truncated"] = False
@@ -746,7 +879,7 @@ def _expand_rrule(
     expand_start = start - duration
     results = []
     truncated = False
-    base = _event_to_dict(ev)
+    base = _event_to_dict(ev, db=db, owner=owner)
     exdates = set(_recurrence_exdates(ev))
 
     for occ_start in rule.xafter(expand_start, inc=True):
@@ -1185,7 +1318,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
             # Expand recurring events into individual occurrences.
             expanded = []
             for e in events:
-                expanded.extend(_expand_rrule(e, start_dt, end_dt))
+                expanded.extend(_expand_rrule(e, start_dt, end_dt, db=db, owner=owner))
 
             # Sort by occurrence start time for consistent frontend ordering.
             truncated = any(e.get("truncated") for e in expanded)
@@ -1251,10 +1384,19 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 caldav_sync_pending="create" if cal.source == "caldav" else None,
             )
             db.add(ev)
+            reminder = None
+            if data.reminder_minutes is not None:
+                reminder = _create_calendar_reminder_for_event(db, owner, ev, data.reminder_minutes)
             db.commit()
+            db.refresh(ev)
             if cal.source == "caldav":
                 await _push_caldav_event_after_commit(owner, uid, "create")
-            return {"ok": True, "uid": uid}
+            return {
+                "ok": True,
+                "uid": uid,
+                "event": _event_to_dict(ev, db=db, owner=owner),
+                "reminder": reminder,
+            }
         except HTTPException:
             raise
         except Exception as e:
@@ -1264,6 +1406,17 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
         finally:
             db.close()
 
+    @router.get("/events/{uid}")
+    async def get_event(request: Request, uid: str):
+        owner = _require_user(request)
+        db = SessionLocal()
+        try:
+            base_uid = _resolve_base_uid(uid)
+            ev = _get_or_404_event(db, base_uid, owner)
+            return {"event": _event_to_dict(ev, db=db, owner=owner)}
+        finally:
+            db.close()
+
     @router.put("/events/{uid}")
     async def update_event(request: Request, uid: str, data: EventUpdate):
         owner = _require_user(request)
@@ -1300,13 +1453,24 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 ev.rrule = data.rrule
             if data.color is not None:
                 ev.color = data.color if data.color else None
+            reminder = None
+            reminder_fields = getattr(data, "model_fields_set", getattr(data, "__fields_set__", set()))
+            if "reminder_minutes" in reminder_fields:
+                _delete_calendar_reminders_for_event(db, owner, ev)
+                if data.reminder_minutes is not None:
+                    reminder = _create_calendar_reminder_for_event(db, owner, ev, data.reminder_minutes)
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
             if is_caldav:
                 ev.caldav_sync_pending = "update"
             db.commit()
+            db.refresh(ev)
             if is_caldav:
                 await _push_caldav_event_after_commit(owner, base_uid, "update")
-            return {"ok": True}
+            return {
+                "ok": True,
+                "event": _event_to_dict(ev, db=db, owner=owner),
+                "reminder": reminder,
+            }
         except HTTPException:
             raise
         except Exception as e:
@@ -1328,6 +1492,8 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
             ev = _get_or_404_event(db, base_uid, owner)
             is_occurrence_delete = scope in {"occurrence", "instance"} and "::" in uid and bool(ev.rrule)
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
+            if scope in {"occurrence", "instance"} and not is_occurrence_delete:
+                raise HTTPException(400, "Occurrence delete requires a recurring occurrence uid")
             if is_occurrence_delete:
                 key = _occurrence_exdate_key(uid, ev)
                 if not key:
@@ -1344,6 +1510,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 return {"ok": True, "scope": "occurrence", "exdate": key}
             if is_caldav:
                 _record_caldav_delete_tombstone(db, ev, owner)
+            _delete_calendar_reminders_for_event(db, owner, ev)
             db.delete(ev)
             db.commit()
             if is_caldav:
@@ -1423,7 +1590,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 raise HTTPException(400, f"Invalid ICS file: {e}")
 
             # Sanitize display name — length cap + strip control chars
-            raw_name = calendar_name.strip() or (file.filename or "").replace(".ics", "").replace("_", " ").strip() or "Imported"
+            raw_name = calendar_name.strip() or re.sub(r"\.(?:calendar|ics|ical)$", "", file.filename or "", flags=re.IGNORECASE).replace("_", " ").strip() or "Imported"
             cal_display = "".join(c for c in raw_name if c.isprintable())[:120] or "Imported"
 
             target_cal = db.query(CalendarCal).filter(

routes/chat_helpers.py

@@ -3,6 +3,7 @@
 import asyncio
 import json
 import logging
+import math
 import os
 import re
 import time
@@ -25,6 +26,56 @@ from fastapi import HTTPException
 
 logger = logging.getLogger(__name__)
 
+_INVISIBLE_RESPONSE_CHARS = "\u2063\u200b\u200c\u200d\ufeff"
+
+
+def _skill_run_is_complex(agent_rounds: int, agent_tool_calls: int) -> bool:
+    """Keep one-off TUI edit loops out of automatic skill extraction."""
+    return agent_tool_calls >= 4 or (agent_rounds >= 5 and agent_tool_calls >= 3)
+
+
+def clean_repeated_assistant_content(text: object) -> str:
+    """Collapse repeated terminal assistant prose before history/SFT storage."""
+    value = str(text or "")
+    for char in _INVISIBLE_RESPONSE_CHARS:
+        value = value.replace(char, "")
+    value = value.strip()
+    if not value:
+        return ""
+
+    # Stream rejoin/finalization races can concatenate the same complete
+    # answer without separators. Collapse only exact 2-4x repetitions.
+    for copies in range(4, 1, -1):
+        if len(value) % copies == 0:
+            width = len(value) // copies
+            unit = value[:width]
+            if unit and unit * copies == value:
+                value = unit.strip()
+                break
+
+    # Interrupted/rejoined streams can leave a short suffix before a closing
+    # think tag at the edge of visible prose, e.g. "ls.\n</think>\n\nHere's...".
+    edge_close_re = re.compile(r"(?is)^\s*(?!<\s*think\b)[^<\n]{0,120}\s*</\s*think\s*>\s*")
+    while True:
+        cleaned = edge_close_re.sub("", value, count=1).strip()
+        if cleaned == value:
+            break
+        value = cleaned
+
+    first_line = next((line.strip() for line in value.splitlines() if line.strip()), "")
+    if 8 <= len(first_line) <= 180:
+        matches = list(re.finditer(r"(?m)^" + re.escape(first_line) + r"\s*$", value))
+        if len(matches) >= 2:
+            value = value[matches[0].start():matches[1].start()].strip()
+
+    value = re.sub(
+        r"(?is)(?<=[.!?])(?:[a-z]{1,12}\.)\s*</\s*think\s*>\s*$",
+        "",
+        value,
+    ).strip()
+    value = re.sub(r"(?is)\s*</\s*think\s*>\s*$", "", value).strip()
+    return value
+
 _CASUAL_OPENING_RE = re.compile(
     r"^\s*(?:h+i+|hey+|hello+|yo+|sup+|what'?s up|wass?up|hiya|howdy|"
     r"lol|lmao|haha+|hehe+|thanks?|thank you|ty|idk|dunno|meh|bruh|bro)\b(?P<tail>.*)$",
@@ -36,6 +87,14 @@ _CASUAL_BLOCKLIST_RE = re.compile(
     r"file|folder|repo|git|settings?|endpoint|api|token|mcp)\b",
     re.IGNORECASE,
 )
+_PERSONAL_TOOL_CONTEXT_RE = re.compile(
+    r"\b(?:"
+    r"email|emails|mail|inbox|gmail|"
+    r"calendar|events?|meetings?|appointments?|schedule|"
+    r"notes?|todo|checklist|reminders?|tasks?"
+    r")\b",
+    re.IGNORECASE,
+)
 
 
 def _is_casual_low_signal(text: str) -> bool:
@@ -51,6 +110,14 @@ def _is_casual_low_signal(text: str) -> bool:
     return len(tail_words) <= 2
 
 
+def _truthy_request_flag(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return False
+    return str(value).strip().lower() in {"1", "true", "yes", "on"}
+
+
 # Strong references to in-flight fire-and-forget tasks scheduled from this
 # module. asyncio only keeps weak references to tasks created via
 # create_task, so without this the GC can collect a task mid-execution and
@@ -60,6 +127,197 @@ _BG_TASKS: set[asyncio.Task] = set()
 _INCOGNITO_CONTEXTS: dict[str, dict[str, Any]] = {}
 _INCOGNITO_CONTEXT_TTL_SECONDS = 6 * 60 * 60
 _INCOGNITO_CONTEXT_MAX_MESSAGES = 80
+_SFT_TRACE_CAPTURE_ENV = "ODYSSEUS_SFT_TRACE_CAPTURE"
+_SFT_TRACE_DIR_ENV = "ODYSSEUS_SFT_TRACE_DIR"
+_RUNTIME_REVISION_ENV = "ODYSSEUS_RUNTIME_REVISION"
+
+
+def _sft_trace_capture_enabled(owner: str | None) -> bool:
+    flag = os.getenv(_SFT_TRACE_CAPTURE_ENV, "1").strip().lower()
+    return flag not in {"0", "false", "no", "off"} and str(owner or "").startswith("sft_")
+
+
+def _json_safe(value: Any) -> Any:
+    try:
+        json.dumps(value)
+        return value
+    except TypeError:
+        return str(value)
+
+
+def _last_user_message_for_trace(sess) -> str:
+    for msg in reversed(getattr(sess, "history", []) or []):
+        if getattr(msg, "role", None) == "user":
+            return str(getattr(msg, "content", "") or "").strip()
+    return ""
+
+
+def _append_sft_trace_record(
+    *,
+    owner: str | None,
+    session_id: str,
+    sess,
+    assistant_content: str,
+    metadata: dict,
+    message_id: Any = None,
+) -> None:
+    """Append one training-ready trace record for synthetic SFT users."""
+    if not _sft_trace_capture_enabled(owner):
+        return
+    try:
+        from src.constants import DATA_DIR
+
+        trace_dir = os.getenv(_SFT_TRACE_DIR_ENV) or os.path.join(DATA_DIR, "sft_traces")
+        os.makedirs(trace_dir, exist_ok=True)
+        path = os.path.join(trace_dir, f"{owner}.jsonl")
+        runtime_revision = os.getenv(_RUNTIME_REVISION_ENV, "").strip()
+        record = {
+            "format": "odysseus_sft_trace_turn_v1",
+            "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "owner": owner,
+            "session_id": session_id,
+            "session_name": getattr(sess, "name", "") or "",
+            "message_id": message_id,
+            "user": _last_user_message_for_trace(sess),
+            "assistant": str(assistant_content or "").strip(),
+            "thinking": str((metadata or {}).get("thinking") or "").strip(),
+            "tool_events": _json_safe((metadata or {}).get("tool_events") or []),
+            "round_texts": _json_safe((metadata or {}).get("round_texts") or []),
+            "runtime_revision": runtime_revision,
+            "metadata": {
+                "model": (metadata or {}).get("model"),
+                "requested_model": (metadata or {}).get("requested_model"),
+                "endpoint_label": (metadata or {}).get("endpoint_label"),
+                "endpoint_id": (metadata or {}).get("endpoint_id"),
+                "response_time": (metadata or {}).get("response_time"),
+                "input_tokens": (metadata or {}).get("input_tokens"),
+                "output_tokens": (metadata or {}).get("output_tokens"),
+                "usage_buckets": _json_safe((metadata or {}).get("usage_buckets") or []),
+                "runtime_revision": runtime_revision,
+            },
+        }
+        _prune_sft_retry_rows_before_append(path, record)
+        with open(path, "a", encoding="utf-8") as f:
+            f.write(json.dumps(record, ensure_ascii=False) + "\n")
+    except Exception as exc:
+        logger.warning("Failed to append SFT trace record for %s/%s: %s", owner, session_id, exc)
+
+
+def remove_session_sft_trace_rows(owner: str | None, session_id: str) -> int:
+    """Remove every captured training row for a deleted synthetic session."""
+    if not _sft_trace_capture_enabled(owner) or not str(session_id or "").strip():
+        return 0
+    try:
+        from src.constants import DATA_DIR
+
+        trace_dir = os.getenv(_SFT_TRACE_DIR_ENV) or os.path.join(DATA_DIR, "sft_traces")
+        path = os.path.join(trace_dir, f"{owner}.jsonl")
+        if not os.path.exists(path):
+            return 0
+        kept: list[str] = []
+        removed: list[str] = []
+        with open(path, "r", encoding="utf-8") as source:
+            for line in source:
+                raw = line.rstrip("\n")
+                if not raw.strip():
+                    continue
+                try:
+                    row = json.loads(raw)
+                except json.JSONDecodeError:
+                    kept.append(raw)
+                    continue
+                if str(row.get("session_id") or "") != session_id:
+                    kept.append(raw)
+                    continue
+                row["deleted_from_training"] = True
+                removed.append(json.dumps(row, ensure_ascii=False))
+        if not removed:
+            return 0
+        tmp_path = f"{path}.{os.getpid()}.{time.time_ns()}.tmp"
+        with open(tmp_path, "w", encoding="utf-8") as target:
+            for raw in kept:
+                target.write(raw + "\n")
+        os.replace(tmp_path, path)
+        with open(path + ".trash", "a", encoding="utf-8") as trash:
+            for raw in removed:
+                trash.write(raw + "\n")
+        logger.info("Removed %d SFT trace row(s) for deleted session %s", len(removed), session_id)
+        return len(removed)
+    except Exception as exc:
+        logger.warning("Failed to remove SFT trace rows for session %s: %s", session_id, exc)
+        return 0
+
+
+def _prune_sft_retry_rows_before_append(path: str, record: dict[str, Any]) -> None:
+    """For SFT traces, keep only the latest retry for a repeated user send.
+
+    The browser resend flow can append a second identical user turn without
+    first calling the delete endpoint. Training wants the final attempt, not
+    both sends, so remove prior trailing rows in the same session with the same
+    user prompt before appending the replacement.
+    """
+    current_session = str(record.get("session_id") or "")
+    current_user = str(record.get("user") or "").strip()
+    if not current_session or not current_user or not os.path.exists(path):
+        return
+    kept: list[str] = []
+    parsed: list[tuple[str, dict | None]] = []
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            for line in f:
+                raw = line.rstrip("\n")
+                if not raw.strip():
+                    continue
+                try:
+                    parsed.append((raw, json.loads(raw)))
+                except json.JSONDecodeError:
+                    parsed.append((raw, None))
+
+        last_different_same_session = -1
+        for idx, (_raw, row) in enumerate(parsed):
+            if not isinstance(row, dict) or row.get("session_id") != current_session:
+                continue
+            if str(row.get("user") or "").strip() != current_user:
+                last_different_same_session = idx
+
+        removed: list[str] = []
+        for idx, (raw, row) in enumerate(parsed):
+            should_remove = (
+                idx > last_different_same_session
+                and isinstance(row, dict)
+                and row.get("session_id") == current_session
+                and str(row.get("user") or "").strip() == current_user
+            )
+            if should_remove:
+                tombstone = dict(row)
+                tombstone["deleted_from_training"] = True
+                tombstone["delete_reason"] = "sft_retry_replaced"
+                removed.append(json.dumps(tombstone, ensure_ascii=False))
+            else:
+                kept.append(raw)
+
+        if not removed:
+            return
+        with open(path, "w", encoding="utf-8") as f:
+            for raw in kept:
+                f.write(raw + "\n")
+        with open(path + ".trash", "a", encoding="utf-8") as f:
+            for raw in removed:
+                f.write(raw + "\n")
+        logger.info(
+            "Removed %d prior SFT retry row(s) before appending replacement for session %s",
+            len(removed),
+            current_session,
+        )
+    except Exception as exc:
+        logger.warning("Failed to prune prior SFT retry rows for %s: %s", current_session, exc)
+
+
+def strip_tui_local_context(content: Any) -> Any:
+    """Remove client-only workspace metadata before persistence/display."""
+    if not isinstance(content, str):
+        return content
+    return re.sub(r"\s*<local_context\b[^>]*>.*?</local_context>\s*", "", content, flags=re.IGNORECASE | re.DOTALL).strip()
 
 
 def _spawn_bg(coro) -> asyncio.Task:
@@ -113,6 +371,8 @@ class PresetInfo:
     max_tokens: Optional[int]
     system_prompt: Optional[str]
     character_name: Optional[str]
+    persona_memory: Optional[str] = None
+    persona_memory_schema: str = "general"
 
 
 @dataclass
@@ -251,6 +511,14 @@ def needs_auto_name(name: str) -> bool:
     return False
 
 
+def fallback_session_title(text: str, *, max_words: int = 6) -> str:
+    words = re.findall(r"[A-Za-z0-9@._'-]+", text)
+    if not words:
+        return "New chat"
+    title = " ".join(words[:max_words]).strip()
+    return title[:60] or "New chat"
+
+
 async def auto_name_session(session_manager, sess):
     """Generate a short title for a session from its first user message."""
     try:
@@ -273,6 +541,17 @@ async def auto_name_session(session_manager, sess):
         if not first_msg:
             return
 
+        endpoint_url = str(getattr(sess, "endpoint_url", "") or "")
+        model_name = str(getattr(sess, "model", "") or "")
+        if (
+            "ttft" in model_name.lower()
+            or re.search(r":18\d{3}\b", endpoint_url)
+        ):
+            title = fallback_session_title(first_msg)
+            session_manager.update_session_name(sess.id, title)
+            logger.info(f"Auto-named session {sess.id} deterministically: {title}")
+            return
+
         owner = getattr(sess, "owner", None)
         t_url, t_model, t_headers = resolve_task_endpoint(
             sess.endpoint_url, sess.model, sess.headers, owner=owner
@@ -294,9 +573,9 @@ async def auto_name_session(session_manager, sess):
                 {"role": "user", "content": first_msg},
             ],
             temperature=0.3,
-            max_tokens=4096,
+            max_tokens=64,
             headers=t_headers,
-            timeout=60,
+            timeout=15,
         )
 
         title = title.strip().strip('"\'').strip()
@@ -304,18 +583,47 @@ async def auto_name_session(session_manager, sess):
         # via the central helper.
         from src.text_helpers import strip_think
         title = strip_think(title, prose=False, prompt_echo=False)
-        if title and len(title) < 80:
-            session_manager.update_session_name(sess.id, title)
-            logger.info(f"Auto-named session {sess.id}: {title}")
+        if not title or len(title) >= 80 or "\n" in title:
+            fallback = fallback_session_title(first_msg)
+            session_manager.update_session_name(sess.id, fallback)
+            logger.info(
+                "Auto-named session %s with fallback title after unusable model title: %s",
+                sess.id,
+                fallback,
+            )
+            return
+
+        session_manager.update_session_name(sess.id, title)
+        logger.info(f"Auto-named session {sess.id}: {title}")
 
     except Exception as e:
         import traceback
         logger.error(f"Auto-name failed for {sess.id}: {e}\n{traceback.format_exc()}")
 
 
+async def auto_name_session_after_stream(session_id: str, session_manager, sess):
+    """Delay chat title generation until the first response stream is settled."""
+    try:
+        waited = 0.0
+        while _is_session_stream_active(session_id) and waited < 30.0:
+            await asyncio.sleep(0.25)
+            waited += 0.25
+        # Let the final SSE chunk/message_saved bookkeeping clear before any
+        # title model call can contend with the user's visible response.
+        await asyncio.sleep(0.5)
+        try:
+            sess = session_manager.get_session(session_id)
+        except Exception as e:
+            logger.warning("[auto-name] Could not reload session %s before naming: %s", session_id, e)
+        await auto_name_session(session_manager, sess)
+    except Exception as e:
+        import traceback
+        logger.error(f"Deferred auto-name failed for {session_id}: {e}\n{traceback.format_exc()}")
+
+
 def extract_preset(chat_handler, preset_id) -> PresetInfo:
     """Extract preset parameters via chat_handler."""
-    temperature, max_tokens, system_prompt, char_name = (
+    temperature, max_tokens, system_prompt, char_name, persona_memory, persona_memory_schema = (
         chat_handler.validate_and_extract_preset(preset_id)
     )
     return PresetInfo(
@@ -323,6 +631,8 @@ def extract_preset(chat_handler, preset_id) -> PresetInfo:
         max_tokens=max_tokens,
         system_prompt=system_prompt,
         character_name=char_name,
+        persona_memory=persona_memory,
+        persona_memory_schema=persona_memory_schema,
     )
 
 
@@ -406,14 +716,28 @@ def build_uploaded_file_manifest(att_ids: list, upload_handler, owner: Optional[
     return manifest
 
 
-def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, incognito: bool = False):
+def add_user_message(
+    sess,
+    chat_handler,
+    preprocessed: PreprocessedMessage,
+    incognito: bool = False,
+    interaction_mode: str | None = None,
+    auto_escalated: bool = False,
+):
     """Add user message to session history and update session name.
     Incognito messages must not mutate persistent session history, even in
     memory, because a later normal turn can persist the same session object."""
     if incognito:
         return
-    user_meta = {"attachments": preprocessed.attachment_meta} if preprocessed.attachment_meta else None
-    sess.add_message(ChatMessage("user", preprocessed.user_content, metadata=user_meta))
+    user_meta = {}
+    if preprocessed.attachment_meta:
+        user_meta["attachments"] = preprocessed.attachment_meta
+    if interaction_mode in {"chat", "agent", "research"}:
+        user_meta["interaction_mode"] = interaction_mode
+    if auto_escalated:
+        user_meta["auto_escalated"] = True
+    clean_content = strip_tui_local_context(preprocessed.user_content)
+    sess.add_message(ChatMessage("user", clean_content, metadata=user_meta or None))
     chat_handler.update_session_name_if_needed(sess, preprocessed.text_for_context)
 
 
@@ -626,6 +950,8 @@ async def build_chat_context(
     defer_context_shaping: bool = False,
     continuation_context_message: str | None = None,
     persist_user_message: bool = True,
+    interaction_mode: str | None = None,
+    auto_escalated: bool = False,
 ) -> ChatContext:
     """Build the full context (preface + messages) for an LLM call.
 
@@ -650,10 +976,23 @@ async def build_chat_context(
     # transcript store instead of session history so stale saved chats cannot
     # bleed into context and the turn is not persisted.
     if persist_user_message and incognito:
-        user_meta = {"attachments": preprocessed.attachment_meta} if preprocessed.attachment_meta else None
+        user_meta = {}
+        if preprocessed.attachment_meta:
+            user_meta["attachments"] = preprocessed.attachment_meta
+        if interaction_mode in {"chat", "agent", "research"}:
+            user_meta["interaction_mode"] = interaction_mode
+        if auto_escalated:
+            user_meta["auto_escalated"] = True
         _append_incognito_message(session_id, "user", preprocessed.user_content, user_meta)
     elif persist_user_message:
-        add_user_message(sess, chat_handler, preprocessed, incognito=False)
+        add_user_message(
+            sess,
+            chat_handler,
+            preprocessed,
+            incognito=False,
+            interaction_mode=interaction_mode,
+            auto_escalated=auto_escalated,
+        )
 
     # Fire events
     if persist_user_message and not incognito:
@@ -679,7 +1018,11 @@ async def build_chat_context(
     mem_enabled = not incognito and not no_memory and uprefs.get("memory_enabled", True)
     # Skills injection respects its own enable toggle (mirrors memory_enabled).
     # When off, the "Available skills" index is not added to the prompt.
-    skills_enabled = not incognito and uprefs.get("skills_enabled", True)
+    skills_enabled = (
+        not incognito
+        and uprefs.get("skills_enabled", True)
+        and getattr(sess, "skill_injection_enabled", True) is not False
+    )
     if not allow_tool_preprocessing:
         mem_enabled = False
         skills_enabled = False
@@ -704,8 +1047,17 @@ async def build_chat_context(
     if incognito or not allow_tool_preprocessing or is_research_spinoff or casual_low_signal:
         use_rag_val = False
 
-    # If pre-fetched search context was provided (compare mode), skip live web search
-    skip_web = bool(search_context) or not allow_tool_preprocessing or casual_low_signal
+    use_web_val = _truthy_request_flag(use_web)
+    # If pre-fetched search context was provided (compare mode), skip live web
+    # search. Personal app requests should be served by their tools; pre-search
+    # here caused calendar/email turns with use_web="false" to run irrelevant
+    # web searches before the agent even saw the tool surface.
+    skip_web = (
+        bool(search_context)
+        or not allow_tool_preprocessing
+        or casual_low_signal
+        or bool(agent_mode and _PERSONAL_TOOL_CONTEXT_RE.search(context_message or ""))
+    )
 
     # Build context preface
     # The stream path uses enhanced_message (with CoT/preprocessing applied),
@@ -722,12 +1074,13 @@ async def build_chat_context(
     _preface_kwargs = dict(
         message=_ctx_msg,
         session=sess,
-        use_web=use_web and not skip_web,
+        use_web=use_web_val and not skip_web,
         use_memory=mem_enabled,
         time_filter=time_filter,
         preset_system_prompt=preset.system_prompt,
         owner=user,
         character_name=preset.character_name,
+        persona_memory=preset.persona_memory,
         agent_mode=agent_mode,
         incognito=incognito,
         use_skills=skills_enabled,
@@ -826,10 +1179,17 @@ async def build_chat_context(
 
 
 def accumulate_token_usage(session_id: str, metrics: dict):
-    """Add input/output token counts to the session's running totals."""
+    """Add input/output token counts (and USD cost) to the session's totals."""
     in_t = metrics.get("input_tokens", 0)
     out_t = metrics.get("output_tokens", 0)
-    if not (in_t or out_t):
+    cost = metrics.get("cost_usd")
+    try:
+        cost = float(cost) if cost is not None else 0.0
+        if not math.isfinite(cost) or cost < 0:
+            cost = 0.0
+    except (TypeError, ValueError):
+        cost = 0.0
+    if not (in_t or out_t or cost):
         return
     db = SessionLocal()
     try:
@@ -837,6 +1197,8 @@ def accumulate_token_usage(session_id: str, metrics: dict):
         if db_s:
             db_s.total_input_tokens = (db_s.total_input_tokens or 0) + in_t
             db_s.total_output_tokens = (db_s.total_output_tokens or 0) + out_t
+            if cost:
+                db_s.total_cost_usd = (db_s.total_cost_usd or 0.0) + cost
             db.commit()
     except Exception:
         db.rollback()
@@ -889,6 +1251,21 @@ def _normalize_thinking(text: str) -> str:
 
     # Qwen3.5: "Thinking Process:" or "Thinking:" prefix
     if thinking_prefix_re.match(text.lstrip()):
+        # Tool-router checkpoints sometimes narrate several drafts and then
+        # emit an explicit final marker near the end. Prefer the last marker;
+        # the first ordinary-looking paragraph can still be internal review.
+        final_markers = list(re.finditer(
+            r"(?im)^\s*Final\s+(?:decision|answer|output(?:\s+generation)?)\s*:\s*",
+            text,
+        ))
+        if final_markers:
+            marker = final_markers[-1]
+            think = thinking_prefix_re.sub('', text[:marker.start()]).strip()
+            reply = text[marker.end():].strip()
+            if len(reply) >= 2 and reply[0] in {'\"', '\u201c'} and reply[-1] in {'\"', '\u201d'}:
+                reply = reply[1:-1].strip()
+            if reply:
+                return '<think>' + think + '</think>\n\n' + reply
         # Try clean boundary first
         m = re.match(
             r'^(Thinking(?:\s+Process)?:[\s\S]*?)(\n\n(?=[A-Z]|Hey|Yo|Hi|Sure|I |What|Here|Let|The |This |OK|Ok|Yes|No |So |Well |Thank|Alright|Of course|Absolutely|Great|Hello|As ))',
@@ -1017,6 +1394,23 @@ def clean_thinking_for_save(content: str, metadata: dict | None = None) -> tuple
         if info.get("time"):
             md["thinking_time"] = info["time"]
         return info["reply"], md
+    # A stopped stream can end before producing any answer prose. Preserve its
+    # partial reasoning as structured metadata so history rendering and the
+    # next Resume request can both recover it. Normal reasoning-only completed
+    # turns retain the legacy raw-content behavior.
+    if md.get("stopped"):
+        raw = str(content or "")
+        partial = re.match(
+            r'^\s*<think(?:ing)?(?:\s+time="([\d.]+)")?>([\s\S]*?)(?:</think(?:ing)?>\s*)?$',
+            raw,
+            re.IGNORECASE,
+        )
+        if partial and partial.group(2).strip():
+            md["thinking"] = partial.group(2).strip()
+            md["thinking_interrupted"] = True
+            if partial.group(1):
+                md["thinking_time"] = partial.group(1)
+            return "", md
     return content, md
 
 
@@ -1071,6 +1465,16 @@ def save_assistant_response(
     if tool_events:
         md["tool_events"] = tool_events
 
+    # The streaming route may have forwarded textual DSML/XML tool calls as
+    # deltas before the agent loop parsed them. Strip them again at the
+    # persistence boundary so raw tool markup cannot survive in history.
+    try:
+        from src.tool_parsing import strip_tool_blocks
+        full_response = strip_tool_blocks(str(full_response or "")).strip()
+    except Exception:
+        full_response = str(full_response or "")
+    full_response = clean_repeated_assistant_content(full_response)
+
     # Extract thinking into metadata (don't pollute message content with <think> tags)
     _think_info = _extract_thinking_meta(full_response)
     if _think_info:
@@ -1096,10 +1500,25 @@ def save_assistant_response(
     try:
         _last = sess.history[-1]
         _meta = getattr(_last, "metadata", None)
+        _message_id = _meta.get("_db_id") if isinstance(_meta, dict) else None
+        _append_sft_trace_record(
+            owner=getattr(sess, "owner", None),
+            session_id=session_id,
+            sess=sess,
+            assistant_content=_content,
+            metadata=md,
+            message_id=_message_id,
+        )
         if isinstance(_meta, dict):
-            return _meta.get("_db_id")
+            return _message_id
     except (IndexError, AttributeError):
-        pass
+        _append_sft_trace_record(
+            owner=getattr(sess, "owner", None),
+            session_id=session_id,
+            sess=sess,
+            assistant_content=_content,
+            metadata=md,
+        )
     return None
 
 
@@ -1172,6 +1591,8 @@ def run_post_response_tasks(
     owner: str = None,
     extract_skills: bool = True,
     allow_background_extraction: bool = True,
+    preset_manager=None,
+    persona_memory_schema: str = "general",
 ):
     """Fire background tasks after a completed response: memory extraction, webhooks, auto-name, skill extraction.
 
@@ -1192,7 +1613,8 @@ def run_post_response_tasks(
     # Memory extraction — only every 4th message pair to avoid excess LLM calls
     _msg_count = len(sess.history) if hasattr(sess, 'history') else 0
     _should_extract = (_msg_count >= 4) and (_msg_count % 4 == 0)
-    if allow_background_extraction and not incognito and not compare_mode and _should_extract and uprefs.get("auto_memory", True):
+    _chat_memory_extract = getattr(sess, "memory_extraction_enabled", True) is not False
+    if allow_background_extraction and not incognito and not compare_mode and _chat_memory_extract and _should_extract and uprefs.get("auto_memory", True):
         from services.memory.memory_extractor import extract_and_store
         from src.task_endpoint import resolve_task_endpoint
         t_url, t_model, t_headers = resolve_task_endpoint(
@@ -1203,6 +1625,27 @@ def run_post_response_tasks(
             t_url, t_model, t_headers,
         )))
 
+    if (
+        allow_background_extraction
+        and not incognito
+        and not compare_mode
+        and _chat_memory_extract
+        and _should_extract
+        and uprefs.get("auto_memory", True)
+        and character_name
+    ):
+        if preset_manager is not None:
+            from services.memory.memory_extractor import update_persona_memory
+            from src.task_endpoint import resolve_task_endpoint
+            p_url, p_model, p_headers = resolve_task_endpoint(
+                sess.endpoint_url, sess.model, sess.headers, owner=owner,
+            )
+            _extraction_jobs.append(("persona-memory", update_persona_memory(
+                sess, preset_manager, character_name,
+                p_url, p_model, p_headers,
+                schema=persona_memory_schema,
+            )))
+
     # Skill extraction from complex agent runs. Only when the user actually
     # chose agent mode — not a chat we auto-escalated for a notes/calendar
     # intent, and never in incognito/compare.
@@ -1217,13 +1660,17 @@ def run_post_response_tasks(
         extract_skills, auto_skills_enabled, incognito, compare_mode,
         agent_rounds, agent_tool_calls, "set" if skills_manager else "MISSING",
     )
+    # A normal inspect/edit/verify turn is commonly three calls. Treating that
+    # as a reusable skill creates one-off titles and makes the skill library
+    # noisy. Automatic extraction is reserved for runs that demonstrate a
+    # genuinely longer procedure; explicit skill tools remain unaffected.
     if (
         extract_skills
         and allow_background_extraction
         and auto_skills_enabled
         and not incognito
         and not compare_mode
-        and (agent_rounds >= 2 or agent_tool_calls >= 2)
+        and _skill_run_is_complex(agent_rounds, agent_tool_calls)
     ):
         if skills_manager is None:
             logger.warning(
@@ -1260,4 +1707,4 @@ def run_post_response_tasks(
 
     # Auto-name
     if needs_auto_name(sess.name):
-        _spawn_bg(auto_name_session(session_manager, sess))
+        _spawn_bg(auto_name_session_after_stream(session_id, session_manager, sess))

routes/contacts/contacts_routes.py

@@ -5,8 +5,10 @@ CardDAV contacts integration. Reads from local Radicale, supports
 search and adding new contacts.
 """
 
+import asyncio
 import re
 import logging
+import threading
 import uuid
 import json
 import csv
@@ -19,10 +21,11 @@ from datetime import datetime
 from urllib.parse import urljoin, urlparse, urlunparse
 
 from core.log_safety import redact_url
-from fastapi import APIRouter, Query, Depends, Response, HTTPException
+from fastapi import APIRouter, Query, Depends, Request, Response, HTTPException
 from typing import List, Dict, Optional
 
 from core.middleware import require_admin
+from src.auth_helpers import effective_user
 from src.url_safety import check_outbound_url
 
 logger = logging.getLogger(__name__)
@@ -93,22 +96,37 @@ def _normalize_contact(contact: Dict) -> Dict:
     if not name and emails:
         name = emails[0].split("@")[0]
     address = str(contact.get("address") or "").strip()
-    return {
+    out = {
         "uid": str(contact.get("uid") or uuid.uuid4()),
         "name": name,
         "emails": emails,
         "phones": phones,
         "address": address,
     }
+    owner = str(contact.get("owner") or "").strip()
+    if owner:
+        out["owner"] = owner
+    return out
 
 
-def _load_local_contacts() -> List[Dict]:
+def _contact_visible_to_owner(contact: Dict, owner: Optional[str]) -> bool:
+    owner = str(owner or "").strip()
+    row_owner = str(contact.get("owner") or "").strip()
+    if owner:
+        if row_owner:
+            return row_owner == owner
+        return not owner.startswith("sft_")
+    return True
+
+
+def _load_local_contacts(owner: Optional[str] = None) -> List[Dict]:
     try:
         if not LOCAL_CONTACTS_FILE.exists():
             return []
         data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
         rows = data.get("contacts", data) if isinstance(data, dict) else data
-        return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
+        contacts = [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
+        return [c for c in contacts if _contact_visible_to_owner(c, owner)]
     except Exception as e:
         logger.error(f"Failed to load local contacts: {e}")
         return []
@@ -119,7 +137,9 @@ def _save_local_contacts(contacts: List[Dict]) -> None:
     DATA_DIR.mkdir(parents=True, exist_ok=True)
     atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
     _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
+    _contact_cache["by_owner"] = {}
     _contact_cache["fetched_at"] = datetime.utcnow()
+    _contact_cache["failed_at"] = None
 
 
 # ── vCard parsing ──
@@ -264,7 +284,58 @@ def _build_vcard(name: str, email: str, uid: Optional[str] = None,
 
 # ── In-memory cache ──
 
-_contact_cache = {"contacts": [], "fetched_at": None}
+_CONTACT_CACHE_TTL_SECONDS = 60
+_CONTACT_FAILURE_BACKOFF_SECONDS = 120
+_CARDDAV_TIMEOUT = httpx.Timeout(5.0, connect=2.0)
+
+# CardDAV can be unavailable for a while. Keep the UI responsive by serving
+# the last known result (or an empty list on first use) while a single worker
+# attempts a refresh in the background.
+_contact_cache = {
+    "contacts": [],
+    "fetched_at": None,
+    "failed_at": None,
+    "by_owner": {},
+}
+_contact_fetch_lock = threading.Lock()
+
+
+def _cached_contacts(owner_key: str) -> List[Dict]:
+    cached = (_contact_cache.get("by_owner") or {}).get(owner_key) or {}
+    if owner_key and cached:
+        return cached.get("contacts") or []
+    return _contact_cache.get("contacts") or []
+
+
+def _mark_contact_fetch_failure(owner_key: str) -> List[Dict]:
+    now = datetime.utcnow()
+    stale_contacts = _cached_contacts(owner_key)
+    _contact_cache["failed_at"] = now
+    if owner_key:
+        _contact_cache.setdefault("by_owner", {})[owner_key] = {
+            "contacts": stale_contacts,
+            "fetched_at": now,
+        }
+    else:
+        _contact_cache["fetched_at"] = now
+    return stale_contacts
+
+
+def _contact_sync_status() -> Dict[str, str]:
+    """Return a safe, user-facing summary for contact autocomplete clients."""
+    if not _carddav_configured():
+        return {"state": "local", "message": "No contact sync is configured."}
+    if _contact_fetch_lock.locked():
+        return {"state": "syncing", "message": "Syncing contacts..."}
+    failed_at = _contact_cache.get("failed_at")
+    if failed_at:
+        age = (datetime.utcnow() - failed_at).total_seconds()
+        if age < _CONTACT_FAILURE_BACKOFF_SECONDS:
+            return {
+                "state": "unavailable",
+                "message": "Contacts sync is unavailable. Try again later.",
+            }
+    return {"state": "ready", "message": ""}
 
 
 def _abs_url(href: str) -> str:
@@ -306,7 +377,7 @@ def _fetch_via_report(cfg, auth):
             "REPORT", cfg["url"],
             content=_ADDRESSBOOK_QUERY.encode("utf-8"),
             headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
-            auth=auth, timeout=10,
+            auth=auth, timeout=_CARDDAV_TIMEOUT,
         )
         if r.status_code not in (207, 200):
             return None
@@ -337,20 +408,51 @@ def _fetch_via_report(cfg, auth):
         return None
 
 
-def _fetch_contacts(force=False):
+def _fetch_contacts(force=False, owner: Optional[str] = None):
     """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
-    if not force and _contact_cache["fetched_at"]:
+    owner_key = str(owner or "").strip()
+    by_owner = _contact_cache.setdefault("by_owner", {})
+    if owner_key and not force and owner_key in by_owner:
+        cached = by_owner.get(owner_key) or {}
+        fetched_at = cached.get("fetched_at")
+        if fetched_at:
+            age = (datetime.utcnow() - fetched_at).total_seconds()
+            if age < _CONTACT_CACHE_TTL_SECONDS:
+                return cached.get("contacts") or []
+
+    if not owner_key and not force and _contact_cache["fetched_at"]:
         age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
-        if age < 60:
+        if age < _CONTACT_CACHE_TTL_SECONDS:
             return _contact_cache["contacts"]
 
+    failed_at = _contact_cache.get("failed_at")
+    if not force and failed_at:
+        failure_age = (datetime.utcnow() - failed_at).total_seconds()
+        if failure_age < _CONTACT_FAILURE_BACKOFF_SECONDS:
+            return _cached_contacts(owner_key)
+
+    # SFT users must not see the operator's personal/CardDAV contact book.
+    # Their training contacts are seeded as owner-scoped local rows.
+    if owner_key.startswith("sft_"):
+        contacts = _load_local_contacts(owner_key)
+        by_owner[owner_key] = {"contacts": contacts, "fetched_at": datetime.utcnow()}
+        return contacts
+
     cfg = _get_carddav_config()
     if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
+        contacts = _load_local_contacts(owner_key or None)
+        if owner_key:
+            by_owner[owner_key] = {"contacts": contacts, "fetched_at": datetime.utcnow()}
+        else:
+            _contact_cache["contacts"] = contacts
+            _contact_cache["fetched_at"] = datetime.utcnow()
         return contacts
 
+    # Do not let a burst of typeahead requests start parallel CardDAV timeouts.
+    # A caller that arrives during a refresh gets the most recent cache instead.
+    if not _contact_fetch_lock.acquire(blocking=False):
+        return _cached_contacts(owner_key)
+
     try:
         cfg["url"] = _carddav_base_url(cfg)
         auth = None
@@ -360,17 +462,23 @@ def _fetch_contacts(force=False):
         contacts = _fetch_via_report(cfg, auth)
         if contacts is None:
             # Fallback: plain GET, concatenated vCards, no hrefs.
-            r = httpx.get(cfg["url"], auth=auth, timeout=10)
+            r = httpx.get(cfg["url"], auth=auth, timeout=_CARDDAV_TIMEOUT)
             if r.status_code != 200:
                 logger.warning(f"CardDAV returned {r.status_code}")
-                return _contact_cache["contacts"]
+                return _mark_contact_fetch_failure(owner_key)
             contacts = _parse_vcards(r.text)
+        fetched_at = datetime.utcnow()
         _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
+        _contact_cache["fetched_at"] = fetched_at
+        _contact_cache["failed_at"] = None
+        if owner_key:
+            by_owner[owner_key] = {"contacts": contacts, "fetched_at": fetched_at}
         return contacts
     except Exception as e:
         logger.error(f"Failed to fetch contacts: {e}")
-        return _contact_cache["contacts"]
+        return _mark_contact_fetch_failure(owner_key)
+    finally:
+        _contact_fetch_lock.release()
 
 
 def _resolve_resource_url(uid: str) -> str:
@@ -394,25 +502,31 @@ def _resolve_resource_url(uid: str) -> str:
     return _lookup() or _vcard_url(uid)
 
 
-def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None) -> bool:
+def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None, owner: Optional[str] = None) -> bool:
     """Add a new contact via CardDAV or local contacts."""
     email = (email or "").strip()
     phone_list = [str(p or "").strip() for p in (phones or []) if str(p or "").strip()]
     cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
+    owner_key = str(owner or "").strip()
+    if owner_key.startswith("sft_") or not _carddav_configured(cfg):
         contacts = _load_local_contacts()
         email_l = email.lower()
         for c in contacts:
+            if owner_key and not _contact_visible_to_owner(c, owner_key):
+                continue
             if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
                 return True
             if phone_list and any(p in (c.get("phones") or []) for p in phone_list):
                 return True
-        contacts.append(_normalize_contact({
+        row = {
             "name": name,
             "emails": [email] if email else [],
             "phones": phone_list,
             "address": address,
-        }))
+        }
+        if owner_key:
+            row["owner"] = owner_key
+        contacts.append(_normalize_contact(row))
         _save_local_contacts(contacts)
         return True
 
@@ -650,24 +764,34 @@ def _contacts_to_csv(contacts: List[Dict]) -> str:
     return out.getvalue()
 
 
-def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
+def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "", owner: Optional[str] = None) -> bool:
     """Rewrite an existing contact via CardDAV or local contacts."""
     cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
+    owner_key = str(owner or "").strip()
+    if owner_key.startswith("sft_") or not _carddav_configured(cfg):
         contacts = _load_local_contacts()
         found = False
         out = []
         for c in contacts:
             if c.get("uid") == uid:
+                if owner_key and not _contact_visible_to_owner(c, owner_key):
+                    out.append(c)
+                    continue
                 # Preserve existing address when caller passes "" (only
                 # updating name/emails/phones, not touching address).
                 addr = address if address else c.get("address", "")
-                out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
+                row = {"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}
+                if owner_key:
+                    row["owner"] = owner_key
+                out.append(_normalize_contact(row))
                 found = True
             else:
                 out.append(c)
         if not found:
-            out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
+            row = {"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}
+            if owner_key:
+                row["owner"] = owner_key
+            out.append(_normalize_contact(row))
         _save_local_contacts(out)
         return True
 
@@ -694,12 +818,16 @@ def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], a
         return False
 
 
-def _delete_contact(uid: str) -> bool:
+def _delete_contact(uid: str, owner: Optional[str] = None) -> bool:
     """Delete a contact via CardDAV or local contacts."""
     cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
+    owner_key = str(owner or "").strip()
+    if owner_key.startswith("sft_") or not _carddav_configured(cfg):
         contacts = _load_local_contacts()
-        remaining = [c for c in contacts if c.get("uid") != uid]
+        remaining = [
+            c for c in contacts
+            if c.get("uid") != uid or (owner_key and not _contact_visible_to_owner(c, owner_key))
+        ]
         _save_local_contacts(remaining)
         return True
 
@@ -739,17 +867,17 @@ def setup_contacts_routes():
     router = APIRouter(prefix="/api/contacts", tags=["contacts"])
 
     @router.get("/list")
-    async def list_contacts(_admin: str = Depends(require_admin)):
+    async def list_contacts(request: Request, _admin: str = Depends(require_admin)):
         """List all contacts."""
-        contacts = _fetch_contacts()
-        return {"contacts": contacts, "count": len(contacts)}
+        contacts = await asyncio.to_thread(_fetch_contacts, owner=effective_user(request))
+        return {"contacts": contacts, "count": len(contacts), "sync": _contact_sync_status()}
 
     @router.get("/search")
-    async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
+    async def search_contacts(request: Request, q: str = Query(""), _admin: str = Depends(require_admin)):
         """Search contacts by name or email. Returns up to 10 matches."""
-        contacts = _fetch_contacts()
+        contacts = await asyncio.to_thread(_fetch_contacts, owner=effective_user(request))
         if not q:
-            return {"results": []}
+            return {"results": [], "sync": _contact_sync_status()}
         q_lower = q.lower()
         results = []
         for c in contacts:
@@ -760,11 +888,12 @@ def setup_contacts_routes():
                 if q_lower in em.lower():
                     results.append(c)
                     break
-        return {"results": results[:10]}
+        return {"results": results[:10], "sync": _contact_sync_status()}
 
     @router.post("/add")
-    async def add_contact(data: dict, _admin: str = Depends(require_admin)):
+    async def add_contact(data: dict, request: Request, _admin: str = Depends(require_admin)):
         """Add a new contact."""
+        owner = effective_user(request)
         name = (data.get("name") or "").strip()
         email = (data.get("email") or "").strip()
         phone = (data.get("phone") or "").strip()
@@ -778,17 +907,20 @@ def setup_contacts_routes():
             return {"success": False, "error": "Name, email, phone, or address required"}
         if not name:
             name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
-        contacts = _fetch_contacts()
+        contacts = _fetch_contacts(owner=owner)
         for c in contacts:
             if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
                 return {"success": True, "message": "Already exists", "contact": c}
             if phones and any(p in (c.get("phones") or []) for p in phones):
                 return {"success": True, "message": "Already exists", "contact": c}
         create_params = inspect.signature(_create_contact).parameters
-        if "phones" in create_params:
-            ok = _create_contact(name, email, address, phones=phones)
-        elif len(create_params) >= 3:
-            ok = _create_contact(name, email, address)
+        if len(create_params) >= 3:
+            create_kwargs = {}
+            if "phones" in create_params:
+                create_kwargs["phones"] = phones
+            if "owner" in create_params:
+                create_kwargs["owner"] = owner
+            ok = _create_contact(name, email, address, **create_kwargs)
         else:
             ok = _create_contact(name, email)
         # If a phone was provided, do an immediate update to thread it
@@ -796,7 +928,7 @@ def setup_contacts_routes():
         # email + address; phones happen via update).
         if ok and phones and "phones" not in create_params:
             try:
-                fresh = _fetch_contacts(force=True)
+                fresh = _fetch_contacts(force=True, owner=owner)
                 created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
                 if created:
                     _update_contact(
@@ -804,6 +936,7 @@ def setup_contacts_routes():
                         created.get("emails", []),
                         phones,
                         address,
+                        owner=owner,
                     )
             except Exception:
                 pass
@@ -830,11 +963,16 @@ def setup_contacts_routes():
 
     @router.get("/export")
     async def export_contacts(
+        request: Request,
         format: str = Query("vcf", pattern="^(vcf|csv)$"),
         _admin: str = Depends(require_admin),
     ):
         """Export all contacts as vCard or CSV."""
-        contacts = _fetch_contacts(force=True)
+        contacts = await asyncio.to_thread(
+            _fetch_contacts,
+            force=True,
+            owner=effective_user(request),
+        )
         if format == "csv":
             content = _contacts_to_csv(contacts)
             media_type = "text/csv; charset=utf-8"
@@ -876,19 +1014,28 @@ def setup_contacts_routes():
         _save_settings(settings)
         # Force re-fetch
         _contact_cache["fetched_at"] = None
+        _contact_cache["failed_at"] = None
         return {"success": True}
 
     @router.delete("/clear")
-    async def clear_contacts(_admin: str = Depends(require_admin)):
+    async def clear_contacts(request: Request, _admin: str = Depends(require_admin)):
         """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
-        _save_local_contacts([])
+        owner = effective_user(request)
+        if owner:
+            remaining = [
+                c for c in _load_local_contacts()
+                if not _contact_visible_to_owner(c, owner)
+            ]
+            _save_local_contacts(remaining)
+        else:
+            _save_local_contacts([])
         return {"success": True}
 
     # NOTE: the /{uid} routes are declared LAST so the literal paths above
     # (/list, /search, /add, /config) win — otherwise PUT /config would
     # match PUT /{uid} with uid="config".
     @router.put("/{uid}")
-    async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
+    async def edit_contact(uid: str, data: dict, request: Request, _admin: str = Depends(require_admin)):
         """Edit an existing contact — name / emails / phones / address."""
         name = (data.get("name") or "").strip()
         emails = data.get("emails")
@@ -902,15 +1049,15 @@ def setup_contacts_routes():
             return {"success": False, "error": "Name, email, or address required"}
         if not name and emails:
             name = emails[0].split("@")[0]
-        ok = _update_contact(uid, name, emails, phones, address)
+        ok = _update_contact(uid, name, emails, phones, address, owner=effective_user(request))
         return {"success": ok}
 
     @router.delete("/{uid}")
-    async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
+    async def delete_contact(uid: str, request: Request, _admin: str = Depends(require_admin)):
         """Delete a contact by UID."""
         if not uid:
             return {"success": False, "error": "UID required"}
-        ok = _delete_contact(uid)
+        ok = _delete_contact(uid, owner=effective_user(request))
         return {"success": ok}
 
     return router

routes/cookbook_helpers.py

@@ -1085,6 +1085,10 @@ class ServeRequest(BaseModel):
     hf_token: str | None = None
     gpus: str | None = None
     platform: str | None = None    # "linux", "termux", or "windows"
+    # Optional explicit image runtime adapter. "auto" preserves compatibility
+    # with older callers; catalog-backed launches can set this without relying
+    # on model-name heuristics in the generated runner.
+    runtime_adapter: str | None = None
 
 
 def _parse_serve_phase(snapshot: str, task_type: str = "serve") -> dict:

routes/cookbook_routes.py

@@ -114,6 +114,16 @@ def _append_mlx_image_server_script(runner_lines: list[str]) -> None:
     runner_lines.append('chmod +x scripts/mlx_image_server.py 2>/dev/null || true')
 
 
+def _normalize_runtime_adapter(value: str | None) -> str:
+    """Return a shell-safe explicit image adapter name."""
+    value = (value or "auto").strip().lower()
+    if not value:
+        return "auto"
+    if not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,39}", value):
+        raise HTTPException(400, "Invalid runtime adapter")
+    return value
+
+
 def _venv_root_from_serve_cmd(cmd: str) -> str:
     """Best-effort venv root from an absolute venv python in a serve command."""
     try:
@@ -1411,7 +1421,6 @@ def setup_cookbook_routes() -> APIRouter:
         # unvalidated value (e.g. "x'; rm -rf ~ #") would be command injection.
         host = validate_remote_host(host)
         ssh_port = validate_ssh_port(ssh_port)
-        TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True)
 
         model_dirs = []
         if model_dir:
@@ -1423,20 +1432,17 @@ def setup_cookbook_routes() -> APIRouter:
                     model_dirs.append(d)
         paths_code = _cached_model_scan_script(model_dirs)
 
-        scan_py = TMUX_LOG_DIR / "scan_cache.py"
-        scan_py.write_text(paths_code, encoding="utf-8")
-
         async def _run_cached_scan_once():
+            # Each request owns its script bytes. A shared scan_cache.py races
+            # when the tool scans several hosts/directories concurrently.
             if host:
-                _ssh_opts = "-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=4 -o ServerAliveCountMax=1 "
-                _pf = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
-                if platform == "windows":
-                    # Windows: use 'python' and pipe via stdin with double-quote wrapping
-                    cmd = f'ssh {_ssh_opts}{_pf}{host} "python -" < \'{scan_py}\''
-                else:
-                    cmd = f"ssh {_ssh_opts}{_pf}{host} 'python3 -' < '{scan_py}'"
-                proc = await asyncio.create_subprocess_shell(
-                    cmd,
+                ssh_args = ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8',
+                            '-o', 'ServerAliveInterval=4', '-o', 'ServerAliveCountMax=1']
+                if ssh_port and ssh_port != '22':
+                    ssh_args.extend(['-p', ssh_port])
+                proc = await asyncio.create_subprocess_exec(
+                    *ssh_args, host, 'python -' if platform == 'windows' else 'python3 -',
+                    stdin=asyncio.subprocess.PIPE,
                     stdout=asyncio.subprocess.PIPE,
                     stderr=asyncio.subprocess.PIPE,
                     cwd=str(Path.home()),
@@ -1454,12 +1460,31 @@ def setup_cookbook_routes() -> APIRouter:
                     or which_tool("py") or "python"
                 )
                 proc = await asyncio.create_subprocess_exec(
-                    local_py, str(scan_py),
+                    local_py, '-',
+                    stdin=asyncio.subprocess.PIPE,
                     stdout=asyncio.subprocess.PIPE,
                     stderr=asyncio.subprocess.PIPE,
                     cwd=str(Path.home()),
                 )
-            return await asyncio.wait_for(proc.communicate(), timeout=60), proc.returncode
+            try:
+                output = await asyncio.wait_for(proc.communicate(paths_code.encode('utf-8')), timeout=60)
+                return output, proc.returncode
+            finally:
+                # A timed-out/cancelled request must not abandon its scanner.
+                # This handle belongs only to this request, never a model job.
+                if proc.returncode is None:
+                    try:
+                        proc.terminate()
+                    except ProcessLookupError:
+                        pass
+                    try:
+                        await asyncio.wait_for(proc.wait(), timeout=2)
+                    except asyncio.TimeoutError:
+                        try:
+                            proc.kill()
+                        except ProcessLookupError:
+                            pass
+                        await asyncio.wait_for(proc.wait(), timeout=2)
 
         (stdout_b, stderr_b), returncode = await _run_cached_scan_once()
         stderr_txt = stderr_b.decode(errors="replace").strip()
@@ -1974,6 +1999,7 @@ def setup_cookbook_routes() -> APIRouter:
         validate_remote_host(req.remote_host)
         req.ssh_port = validate_ssh_port(req.ssh_port)
         req.gpus = _validate_gpus(req.gpus)
+        req.runtime_adapter = _normalize_runtime_adapter(req.runtime_adapter)
         req.hf_token = req.hf_token or _load_stored_hf_token()
         _validate_token(req.hf_token)
         # Cookbook emits two fixed Docker exec forms for its Ollama sidecars.
@@ -2602,19 +2628,20 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('print(model)')
                 runner_lines.append('PY')
                 runner_lines.append(')"')
-                runner_lines.append('if printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi hidream; then')
+                runner_lines.append(f"export ODYSSEUS_MLX_IMAGE_ADAPTER='{_bash_squote(req.runtime_adapter or 'auto')}'")
+                runner_lines.append('if [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "hidream" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi hidream; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import mlx, mlx_vlm, transformers, huggingface_hub, safetensors, numpy, PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: HiDream MLX serving needs the model requirements in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart mlx mlx-vlm \'transformers>=4.57.0,<6.0\' huggingface_hub safetensors numpy pillow tqdm sentencepiece hf_transfer"')
                 runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('  fi')
-                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi boogu; then')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "boogu" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi boogu; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import boogu_image_mlx, mlx, huggingface_hub, safetensors, numpy, PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: Boogu MLX serving needs boogu-image-mlx in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U git+https://github.com/xocialize/boogu-image-mlx.git fastapi uvicorn python-multipart pillow"')
                 runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('  fi')
-                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "ddcolor"; then')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "ddcolor" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "ddcolor"; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: DDColor MLX serving needs Pillow in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart pillow huggingface_hub"')
@@ -2634,7 +2661,7 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('      ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('    fi')
                 runner_lines.append('  fi')
-                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "mi-gan|migan|lama"; then')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "inpaint" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "mi-gan|migan|lama"; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: LaMa / MI-GAN MLX serving needs Pillow in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart pillow huggingface_hub"')
@@ -2654,10 +2681,12 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('      ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('    fi')
                 runner_lines.append('  fi')
-                runner_lines.append('elif ! command -v mflux-generate >/dev/null 2>&1 && ! command -v mflux-generate-qwen >/dev/null 2>&1; then')
-                runner_lines.append('  echo "ERROR: mflux-compatible MLX image serving requires mflux-generate or mflux-generate-qwen in PATH for launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
-                runner_lines.append('  echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U mflux fastapi uvicorn python-multipart"')
-                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "mflux" ] || [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ]; then')
+                runner_lines.append('  if ! command -v mflux-generate >/dev/null 2>&1 && ! command -v mflux-generate-qwen >/dev/null 2>&1; then')
+                runner_lines.append('    echo "ERROR: mflux-compatible MLX image serving requires mflux-generate or mflux-generate-qwen in PATH for launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
+                runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U mflux fastapi uvicorn python-multipart"')
+                runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('  fi')
                 runner_lines.append('fi')
             elif "scripts/diffusion_server.py" in req.cmd or ".diffusion_server.py" in req.cmd:
                 runner_lines.append('export PATH="$HOME/.local/bin:$PATH"')
@@ -3516,12 +3545,19 @@ def setup_cookbook_routes() -> APIRouter:
             return {"ok": False, "error": str(e)}
 
     @router.get("/api/cookbook/hf-latest")
-    async def hf_latest(vram_gb: float = 0, limit: int = 10, pipeline: str = "text-generation", owner: str = Depends(require_user)):
+    async def hf_latest(
+        vram_gb: float = 0,
+        limit: int = 10,
+        pipeline: str = "text-generation",
+        official_only: bool = False,
+        owner: str = Depends(require_user),
+    ):
         """Fetch latest HuggingFace models, filtered by what fits in available VRAM.
 
         vram_gb: total available VRAM in GB. 0 = no filter (return everything).
         limit:   how many models to return (default 10).
         pipeline: HF pipeline_tag filter (text-generation, text-to-image, etc.).
+        official_only: restrict results to recognized first-party provider namespaces.
         """
         import re
         import httpx
@@ -3587,6 +3623,20 @@ def setup_cookbook_routes() -> APIRouter:
                     return True
             return False
 
+        # HF does not expose a universal "first-party" flag. Keep this as a
+        # namespace policy rather than a model-name list, so newly published
+        # provider models are included without recommending community forks.
+        OFFICIAL_NAMESPACES = {
+            "apple", "black-forest-labs", "deepseek-ai", "google", "lightricks",
+            "meta-llama", "microsoft", "mistralai", "nvidia", "openai", "qwen",
+            "stabilityai", "tencent", "runwayml",
+        }
+
+        def _is_official(entry: dict, repo_id: str) -> bool:
+            namespace = repo_id.split("/", 1)[0].strip().lower() if "/" in repo_id else ""
+            author = str(entry.get("author") or "").strip().lower()
+            return namespace in OFFICIAL_NAMESPACES and (not author or author == namespace)
+
         out = []
         for entry in raw:
             repo_id = entry.get("modelId") or entry.get("id") or ""
@@ -3601,6 +3651,8 @@ def setup_cookbook_routes() -> APIRouter:
             # Skip adapters, LoRAs, datasets, etc.
             if _is_excluded(repo_id, tags):
                 continue
+            if official_only and not _is_official(entry, repo_id):
+                continue
 
             est_fp16 = _est_vram_fp16(repo_id)
             quant_mult = _quant_factor(repo_id, tags)
@@ -3614,7 +3666,11 @@ def setup_cookbook_routes() -> APIRouter:
                     # if we cannot estimate size from the repo id/tags, do not
                     # present it as runnable on this hardware.
                     continue
-                if needed_vram > vram_gb:
+                # Leave allocator/runtime headroom instead of treating the
+                # reported total as a safe load budget. This keeps the
+                # official-only list honest on tight GPUs as well.
+                usable_vram = vram_gb * 0.90
+                if needed_vram > usable_vram:
                     continue
 
             out.append({
@@ -4412,6 +4468,7 @@ def setup_cookbook_routes() -> APIRouter:
 
             progress_text = ""
             full_snapshot = (task.get("output") or "")[-12000:] if task_type == "serve" else ""
+            _persisted_terminal = False
 
             if local_win_task:
                 # File-based liveness + output for the detached-process model.
@@ -4445,9 +4502,10 @@ def setup_cookbook_routes() -> APIRouter:
                     and bool(full_snapshot)
                     and _parse_serve_phase(full_snapshot, task_type).get("status") == "ready"
                 )
-                if _task_status in {"stopped", "done", "completed",
+                _persisted_terminal = _task_status in {"stopped", "done", "completed",
                                     "crashed", "error", "failed",
-                                    "ended", "killed"} and not _persisted_serve_ready:
+                                    "ended", "killed"} and not _persisted_serve_ready
+                if _persisted_terminal:
                     is_alive = False
                     # Keep the persisted output_tail for the UI — it's
                     # what the agent uses to diagnose past failures.
@@ -4486,7 +4544,9 @@ def setup_cookbook_routes() -> APIRouter:
                 and (
                     ".incomplete" in full_snapshot
                     or bool(re.search(r'model-\d+-of-\d+\.[A-Za-z0-9_.-]+:\s+(?:[0-9]|[1-8][0-9])%', full_snapshot))
-                    or _download_cache_incomplete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "")
+                    or (not _persisted_terminal and _download_cache_incomplete(
+                        _payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or ""
+                    ))
                 )
             )
             if is_alive or (local_win_task and full_snapshot):
@@ -4538,6 +4598,7 @@ def setup_cookbook_routes() -> APIRouter:
                         progress_text = "Download complete"
                 elif (
                     task_type == "download"
+                    and not _persisted_terminal
                     and not download_has_incomplete_evidence
                     and _download_cache_complete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "")
                 ):

routes/document/document_routes.py

@@ -6,6 +6,7 @@ from datetime import datetime, timezone
 from typing import Dict, Any, List, Optional
 
 from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Form
+from fastapi.responses import HTMLResponse
 
 from sqlalchemy import case, func, or_
 from core.database import SessionLocal, Document, DocumentVersion
@@ -479,6 +480,32 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
         finally:
             db.close()
 
+    # ---- GET /api/document/{doc_id}/visual-report ----
+    @router.get("/api/document/{doc_id}/visual-report", response_class=HTMLResponse)
+    async def document_visual_report(request: Request, doc_id: str) -> HTMLResponse:
+        """Render a Markdown document with the same standalone report UI used by Deep Research."""
+        user = get_current_user(request)
+        db = SessionLocal()
+        try:
+            doc = db.query(Document).filter(Document.id == doc_id).first()
+            if not doc:
+                raise HTTPException(404, "Document not found")
+            _verify_doc_owner(db, doc, user)
+            if (doc.language or "").lower() != "markdown":
+                raise HTTPException(400, "Visual reports are available for Markdown documents")
+
+            from src.visual_report import generate_visual_report
+
+            html_content = generate_visual_report(
+                question=doc.title or "Document",
+                report_markdown=doc.current_content or "",
+                sources=[],
+                stats={},
+            )
+            return HTMLResponse(content=html_content)
+        finally:
+            db.close()
+
     # ---- POST /api/document/{doc_id}/archive — soft-archive / restore ----
     @router.post("/api/document/{doc_id}/archive")
     async def archive_document(request: Request, doc_id: str, archived: bool = Query(True)) -> Dict[str, Any]:
@@ -575,7 +602,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
             "markdown": ".md", "json": ".json", "yaml": ".yml", "bash": ".sh",
             "sql": ".sql", "rust": ".rs", "go": ".go", "java": ".java", "c": ".c",
             "cpp": ".cpp", "typescript": ".ts", "ruby": ".rb", "php": ".php",
-            "text": ".txt", "xml": ".xml", "toml": ".toml", "ini": ".ini",
+            "text": ".txt", "email": ".eml", "xml": ".xml", "toml": ".toml", "ini": ".ini",
         }
         db = SessionLocal()
         try:
@@ -602,7 +629,10 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                         name = f"{base}-{i}" + ("" if "." in base else ext)
                         i += 1
                     used.add(name)
-                    zf.writestr(name, doc.current_content or "")
+                    content = doc.current_content or ""
+                    if (doc.language or "").lower() == "email":
+                        content = re.sub(r"\r?\n---\r?\n", "\r\n\r\n", content, count=1)
+                    zf.writestr(name, content)
                     wrote += 1
             if not wrote:
                 raise HTTPException(404, "No documents found")

routes/editor_draft_routes.py

@@ -26,6 +26,7 @@ from pydantic import BaseModel
 
 from core.database import EditorDraft, SessionLocal
 from src.auth_helpers import get_current_user
+from src.upload_limits import EDITOR_DRAFT_MAX_BYTES
 
 logger = logging.getLogger(__name__)
 
@@ -75,6 +76,16 @@ def _load_payload(raw: Optional[str]) -> Dict[str, Any]:
     return payload if isinstance(payload, dict) else {}
 
 
+def _dump_payload(payload: Dict[str, Any]) -> str:
+    raw = json.dumps(payload or {}, separators=(",", ":"))
+    if len(raw.encode("utf-8")) > EDITOR_DRAFT_MAX_BYTES:
+        raise HTTPException(
+            413,
+            f"Editor draft exceeds the {EDITOR_DRAFT_MAX_BYTES // (1024 * 1024)} MB safety limit",
+        )
+    return raw
+
+
 def setup_editor_draft_routes() -> APIRouter:
     router = APIRouter(tags=["editor-drafts"])
 
@@ -120,13 +131,15 @@ def setup_editor_draft_routes() -> APIRouter:
                 source_image_id=body.source_image_id,
                 width=body.width,
                 height=body.height,
-                payload=json.dumps(body.payload or {}),
+                payload=_dump_payload(body.payload),
                 thumbnail=body.thumbnail,
             )
             db.add(d)
             db.commit()
             db.refresh(d)
             return _summary(d)
+        except HTTPException:
+            raise
         except Exception as e:
             db.rollback()
             logger.warning(f"editor-draft create failed: {e}")
@@ -151,7 +164,7 @@ def setup_editor_draft_routes() -> APIRouter:
             if body.height is not None:
                 d.height = body.height
             if body.payload is not None:
-                d.payload = json.dumps(body.payload)
+                d.payload = _dump_payload(body.payload)
             if body.thumbnail is not None:
                 d.thumbnail = body.thumbnail
             db.commit()

routes/email_helpers.py

@@ -886,10 +886,16 @@ def _init_scheduled_db():
             size INTEGER DEFAULT 0,
             flags TEXT DEFAULT '',
             has_attachments INTEGER DEFAULT 0,
+            attachment_names TEXT DEFAULT '',
             updated_at TEXT NOT NULL,
             PRIMARY KEY (owner, account_key, folder, uid)
         )
     """)
+    _message_index_cols = {
+        row[1] for row in conn.execute("PRAGMA table_info(email_message_index)").fetchall()
+    }
+    if "attachment_names" not in _message_index_cols:
+        conn.execute("ALTER TABLE email_message_index ADD COLUMN attachment_names TEXT DEFAULT ''")
     conn.execute("""
         CREATE INDEX IF NOT EXISTS ix_email_message_index_folder_date
         ON email_message_index(owner, account_key, folder, date_epoch DESC)
@@ -1667,7 +1673,15 @@ def _extract_text(msg):
         payload = msg.get_payload(decode=True)
         if payload:
             charset = msg.get_content_charset() or "utf-8"
-            return payload.decode(charset, errors="replace")
+            text = payload.decode(charset, errors="replace")
+            if msg.get_content_type() == "text/html":
+                text = re.sub(r"<br\s*/?>", "\n", text, flags=re.I)
+                text = re.sub(r"</(?:p|div|li|tr|h[1-6])\s*>", "\n", text, flags=re.I)
+                text = re.sub(r"<[^>]+>", "", text)
+                text = html.unescape(text)
+                text = re.sub(r"[ \t]+\n", "\n", text)
+                text = re.sub(r"\n{3,}", "\n\n", text)
+            return text.strip()
     return ""
 
 
@@ -1998,6 +2012,9 @@ class SendEmailRequest(BaseModel):
     # answered after successful delivery so it leaves undone/reply-soon views.
     source_uid: Optional[str] = None
     source_folder: Optional[str] = None
+    # Exact IMAP draft to remove after successful delivery.
+    draft_uid: Optional[str] = None
+    draft_folder: Optional[str] = None
     # Internal marker for Odysseus-generated mail (e.g. reminder, scheduled).
     odysseus_kind: Optional[str] = None
     # If true, /send waits for SMTP + Sent append and returns the sent UID.

routes/email_pollers.py

@@ -228,6 +228,9 @@ def _ensure_away_reply_table():
 
 
 def _sender_is_automated(msg, sender_addr: str) -> bool:
+    subject = str(msg.get("Subject") or "").lower()
+    if re.search(r"automatic\s+reply|auto(?:matic)?[- ]?reply|out\s+of\s+office|\booo\b|r[ée]ponse\s+automatique", subject):
+        return True
     auto_submitted = (msg.get("Auto-Submitted") or "").strip().lower()
     if auto_submitted and auto_submitted != "no":
         return True
@@ -244,6 +247,31 @@ def _sender_is_automated(msg, sender_addr: str) -> bool:
     }
 
 
+def _remove_urgent_tag_from_cache(message_id: str, owner: str, account_id: str) -> None:
+    """Remove stale urgent tags from messages identified as automated."""
+    import sqlite3 as _sql3
+    conn = _sql3.connect(SCHEDULED_DB)
+    try:
+        owner_clause, owner_params = _email_cache_owner_clause(owner)
+        rows = conn.execute(
+            f"SELECT rowid, tags FROM email_tags WHERE message_id=? AND {owner_clause} "
+            "AND (account_id=? OR account_id='' OR account_id IS NULL)",
+            (message_id, *owner_params, account_id or ""),
+        ).fetchall()
+        for rowid, raw_tags in rows:
+            try:
+                tags = json.loads(raw_tags or "[]")
+            except Exception:
+                tags = []
+            if not isinstance(tags, list) or "urgent" not in tags:
+                continue
+            cleaned = [tag for tag in tags if str(tag).strip().lower() != "urgent"]
+            conn.execute("UPDATE email_tags SET tags=? WHERE rowid=?", (json.dumps(cleaned), rowid))
+        conn.commit()
+    finally:
+        conn.close()
+
+
 def _away_reply_already_sent(settings: dict, account_owner: str, account_id: str | None,
                              message_id: str, sender_addr: str) -> bool:
     import sqlite3 as _sql3
@@ -712,6 +740,9 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                     _, _from_addr_only = email.utils.parseaddr(_from_raw)
                 except Exception:
                     _from_addr_only = ""
+                _is_automated = _sender_is_automated(msg, _from_addr_only)
+                if _is_automated and auto_tag:
+                    _remove_urgent_tag_from_cache(message_id, account_owner or "", account_id or "")
                 _is_self_mail = bool(_self_self_addr) and _from_addr_only.lower() == _self_self_addr
                 need_sum = auto_sum and message_id not in _sum_existing
                 need_reply = auto_reply_draft and message_id not in _reply_existing
@@ -1286,6 +1317,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
                             tags = ["marketing" if t == "promo" else t for t in tags]
                             tags = [t for t in tags if t in _ALLOWED_TAGS][:3]
+                            if _is_automated:
+                                tags = [t for t in tags if t != "urgent"]
                             is_spam = bool(parsed.get("spam"))
                             spam_reason = str(parsed.get("reason") or "")[:200]
 

routes/history/history_routes.py

@@ -1,25 +1,26 @@
 """History routes — session history, truncation, fork, conversation topics."""
 
 import json
+import os
 import uuid
 import logging
 import re
 from typing import Dict, Any, Optional
 
-from fastapi import APIRouter, Request, HTTPException, Depends
+from fastapi import APIRouter, Request, HTTPException
 
 from core.models import ChatMessage
 from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession
-from src.auth_helpers import effective_user, require_chat_api_token_scope
+from src.auth_helpers import effective_user
 from src.topic_analyzer import analyze_topics
 from src.upload_handler import reserve_message_upload_references
-from src.tool_approval_scopes import sanitize_client_message_metadata
 from routes.session_routes import (
     _message_role,
     _message_text,
     _reject_compact_during_active_run,
     _verify_session_owner,
 )
+from routes.chat_helpers import strip_tui_local_context
 
 logger = logging.getLogger(__name__)
 
@@ -27,6 +28,105 @@ _HISTORY_INLINE_MEDIA_THRESHOLD = 200_000
 _DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+")
 
 
+def _sft_trace_file_for_owner(owner: str | None) -> str | None:
+    if not str(owner or "").startswith("sft_"):
+        return None
+    flag = os.getenv("ODYSSEUS_SFT_TRACE_CAPTURE", "1").strip().lower()
+    if flag in {"0", "false", "no", "off"}:
+        return None
+    try:
+        from src.constants import DATA_DIR
+        trace_dir = os.getenv("ODYSSEUS_SFT_TRACE_DIR") or os.path.join(DATA_DIR, "sft_traces")
+        return os.path.join(trace_dir, f"{owner}.jsonl")
+    except Exception:
+        return None
+
+
+def _remove_deleted_sft_trace_rows(
+    *,
+    owner: str | None,
+    session_id: str,
+    deleted_pairs: list[dict[str, str]],
+) -> None:
+    """Keep the training JSONL aligned with user-deleted chat attempts."""
+    path = _sft_trace_file_for_owner(owner)
+    if not path or not deleted_pairs or not os.path.exists(path):
+        return
+    try:
+        kept: list[str] = []
+        removed: list[str] = []
+        with open(path, "r", encoding="utf-8") as f:
+            for line in f:
+                raw = line.rstrip("\n")
+                if not raw.strip():
+                    continue
+                try:
+                    row = json.loads(raw)
+                except json.JSONDecodeError:
+                    kept.append(raw)
+                    continue
+                if row.get("session_id") != session_id:
+                    kept.append(raw)
+                    continue
+                row_user = str(row.get("user") or "").strip()
+                row_assistant = str(row.get("assistant") or "").strip()
+                should_remove = any(
+                    row_user == pair.get("user", "").strip()
+                    and row_assistant == pair.get("assistant", "").strip()
+                    for pair in deleted_pairs
+                )
+                if should_remove:
+                    tombstone = dict(row)
+                    tombstone["deleted_from_training"] = True
+                    removed.append(json.dumps(tombstone, ensure_ascii=False))
+                else:
+                    kept.append(raw)
+        with open(path, "w", encoding="utf-8") as f:
+            for raw in kept:
+                f.write(raw + "\n")
+        if removed:
+            trash_path = path + ".trash"
+            with open(trash_path, "a", encoding="utf-8") as f:
+                for raw in removed:
+                    f.write(raw + "\n")
+            logger.info(
+                "Removed %d SFT trace row(s) for deleted messages in session %s",
+                len(removed),
+                session_id,
+            )
+    except Exception as exc:
+        logger.warning("Failed to prune SFT trace rows for %s: %s", session_id, exc)
+
+
+def _deleted_sft_pairs_from_db_rows(rows: list[DbChatMessage]) -> list[dict[str, str]]:
+    """Build user/assistant pairs affected by deleted messages.
+
+    The SFT trace row is one assistant turn paired with the nearest preceding
+    user turn. If the user deletes either side of a failed attempt before
+    retrying, remove that pair from the training JSONL.
+    """
+    pairs: list[dict[str, str]] = []
+    last_user = ""
+    pending_deleted_user = ""
+    for row in rows:
+        role = str(getattr(row, "role", "") or "")
+        content = str(getattr(row, "content", "") or "").strip()
+        will_delete = bool(getattr(row, "_will_delete_for_sft", False))
+        if role == "user":
+            last_user = content
+            if will_delete:
+                pending_deleted_user = content
+            continue
+        if role != "assistant":
+            continue
+        if will_delete and last_user:
+            pairs.append({"user": last_user, "assistant": content})
+        elif pending_deleted_user:
+            pairs.append({"user": pending_deleted_user, "assistant": content})
+            pending_deleted_user = ""
+    return pairs
+
+
 def _history_display_content(content: Any) -> Any:
     """Return a lightweight browser-display copy of stored message content.
 
@@ -101,12 +201,44 @@ def _merge_continue_rows_to_delete(db_messages, db1, db2):
     return to_delete
 
 
-def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
-    router = APIRouter(
-        tags=["history"],
-        dependencies=[Depends(require_chat_api_token_scope)],
+def _is_continue_interruption_message(message: Any) -> bool:
+    if isinstance(message, ChatMessage):
+        role = message.role
+        content = message.content
+    elif isinstance(message, dict):
+        role = message.get("role", "")
+        content = message.get("content", "")
+    else:
+        role = getattr(message, "role", "")
+        content = getattr(message, "content", "")
+    normalized = " ".join(str(content or "").strip().lower().split())
+    return role == "user" and (
+        "previous response was interrupted" in normalized
+        or normalized in {
+            "continue from where you left off.",
+            "continue from where you left off",
+        }
     )
 
+
+def _has_immediate_continue_marker(messages: list[Any], idx1: int, idx2: int) -> bool:
+    return idx2 - idx1 == 2 and _is_continue_interruption_message(messages[idx1 + 1])
+
+
+def _keep_count_before_message(db_messages, before_msg_id: str | None) -> int | None:
+    """Return the durable-history keep count before a DB message id."""
+    wanted = str(before_msg_id or "").strip()
+    if not wanted:
+        return None
+    for pos, row in enumerate(db_messages):
+        if str(getattr(row, "id", "")) == wanted:
+            return pos
+    return None
+
+
+def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
+    router = APIRouter(tags=["history"])
+
     def _reserve_message_uploads(
         request: Request,
         content: Any,
@@ -128,13 +260,14 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             )
 
     def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]:
-        entry = {"role": m.role, "content": _history_display_content(m.content)}
+        entry = {"role": m.role, "content": strip_tui_local_context(_history_display_content(m.content))}
         meta = {}
         if m.meta_data:
             try:
                 meta = json.loads(m.meta_data) or {}
             except (json.JSONDecodeError, ValueError):
                 meta = {}
+        meta["_db_id"] = m.id
         if m.timestamp and "timestamp" not in meta:
             meta["timestamp"] = m.timestamp.isoformat() + "Z"
         if meta:
@@ -203,7 +336,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 # Skip hidden messages (e.g. compaction summaries for AI context)
                 if msg.metadata and msg.metadata.get("hidden"):
                     continue
-                entry = {"role": msg.role, "content": _history_display_content(msg.content)}
+                entry = {"role": msg.role, "content": strip_tui_local_context(_history_display_content(msg.content))}
                 if msg.metadata:
                     entry["metadata"] = msg.metadata
                 history_dict.append(entry)
@@ -212,7 +345,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                     continue
                 entry = {
                     "role": msg.get("role", ""),
-                    "content": _history_display_content(msg.get("content", "")),
+                    "content": strip_tui_local_context(_history_display_content(msg.get("content", ""))),
                 }
                 if msg.get("metadata"):
                     entry["metadata"] = msg["metadata"]
@@ -253,11 +386,36 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
         _verify_session_owner(request, session_id)
         try:
             body = await request.json()
-            keep_count = body.get("keep_count", 0)
+            keep_count = int(body.get("keep_count", 0))
+            before_msg_id = str(body.get("before_msg_id") or body.get("message_id") or "").strip()
+            deleted_sft_pairs: list[dict[str, str]] = []
+            if keep_count >= 0:
+                db = SessionLocal()
+                try:
+                    all_db_messages = db.query(DbChatMessage).filter(
+                        DbChatMessage.session_id == session_id
+                    ).order_by(DbChatMessage.timestamp).all()
+                    if before_msg_id:
+                        resolved_keep_count = _keep_count_before_message(all_db_messages, before_msg_id)
+                        if resolved_keep_count is None:
+                            raise HTTPException(404, "Message not found")
+                        keep_count = resolved_keep_count
+                    for pos, row in enumerate(all_db_messages):
+                        row._will_delete_for_sft = pos >= keep_count
+                    deleted_sft_pairs = _deleted_sft_pairs_from_db_rows(all_db_messages)
+                finally:
+                    db.close()
             result = session_manager.truncate_messages(session_id, keep_count)
+            _remove_deleted_sft_trace_rows(
+                owner=effective_user(request),
+                session_id=session_id,
+                deleted_pairs=deleted_sft_pairs,
+            )
             return {"status": "ok", "kept": keep_count, "truncated": result}
         except KeyError:
             raise HTTPException(404, "Session not found")
+        except HTTPException:
+            raise
         except Exception as e:
             logger.error(f"Truncate error {session_id}: {e}")
             raise HTTPException(500, str(e))
@@ -272,7 +430,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             content = body.get("content", "")
             if not content:
                 raise HTTPException(400, "content is required")
-            metadata = sanitize_client_message_metadata(body.get("metadata"))
+            metadata = body.get("metadata")
             _reserve_message_uploads(request, content, metadata)
             msg = ChatMessage(role=role, content=content, metadata=metadata)
             session_manager.add_message(session_id, msg)
@@ -292,6 +450,18 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             session = session_manager.get_session(session_id)
             db = SessionLocal()
             try:
+                all_db_messages = db.query(DbChatMessage).filter(
+                    DbChatMessage.session_id == session_id
+                ).order_by(DbChatMessage.timestamp).all()
+                delete_id_set = set(msg_ids or [])
+                delete_index_set = set(indices or [])
+                for pos, row in enumerate(all_db_messages):
+                    row._will_delete_for_sft = (
+                        (bool(delete_id_set) and row.id in delete_id_set)
+                        or (not delete_id_set and bool(delete_index_set) and pos in delete_index_set)
+                    )
+                deleted_sft_pairs = _deleted_sft_pairs_from_db_rows(all_db_messages)
+
                 if msg_ids:
                     # New ID-based delete
                     deleted = 0
@@ -334,6 +504,11 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                     db_session.updated_at = datetime.now(timezone.utc)
 
                 db.commit()
+                _remove_deleted_sft_trace_rows(
+                    owner=effective_user(request),
+                    session_id=session_id,
+                    deleted_pairs=deleted_sft_pairs,
+                )
                 return {"status": "ok", "deleted": deleted}
             finally:
                 db.close()
@@ -524,6 +699,9 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 return {"status": "ok", "merged": False}
 
             idx1, idx2 = ai_indices[-2], ai_indices[-1]
+            if not _has_immediate_continue_marker(session.history, idx1, idx2):
+                return {"status": "ok", "merged": False, "reason": "no_continue_marker"}
+
             msg1, msg2 = session.history[idx1], session.history[idx2]
 
             content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '')
@@ -534,7 +712,14 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {}
             meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {}
             merged_meta = {**meta1, **meta2}
+            thinking1 = str(meta1.get('thinking') or '').strip()
+            thinking2 = str(meta2.get('thinking') or '').strip()
+            if thinking1 and thinking2:
+                merged_meta['thinking'] = thinking1 + "\n\n(continued)\n\n" + thinking2
+            elif thinking1:
+                merged_meta['thinking'] = thinking1
             merged_meta.pop('stopped', None)  # no longer stopped after continue
+            merged_meta.pop('thinking_interrupted', None)
 
             # Update first message, remove second
             if isinstance(msg1, ChatMessage):
@@ -546,13 +731,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
 
             # Also remove the hidden "continue" user message between them if present
             # It's the message at idx2-1 if it's a user message with continue text
-            remove_indices = [idx2]
-            if idx2 - 1 > idx1:
-                between = session.history[idx2 - 1]
-                between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '')
-                between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '')
-                if between_role == 'user' and 'previous response was interrupted' in between_content:
-                    remove_indices.insert(0, idx2 - 1)
+            remove_indices = [idx2, idx1 + 1]
 
             for ri in sorted(remove_indices, reverse=True):
                 session.history.pop(ri)
@@ -570,19 +749,20 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 # Find last two assistant messages in DB
                 ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant']
                 if len(ai_db) >= 2:
-                    (_, db1), (_, db2) = ai_db[-2], ai_db[-1]
-                    db1.content = merged_content
-                    db1.meta_data = _json.dumps(merged_meta)
+                    (db_idx1, db1), (db_idx2, db2) = ai_db[-2], ai_db[-1]
+                    if _has_immediate_continue_marker(db_messages, db_idx1, db_idx2):
+                        db1.content = merged_content
+                        db1.meta_data = _json.dumps(merged_meta)
 
-                    # Mirror the in-memory deletion: remove the second assistant
-                    # message and ONLY the "continue" user message between them
-                    # (not arbitrary tool/system/user rows). The old
-                    # range-delete destroyed every row between the two assistant
-                    # messages, desyncing the DB from the in-memory history.
-                    for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
-                        db.delete(_row)
+                        # Mirror the in-memory deletion: remove the second assistant
+                        # message and ONLY the "continue" user message between them
+                        # (not arbitrary tool/system/user rows). The old
+                        # range-delete destroyed every row between the two assistant
+                        # messages, desyncing the DB from the in-memory history.
+                        for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
+                            db.delete(_row)
 
-                    db.commit()
+                        db.commit()
             finally:
                 db.close()
             session_manager.save_sessions()
@@ -676,6 +856,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             raise HTTPException(404, "Session not found")
 
         try:
+            from src.context_compactor import auto_compact_threshold_percent
             from src.model_context import estimate_tokens, get_context_length
 
             messages = session.get_context_messages()
@@ -683,6 +864,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             ctx_len = int(get_context_length(session.endpoint_url, session.model) or 0)
             pct = round((used / ctx_len) * 100, 1) if ctx_len else 0.0
             pct = max(0.0, min(100.0, pct))
+            auto_threshold = auto_compact_threshold_percent()
             visible_messages = sum(
                 1 for m in session.history
                 if not (getattr(m, "metadata", None) or {}).get("hidden")
@@ -703,13 +885,119 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 "context_messages": len(messages),
                 "compacted_messages": compacted_messages,
                 "can_compact": can_compact,
-                "should_compact": pct >= 70,
-                "auto_compact_threshold": 85,
+                "should_compact": pct >= auto_threshold,
+                "auto_compact_threshold": auto_threshold,
+                "memory_extraction_enabled": getattr(session, "memory_extraction_enabled", True) is not False,
+                "skill_injection_enabled": getattr(session, "skill_injection_enabled", True) is not False,
+                "thinking_mode": getattr(session, "thinking_mode", "") or "off",
+                "temperature_override": getattr(session, "temperature_override", None),
+                "max_tokens_override": getattr(session, "max_tokens_override", None),
             }
         except Exception as e:
             logger.error(f"Context usage error {session_id}: {e}")
             raise HTTPException(500, str(e))
 
+    @router.post("/api/session/{session_id}/memory-extraction")
+    async def set_session_memory_extraction(request: Request, session_id: str) -> Dict[str, Any]:
+        """Toggle automatic memory extraction for one chat session."""
+        _verify_session_owner(request, session_id)
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+
+        try:
+            body = await request.json()
+        except Exception:
+            body = {}
+        if "enabled" not in body:
+            raise HTTPException(400, "Missing enabled")
+        enabled = bool(body.get("enabled"))
+
+        db = SessionLocal()
+        try:
+            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if not db_session:
+                raise HTTPException(404, "Session not found")
+            db_session.memory_extraction_enabled = enabled
+            db.commit()
+            session.memory_extraction_enabled = enabled
+            return {"status": "success", "memory_extraction_enabled": enabled}
+        except HTTPException:
+            raise
+        except Exception as e:
+            db.rollback()
+            logger.error(f"Memory extraction toggle error {session_id}: {e}")
+            raise HTTPException(500, "Failed to update memory extraction")
+        finally:
+            db.close()
+
+    @router.post("/api/session/{session_id}/skill-injection")
+    async def set_session_skill_injection(request: Request, session_id: str) -> Dict[str, Any]:
+        """Toggle skill injection for one chat session."""
+        _verify_session_owner(request, session_id, session_manager)
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+
+        try:
+            body = await request.json()
+        except Exception:
+            body = {}
+        if "enabled" not in body:
+            raise HTTPException(400, "Missing enabled")
+        enabled = bool(body.get("enabled"))
+
+        db = SessionLocal()
+        try:
+            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if not db_session:
+                # Some active chats exist only in the in-memory manager until
+                # their first persisted write. Keep the toggle usable there.
+                session.skill_injection_enabled = enabled
+                session_manager.save_sessions()
+                return {"status": "success", "skill_injection_enabled": enabled}
+            db_session.skill_injection_enabled = enabled
+            db.commit()
+            session.skill_injection_enabled = enabled
+            return {"status": "success", "skill_injection_enabled": enabled}
+        except HTTPException:
+            raise
+        except Exception as e:
+            db.rollback()
+            logger.error(f"Skill injection toggle error {session_id}: {e}")
+            raise HTTPException(500, "Failed to update skill injection")
+        finally:
+            db.close()
+
+    @router.post("/api/session/{session_id}/generation-settings")
+    async def set_session_generation_settings(request: Request, session_id: str) -> Dict[str, Any]:
+        _verify_session_owner(request, session_id, session_manager)
+        try:
+            session = session_manager.get_session(session_id)
+            body = await request.json()
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        mode = str(body.get("thinking_mode") or "").lower()
+        if mode not in {"", "on", "off"}:
+            raise HTTPException(400, "Invalid thinking mode")
+        temperature = body.get("temperature_override")
+        temperature = None if temperature in (None, "") else max(0.0, min(2.0, float(temperature)))
+        max_tokens = body.get("max_tokens_override")
+        max_tokens = None if max_tokens in (None, "", 0) else max(256, min(32768, int(max_tokens)))
+        db = SessionLocal()
+        try:
+            row = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if not row:
+                raise HTTPException(404, "Session not found")
+            row.thinking_mode, row.temperature_override, row.max_tokens_override = mode, temperature, max_tokens
+            db.commit()
+            session.thinking_mode, session.temperature_override, session.max_tokens_override = mode, temperature, max_tokens
+            return {"status": "success", "thinking_mode": mode, "temperature_override": temperature, "max_tokens_override": max_tokens}
+        finally:
+            db.close()
+
     @router.post("/api/session/{session_id}/compact")
     async def compact_session(request: Request, session_id: str):
         """Manually trigger context compaction for a session."""

routes/hwfit_routes.py

@@ -16,6 +16,24 @@ from routes._validators import validate_remote_host, validate_ssh_port
 # "metal" routes through the Apple-Silicon path (GGUF-only, llama.cpp/Ollama),
 # the CPU backends through the RAM/offload path, cuda/rocm through vLLM.
 _MANUAL_BACKENDS = {"cuda", "rocm", "metal", "cpu_x86", "cpu_arm"}
+_OFFICIAL_NAMESPACES = {
+    "apple", "allenai", "black-forest-labs", "cohere", "deepseek-ai",
+    "google", "ibm", "lightricks", "meta-llama", "microsoft", "mistralai",
+    "nvidia", "openai", "qwen", "stabilityai", "tencent", "tiiuae",
+    "upstage", "zai-org", "runwayml",
+}
+
+
+def _is_official_model(model: dict) -> bool:
+    """Recognize first-party namespaces without maintaining model-name lists."""
+    # Image rows expose a friendly `name` without its namespace, while regular
+    # rows may use `name`. Prefer whichever field still contains `owner/repo`.
+    model_id = str(model.get("id") or model.get("name") or "")
+    namespace = model_id.split("/", 1)[0].strip().lower() if "/" in model_id else ""
+    # `provider` is a display label for image rows (for example, "Stability AI")
+    # and is not a stable repository namespace. The model id is the canonical
+    # source for this filter, so a recognized namespace is sufficient.
+    return namespace in _OFFICIAL_NAMESPACES
 
 
 def _validate_detection_target(host: str = "", ssh_port: str = "") -> tuple[str, str]:
@@ -191,7 +209,7 @@ def setup_hwfit_routes():
         return detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh)
 
     @router.get("/models")
-    def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, refresh_catalog: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
+    def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, refresh_catalog: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False, official_only: bool = False):
         """Rank LLM models against detected hardware and return scored results.
         gpu_count: override GPU count (0 = CPU only, 1-N = simulate N GPUs of the
             active group). gpu_group: index into system.gpu_groups (the homogeneous
@@ -310,6 +328,8 @@ def setup_hwfit_routes():
             rank_kwargs.pop("target_context", None)
             rank_kwargs.pop("fit_only", None)
         results = rank_models(system, **rank_kwargs)
+        if official_only:
+            results = [m for m in results if _is_official_model(m)]
         payload = {"system": system, "models": results}
         if catalog_refresh is not None:
             payload["catalog_refresh"] = catalog_refresh
@@ -410,7 +430,7 @@ def setup_hwfit_routes():
         }
 
     @router.get("/image-models")
-    def get_image_models(sort: str = "fit", search: str = "", host: str = "", gpu_count: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False):
+    def get_image_models(sort: str = "fit", search: str = "", host: str = "", gpu_count: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, official_only: bool = False):
         """Rank image generation models against detected hardware."""
         from services.hwfit.hardware import detect_system
         from services.hwfit.image_models import rank_image_models
@@ -451,6 +471,8 @@ def setup_hwfit_routes():
             system["gpu_count"] = 1 if single_vram > 0 else 0
             system["gpu_only"] = True if single_vram > 0 else False
         results = rank_image_models(system, search=search or None, sort=sort)
+        if official_only:
+            results = [m for m in results if _is_official_model(m)]
         return {"system": system, "models": results}
 
     return router

routes/model_routes.py

@@ -17,7 +17,20 @@ from fastapi import APIRouter, HTTPException, Form, Query, Body, Request, Respon
 from pydantic import BaseModel
 from fastapi.responses import StreamingResponse
 from core.database import SessionLocal, ModelEndpoint, Session as DbSession
-from core.log_safety import redact_url as _redact_url_for_log
+try:
+    from core.log_safety import redact_url as _redact_url_for_log
+except ModuleNotFoundError:
+    def _redact_url_for_log(url: str) -> str:
+        try:
+            parsed = urlparse(url or "")
+            host = parsed.hostname or ""
+            if ":" in host:
+                host = f"[{host}]"
+            if parsed.port:
+                host = f"{host}:{parsed.port}"
+            return urlunparse((parsed.scheme, host, parsed.path, "", "", ""))
+        except Exception:
+            return "<endpoint>"
 from core.middleware import require_admin
 from src.constants import COOKBOOK_STATE_FILE
 from src.llm_core import _detect_provider, _host_match, ANTHROPIC_MODELS
@@ -455,6 +468,7 @@ def _truthy(value: str | None) -> bool:
 
 _ENDPOINT_KINDS = {"auto", "local", "api", "proxy"}
 _REFRESH_MODES = {"auto", "manual", "disabled"}
+_MODEL_TOOL_MODES = {"none", "compact", "full"}
 
 
 def _normalize_endpoint_kind(value: Any) -> str:
@@ -462,6 +476,30 @@ def _normalize_endpoint_kind(value: Any) -> str:
     return kind if kind in _ENDPOINT_KINDS else "auto"
 
 
+def _normalize_model_tool_mode(value: Any) -> str:
+    mode = str(value or "").strip().lower()
+    return mode if mode in _MODEL_TOOL_MODES else ""
+
+
+def _model_tool_modes(ep: Any) -> Dict[str, str]:
+    raw = getattr(ep, "model_tool_modes", None)
+    if not raw:
+        return {}
+    try:
+        data = json.loads(raw) if isinstance(raw, str) else raw
+    except Exception:
+        return {}
+    if not isinstance(data, dict):
+        return {}
+    modes: Dict[str, str] = {}
+    for key, value in data.items():
+        model_id = str(key or "").strip()
+        mode = _normalize_model_tool_mode(value)
+        if model_id and mode:
+            modes[model_id] = mode
+    return modes
+
+
 def _normalize_refresh_mode(value: Any, endpoint_kind: str = "auto") -> str:
     mode = str(value or "").strip().lower()
     kind = _normalize_endpoint_kind(endpoint_kind)
@@ -1973,6 +2011,7 @@ def setup_model_routes(model_discovery):
                     "ping_error": (ping or {}).get("error") if ping else None,
                     "model_type": getattr(r, "model_type", None) or "llm",
                     "supports_tools": getattr(r, "supports_tools", None),
+                    "model_tool_modes": _model_tool_modes(r),
                     "endpoint_kind": kind,
                     "category": _classify_endpoint(base, kind),
                     "model_refresh_mode": _endpoint_refresh_mode(r, kind),
@@ -2344,6 +2383,7 @@ def setup_model_routes(model_discovery):
                     response.headers["X-Model-Refresh-Warning"] = "Model refresh failed or returned no models; kept cached models."
             _, pinned = _picker_models_for_endpoint(ep, base, kind)
             pinned_set = set(pinned)
+            tool_modes = _model_tool_modes(ep)
             return [
                 {
                     "id": m,
@@ -2351,6 +2391,7 @@ def setup_model_routes(model_discovery):
                     "is_hidden": m in hidden,
                     "is_pinned": m in pinned_set,
                     "picker_requires_pinning": picker_requires_pinning,
+                    "tool_mode": tool_modes.get(m, ""),
                 }
                 for m in _merge_model_ids(all_models, pinned)
             ]
@@ -2401,11 +2442,31 @@ def setup_model_routes(model_discovery):
                     ep.hidden_models = None
                 else:
                     ep.pinned_models = json.dumps(pinned) if pinned else None
+            if "model_tool_modes" in body:
+                raw_modes = body.get("model_tool_modes")
+                if not isinstance(raw_modes, dict):
+                    raise HTTPException(400, "model_tool_modes must be an object")
+                modes = _model_tool_modes(ep)
+                for model_id, mode in raw_modes.items():
+                    model_id = str(model_id or "").strip()
+                    if not model_id:
+                        continue
+                    normalized = _normalize_model_tool_mode(mode)
+                    if normalized:
+                        modes[model_id] = normalized
+                    else:
+                        modes.pop(model_id, None)
+                ep.model_tool_modes = json.dumps(modes) if modes else None
             db.commit()
             _invalidate_models_cache()
             hidden_count = len(json.loads(ep.hidden_models)) if ep.hidden_models else 0
             pinned_count = len(json.loads(ep.pinned_models)) if ep.pinned_models else 0
-            return {"id": ep_id, "hidden_count": hidden_count, "pinned_count": pinned_count}
+            return {
+                "id": ep_id,
+                "hidden_count": hidden_count,
+                "pinned_count": pinned_count,
+                "model_tool_modes": _model_tool_modes(ep),
+            }
         finally:
             db.close()
 
@@ -2572,6 +2633,7 @@ def setup_model_routes(model_discovery):
                 "model_type": ep.model_type,
                 "base_url": ep.base_url,
                 "pinned_models": _normalize_model_ids(getattr(ep, "pinned_models", None)),
+                "model_tool_modes": _model_tool_modes(ep),
                 "endpoint_kind": getattr(ep, "endpoint_kind", None) or "auto",
                 "model_refresh_mode": getattr(ep, "model_refresh_mode", None) or "auto",
                 "model_refresh_interval": getattr(ep, "model_refresh_interval", None),

routes/note/note_routes.py

@@ -35,6 +35,7 @@ class NoteCreate(BaseModel):
     source: str = "user"
     session_id: Optional[str] = None
     image_url: Optional[str] = None
+    gallery_id: Optional[str] = None
     repeat: Optional[str] = "none"
     sort_order: Optional[int] = None
 
@@ -50,6 +51,7 @@ class NoteUpdate(BaseModel):
     archived: Optional[bool] = None
     due_date: Optional[str] = None
     image_url: Optional[str] = None
+    gallery_id: Optional[str] = None
     repeat: Optional[str] = None
     sort_order: Optional[int] = None
     agent_session_id: Optional[str] = None
@@ -89,6 +91,7 @@ def _note_to_dict(note: Note) -> Dict[str, Any]:
         "session_id": note.session_id,
         "sort_order": note.sort_order or 0,
         "image_url": note.image_url,
+        "gallery_id": getattr(note, "gallery_id", None),
         "repeat": note.repeat or "none",
         "ai_classification": ai_cls,
         "ai_content_hash": getattr(note, "ai_content_hash", None),
@@ -674,6 +677,7 @@ def setup_note_routes(task_scheduler=None, upload_handler=None):
                 source=body.source,
                 session_id=body.session_id,
                 image_url=body.image_url,
+                gallery_id=body.gallery_id,
                 repeat=body.repeat or "none",
                 sort_order=body.sort_order if body.sort_order is not None else 0,
             )
@@ -743,6 +747,8 @@ def setup_note_routes(task_scheduler=None, upload_handler=None):
                 note.due_date = body.due_date
             if body.image_url is not None:
                 note.image_url = body.image_url
+            if body.gallery_id is not None:
+                note.gallery_id = body.gallery_id
             if body.repeat is not None:
                 note.repeat = body.repeat
             if body.sort_order is not None:

routes/preset_routes.py

@@ -21,6 +21,8 @@ class UserTemplateRequest(BaseModel):
     system_prompt: str = Field("", max_length=10000)
     temperature: float = Field(1.0, ge=0.0, le=2.0)
     max_tokens: int = Field(0, ge=0, le=65536)
+    persona_memory: str = Field("", max_length=6000)
+    persona_memory_schema: str = Field("general", pattern="^(general|health)$")
 
 
 def setup_preset_routes(preset_manager) -> APIRouter:
@@ -41,6 +43,10 @@ def setup_preset_routes(preset_manager) -> APIRouter:
                 preset_update.enabled,
                 preset_update.inject_prefix,
                 preset_update.inject_suffix,
+                preset_update.persona_memory,
+                preset_update.persona_memory_schema,
+                preset_update.thinking_mode,
+                preset_update.show_persona_name,
             )
             if success:
                 return {"success": True, "message": "Custom preset updated"}

routes/research/research_routes.py

@@ -7,7 +7,7 @@ import re
 import uuid
 from datetime import datetime
 from pathlib import Path
-from typing import Optional
+from typing import Literal, Optional
 
 from fastapi import APIRouter, HTTPException, Query, Request
 from fastapi.responses import HTMLResponse, StreamingResponse
@@ -271,7 +271,13 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                     "query": entry.get("query", ""),
                     "status": "running",
                     "progress": entry.get("progress", {}),
+                    "source_state": research_handler.get_source_state(sid),
+                    "source_coverage": research_handler.get_source_coverage(sid),
+                    "navigation_trace": research_handler.get_navigation_trace(sid),
+                    "action_trace": research_handler.get_action_trace(sid),
                     "started_at": entry.get("started_at", 0),
+                    "category": research_handler.get_category(sid),
+                    "mode": research_handler.get_mode(sid),
                 })
         return {"active": active}
 
@@ -284,6 +290,24 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         status = research_handler.get_status(session_id)
         if status is None:
             raise HTTPException(404, "No research found for this session")
+        try:
+            source_state = research_handler.get_source_state(session_id)
+            if isinstance(source_state, str) and source_state:
+                status["source_state"] = source_state
+            source_coverage = research_handler.get_source_coverage(session_id)
+            if isinstance(source_coverage, dict) and source_coverage:
+                status["source_coverage"] = source_coverage
+        except Exception:
+            pass
+        try:
+            navigation_trace = research_handler.get_navigation_trace(session_id)
+            if isinstance(navigation_trace, list) and navigation_trace:
+                status["navigation_trace"] = navigation_trace
+            action_trace = research_handler.get_action_trace(session_id)
+            if isinstance(action_trace, list) and action_trace:
+                status["action_trace"] = action_trace
+        except Exception:
+            pass
         return status
 
     @router.post("/api/research/cancel/{session_id}")
@@ -306,8 +330,26 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
             raise HTTPException(404, "No research result available")
         sources = research_handler.get_sources(session_id) or []
         raw_findings = research_handler.get_raw_findings(session_id) or []
+        analyzed_urls = research_handler.get_analyzed_urls(session_id) or []
+        source_state = research_handler.get_source_state(session_id)
+        source_coverage = research_handler.get_source_coverage(session_id)
+        navigation_trace = research_handler.get_navigation_trace(session_id)
+        action_trace = research_handler.get_action_trace(session_id)
+        category = research_handler.get_category(session_id)
+        mode = research_handler.get_mode(session_id)
         research_handler.clear_result(session_id)
-        return {"result": result, "sources": sources, "raw_findings": raw_findings}
+        return {
+            "result": result,
+            "sources": sources,
+            "raw_findings": raw_findings,
+            "analyzed_urls": analyzed_urls,
+            "source_state": source_state,
+            "source_coverage": source_coverage,
+            "navigation_trace": navigation_trace,
+            "action_trace": action_trace,
+            "category": category,
+            "mode": mode,
+        }
 
     def _assert_owns_research(session_id: str, user: str) -> None:
         """404-not-403 ownership gate for a research session's on-disk JSON.
@@ -394,6 +436,8 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                     "id": p.stem,
                     "query": query,
                     "category": d.get("category") or "",
+                    "mode": d.get("mode") or "research",
+                    "mode": d.get("mode") or "research",
                     "source_count": len(sources),
                     "status": d.get("status", "done"),
                     "duration": d.get("stats", {}).get("Duration", ""),
@@ -479,6 +523,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
 
     class ResearchStartRequest(BaseModel):
         query: str
+        origin_chat_id: Optional[str] = None
         # max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
         max_rounds: int = Field(default=0, ge=0, le=20)
         search_provider: Optional[str] = None
@@ -487,7 +532,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         max_time: int = Field(default=300, ge=60, le=1800)
         extraction_timeout: Optional[int] = Field(default=None, ge=15, le=3600)
         extraction_concurrency: Optional[int] = Field(default=None, ge=1, le=12)
-        category: Optional[str] = None
+        category: Optional[Literal["product", "comparison", "howto", "factcheck"]] = None
 
     @router.post("/api/research/start")
     async def research_start(body: ResearchStartRequest, request: Request):
@@ -509,6 +554,15 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                         pass
                 user = tool_owner
         session_id = f"rp-{uuid.uuid4().hex[:12]}"
+        delivery = getattr(request.app.state, 'background_tool_jobs', None)
+        if body.origin_chat_id:
+            from core.database import SessionLocal, Session as DbSession
+            with SessionLocal() as db:
+                origin = db.get(DbSession, body.origin_chat_id)
+                if origin is None or origin.owner != user:
+                    raise HTTPException(404, 'Origin chat not found')
+            if delivery is None:
+                raise HTTPException(503, 'Background chat delivery is unavailable')
 
         if body.endpoint_id:
             from src.database import SessionLocal
@@ -558,8 +612,12 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
             if body.model:
                 ep_model = body.model
 
-        # max_rounds=0 → "Auto", let AI decide; pass 20 as the safety cap.
-        effective_max_rounds = body.max_rounds if body.max_rounds > 0 else 20
+        # 0 = auto research capped at 20.
+        effective_max_rounds = body.max_rounds if body.max_rounds != 0 else 20
+        if body.origin_chat_id and 'max_rounds' not in body.model_fields_set:
+            effective_max_rounds = 2
+        if body.origin_chat_id:
+            delivery.register(session_id, body.origin_chat_id, user, 'research', body.query, effective_max_rounds)
         research_handler.start_research(
             session_id=session_id,
             query=body.query,
@@ -573,8 +631,22 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
             extraction_timeout=body.extraction_timeout,
             extraction_concurrency=body.extraction_concurrency,
             owner=user,
+            on_complete=(lambda sid, result, sources, findings: delivery.complete(sid, result, sources))
+                if body.origin_chat_id else None,
         )
-        return {"session_id": session_id, "status": "running", "query": body.query}
+        return {
+            "session_id": session_id,
+            "status": "running",
+            "query": body.query,
+            "category": body.category or "",
+            "mode": "research",
+        }
+
+    @router.get('/api/research/chat-jobs/{chat_id}')
+    async def chat_research_jobs(chat_id: str, request: Request):
+        user = _require_user(request)
+        delivery = getattr(request.app.state, 'background_tool_jobs', None)
+        return {'jobs': delivery.list_for_chat(chat_id, user) if delivery else []}
 
     @router.get("/api/research/stream/{session_id}")
     async def research_stream(session_id: str, request: Request):
@@ -584,7 +656,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         if not _owns_in_memory(session_id, user):
             raise HTTPException(404, "No research found for this session")
         async def _generate():
-            last_progress = None
+            last_payload = None
             while True:
                 status = research_handler.get_status(session_id)
                 if status is None:
@@ -592,9 +664,30 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                     return
                 st = status.get("status", "")
                 progress = status.get("progress", {})
-                if progress != last_progress:
-                    last_progress = progress
-                    yield f"data: {json.dumps({**progress, 'status': st})}\n\n"
+                payload = {
+                    **progress,
+                    'status': st,
+                    'category': research_handler.get_category(session_id),
+                    'mode': research_handler.get_mode(session_id),
+                }
+                try:
+                    source_state = research_handler.get_source_state(session_id)
+                    if source_state:
+                        payload["source_state"] = source_state
+                    source_coverage = research_handler.get_source_coverage(session_id)
+                    if source_coverage:
+                        payload["source_coverage"] = source_coverage
+                    navigation_trace = research_handler.get_navigation_trace(session_id)
+                    if navigation_trace:
+                        payload["navigation_trace"] = navigation_trace
+                    action_trace = research_handler.get_action_trace(session_id)
+                    if action_trace:
+                        payload["action_trace"] = action_trace
+                except Exception:
+                    pass
+                if payload != last_payload:
+                    last_payload = payload
+                    yield f"data: {json.dumps(payload)}\n\n"
                 if st != "running":
                     final = {'status': st, 'final': True}
                     task = research_handler._active_tasks.get(session_id, {})
@@ -625,12 +718,33 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                     "result": d.get("result", ""),
                     "sources": d.get("sources", []),
                     "raw_findings": d.get("raw_findings", []),
+                    "analyzed_urls": d.get("analyzed_urls", []),
+                    "source_state": d.get("source_state", ""),
+                    "source_coverage": d.get("source_coverage", {}),
+                    "navigation_trace": d.get("navigation_trace", []),
+                    "action_trace": d.get("action_trace", []),
                     "category": d.get("category") or "",
                 }
             raise HTTPException(404, "No research result available")
         sources = research_handler.get_sources(session_id) or []
         raw_findings = research_handler.get_raw_findings(session_id) or []
-        return {"result": result, "sources": sources, "raw_findings": raw_findings, "category": ""}
+        analyzed_urls = research_handler.get_analyzed_urls(session_id) or []
+        source_state = research_handler.get_source_state(session_id)
+        source_coverage = research_handler.get_source_coverage(session_id)
+        navigation_trace = research_handler.get_navigation_trace(session_id)
+        action_trace = research_handler.get_action_trace(session_id)
+        return {
+            "result": result,
+            "sources": sources,
+            "raw_findings": raw_findings,
+            "analyzed_urls": analyzed_urls,
+            "source_state": source_state,
+            "source_coverage": source_coverage,
+            "navigation_trace": navigation_trace,
+            "action_trace": action_trace,
+            "category": research_handler.get_category(session_id),
+            "mode": research_handler.get_mode(session_id),
+        }
 
     @router.post("/api/research/spinoff/{session_id}")
     async def research_spinoff(session_id: str, request: Request):

routes/search/search_routes.py

@@ -1,9 +1,12 @@
 """Search routes — /api/search/config GET, /api/search POST."""
 
+import html
+import json
 import logging
 from typing import Dict, Any
 
-from fastapi import APIRouter, Request
+from fastapi import APIRouter, Query, Request
+from fastapi.responses import HTMLResponse
 
 import time
 
@@ -39,6 +42,91 @@ async def _request_values(request: Request) -> Dict[str, Any]:
 def setup_search_routes(config) -> APIRouter:
     router = APIRouter(tags=["search"])
 
+    @router.get("/search/web", response_class=HTMLResponse)
+    async def web_search_page(q: str = Query("", min_length=0)) -> HTMLResponse:
+        """Browser-facing search results page for clickable agent web_search rows."""
+        safe_q = str(q or "").strip()
+        title = html.escape(safe_q or "Web search")
+        q_json = json.dumps(safe_q)
+        page = f"""<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <title>{title} - Odysseus Search</title>
+  <style>
+    :root {{ color-scheme: dark; --bg:#111; --fg:#eee; --muted:#999; --border:#333; --accent:#e05252; }}
+    body {{ margin:0; font:14px/1.45 system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; background:var(--bg); color:var(--fg); }}
+    main {{ max-width:900px; margin:0 auto; padding:22px 18px 40px; }}
+    form {{ display:flex; gap:8px; margin:0 0 16px; }}
+    input {{ flex:1; min-width:0; height:34px; padding:0 10px; border:1px solid var(--border); border-radius:7px; background:#181818; color:var(--fg); font:inherit; }}
+    button {{ height:34px; padding:0 13px; border:1px solid color-mix(in srgb,var(--accent) 45%,var(--border)); border-radius:7px; background:color-mix(in srgb,var(--accent) 14%,transparent); color:var(--fg); font:inherit; cursor:pointer; }}
+    h1 {{ margin:0 0 14px; font-size:16px; font-weight:650; }}
+    .status {{ color:var(--muted); font-size:12px; margin:8px 0 14px; }}
+    .result {{ display:block; padding:11px 0; border-top:1px solid var(--border); text-decoration:none; color:inherit; }}
+    .result-title {{ color:var(--fg); font-weight:650; }}
+    .result-url {{ margin-top:3px; color:color-mix(in srgb,var(--accent) 78%,var(--fg)); font-size:12px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }}
+    .result-snippet {{ margin-top:5px; color:color-mix(in srgb,var(--fg) 72%,transparent); font-size:13px; }}
+  </style>
+</head>
+<body>
+  <main>
+    <h1>Web Search</h1>
+    <form id="search-form">
+      <input id="query" value="{html.escape(safe_q, quote=True)}" autocomplete="off">
+      <button type="submit">Search</button>
+    </form>
+    <div class="status" id="status">Loading...</div>
+    <div id="results"></div>
+  </main>
+  <script>
+    const initialQuery = {q_json};
+    const input = document.getElementById('query');
+    const statusEl = document.getElementById('status');
+    const resultsEl = document.getElementById('results');
+    function esc(value) {{
+      return String(value || '').replace(/[&<>"']/g, ch => ({{'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}}[ch]));
+    }}
+    async function runSearch(query) {{
+      query = String(query || '').trim();
+      if (!query) {{ statusEl.textContent = 'Enter a search query.'; resultsEl.innerHTML = ''; return; }}
+      statusEl.textContent = 'Searching...';
+      resultsEl.innerHTML = '';
+      const fd = new FormData();
+      fd.append('query', query);
+      const res = await fetch('/api/search', {{ method: 'POST', credentials: 'same-origin', body: fd }});
+      const data = await res.json().catch(() => ({{}}));
+      const sources = Array.isArray(data.sources) ? data.sources : [];
+      if (!res.ok || data.error) {{
+        statusEl.textContent = data.error || `Search failed (${{res.status}})`;
+        return;
+      }}
+      statusEl.textContent = sources.length ? `${{sources.length}} results` : 'No results';
+      resultsEl.innerHTML = sources.map(s => {{
+        const url = s.url || s.link || '';
+        const title = s.title || url || 'Untitled';
+        const snippet = s.snippet || s.content || '';
+        return `<a class="result" href="${{esc(url)}}" target="_blank" rel="noopener noreferrer">
+          <div class="result-title">${{esc(title)}}</div>
+          <div class="result-url">${{esc(url)}}</div>
+          <div class="result-snippet">${{esc(snippet)}}</div>
+        </a>`;
+      }}).join('');
+    }}
+    document.getElementById('search-form').addEventListener('submit', ev => {{
+      ev.preventDefault();
+      const q = input.value.trim();
+      const url = new URL(window.location.href);
+      url.searchParams.set('q', q);
+      history.replaceState(null, '', url);
+      runSearch(q);
+    }});
+    runSearch(initialQuery);
+  </script>
+</body>
+</html>"""
+        return HTMLResponse(page)
+
     @router.get("/api/search/config")
     async def get_search_settings() -> Dict[str, Any]:
         return get_search_config()

routes/session_routes.py

@@ -3,25 +3,20 @@ import re
 import html
 import json
 import uuid
+import time
+from pathlib import Path
 from datetime import datetime
-from fastapi import APIRouter, Form, HTTPException, Response, Request, Depends
+from fastapi import APIRouter, Form, HTTPException, Response, Request, Query
 import logging
 
 from core.session_manager import SessionManager
 from core.models import ChatMessage
 from src.request_models import SessionResponse
 from core.database import Session as DbSession, SessionLocal, Document, GalleryImage, utcnow_naive
-from src.auth_helpers import (
-    effective_user,
-    _auth_disabled,
-    owner_filter,
-    is_delegated_credential,
-    require_chat_api_token_scope,
-)
+from src.auth_helpers import effective_user, _auth_disabled, owner_filter
 from src.session_image_cleanup import _generated_image_path_for_cleanup, session_image_refs
 from src.session_actions import is_session_recently_active
 from src.upload_handler import reserve_message_upload_references
-from src.tool_approval_scopes import sanitize_client_message_metadata
 
 
 def _sanitize_export_filename(name: str) -> str:
@@ -67,6 +62,114 @@ def _content_to_text(content) -> str:
     return ""
 
 
+def _context_info_skill_inventory(
+    skills_manager, owner: str | None, limit: int = 80
+) -> list[dict]:
+    """Compact skill metadata for TUI context/status/autocomplete.
+
+    This intentionally exposes only the skill index fields. Full SKILL.md
+    bodies remain behind manage_skills/view so context_info cannot become a
+    prompt/body dump path.
+    """
+    if not skills_manager:
+        return []
+    try:
+        indexed = skills_manager.index_for(owner=owner, active_toolsets=None)
+    except Exception:
+        return []
+    try:
+        loaded = skills_manager.load(owner=owner)
+    except Exception:
+        loaded = []
+    paths_by_name = {
+        str(skill.get("name") or ""): str(skill.get("path") or "").strip()
+        for skill in loaded
+        if isinstance(skill, dict)
+    }
+    out: list[dict] = []
+    seen: set[str] = set()
+    for row in indexed:
+        if not isinstance(row, dict):
+            continue
+        name = str(row.get("name") or "").strip()
+        if not name or name in seen:
+            continue
+        item = {"name": name}
+        description = str(row.get("description") or "").strip()
+        if description:
+            item["description"] = description
+        path = paths_by_name.get(name, "")
+        if path:
+            item["source"] = f"file: {path}"
+        out.append(item)
+        seen.add(name)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def _context_info_tool_inventory(limit: int = 80) -> list[dict]:
+    """Compact built-in tool metadata for TUI context/status/autocomplete."""
+    try:
+        from src.tool_index import BUILTIN_TOOL_DESCRIPTIONS
+    except Exception:
+        return []
+    out: list[dict] = []
+    for name, description in BUILTIN_TOOL_DESCRIPTIONS.items():
+        clean_name = str(name or "").strip()
+        if not clean_name:
+            continue
+        item = {"name": clean_name, "source": "backend"}
+        clean_description = re.sub(r"\s+", " ", str(description or "")).strip()
+        if clean_description:
+            item["description"] = clean_description[:280]
+        out.append(item)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def _context_info_agents_md_inventory(workspace: str | None, limit: int = 8) -> list[dict]:
+    """Compact AGENTS.md path metadata for the active workspace.
+
+    Bodies intentionally stay on disk. The TUI can read a selected file only
+    when the user asks for `/agent <path> --show`.
+    """
+    try:
+        from src.tool_execution import vet_workspace
+        root = vet_workspace(workspace or "")
+    except Exception:
+        root = None
+    if not root:
+        return []
+
+    start = Path(root).resolve()
+    candidates = []
+    current = start
+    while True:
+        candidate = current / "AGENTS.md"
+        if candidate.is_file():
+            candidates.append(candidate)
+        if current.parent == current:
+            break
+        current = current.parent
+        if len(candidates) >= limit:
+            break
+
+    # Codex-style precedence reads parent instructions before child overrides.
+    out: list[dict] = []
+    seen: set[str] = set()
+    for candidate in reversed(candidates):
+        path = str(candidate)
+        if path in seen:
+            continue
+        out.append({"path": path, "source": "workspace"})
+        seen.add(path)
+        if len(out) >= limit:
+            break
+    return out
+
+
 def _message_role(message) -> str:
     if isinstance(message, ChatMessage):
         return message.role or ""
@@ -131,15 +234,9 @@ def _verify_session_owner(request: Request, session_id: str, session_manager=Non
 
 logger = logging.getLogger(__name__)
 
-router = APIRouter(
-    prefix="/api",
-    tags=["sessions"],
-    dependencies=[Depends(require_chat_api_token_scope)],
-)
+router = APIRouter(prefix="/api", tags=["sessions"])
 
 def _current_user_is_admin(request: Request, user: str | None) -> bool:
-    if is_delegated_credential(request):
-        return False
     if not user:
         return False
     auth_mgr = getattr(request.app.state, "auth_manager", None)
@@ -170,36 +267,36 @@ def _reject_raw_endpoint_url_for_non_admin(
         raise HTTPException(403, "Choose a registered model endpoint")
 
 
-def _reject_delegated_session_options(
-    request: Request,
-    *,
-    skip_validation: bool = False,
-    api_key: str | None = None,
-) -> None:
-    """Keep bearer credentials from exercising interactive-admin options."""
-    if is_delegated_credential(request) and (
-        skip_validation or bool((api_key or "").strip())
-    ):
-        raise HTTPException(
-            403,
-            "API tokens cannot supply endpoint credentials or skip endpoint validation",
-        )
-
-
-def _persist_session_headers(session_id: str, headers: dict | None) -> None:
+def _persist_session_headers(session_id: str, headers: dict | None) -> bool:
     """Persist endpoint auth headers for DB-backed session metadata."""
-    db = SessionLocal()
-    try:
-        db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
-        if db_session:
-            db_session.headers = headers or {}
-            db_session.updated_at = utcnow_naive()
-            db.commit()
-    except Exception:
-        db.rollback()
-        raise
-    finally:
-        db.close()
+    delays = (0.05, 0.15, 0.35)
+    last_exc: Exception | None = None
+    for attempt in range(len(delays) + 1):
+        db = SessionLocal()
+        try:
+            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if db_session:
+                db_session.headers = headers or {}
+                db_session.updated_at = utcnow_naive()
+                db.commit()
+            return True
+        except Exception as exc:
+            db.rollback()
+            last_exc = exc
+            if attempt >= len(delays):
+                break
+            if "database is locked" not in str(exc).lower():
+                break
+            time.sleep(delays[attempt])
+        finally:
+            db.close()
+
+    logger.warning(
+        "Failed to persist headers for session %s; continuing with in-memory headers: %s",
+        session_id,
+        last_exc,
+    )
+    return False
 
 
 _HIDDEN_SYSTEM_SESSION_NAMES = {
@@ -213,6 +310,16 @@ _HIDDEN_SYSTEM_SESSION_NAMES = {
 }
 
 
+def _is_hidden_session_name(name: str | None) -> bool:
+    """Return whether a session should be omitted from the sidebar list."""
+    clean = (name or "").strip()
+    return (
+        clean in ("Nobody", "Incognito")
+        or clean in _HIDDEN_SYSTEM_SESSION_NAMES
+        or clean.startswith("SFT trace batch ")
+    )
+
+
 def _pick_endpoint_for_sort(owner=None):
     """Pick model endpoint for auto-sort LLM call — uses utility endpoint setting, falls back to default."""
     from src.endpoint_resolver import resolve_endpoint
@@ -239,6 +346,7 @@ def setup_session_routes(
     config: dict,
     webhook_manager=None,
     upload_handler=None,
+    skills_manager=None,
 ):
     """Setup session routes with the provided manager and config"""
 
@@ -287,35 +395,22 @@ def setup_session_routes(
         except Exception:
             pass
         user_sessions = session_manager.get_sessions_for_user(user)
-        # Fetch folder info from DB for each session
+        # The sidebar must be backed by persisted DB rows. SessionManager only
+        # hydrates a bounded recent cache at startup, so older-but-valid
+        # conversations can disappear after refresh if this endpoint trusts
+        # memory as the source of truth.
         db = SessionLocal()
         try:
-            folder_map = {}
-            token_map = {}
-            important_map = {}
-            created_map = {}
-            updated_map = {}
-            last_msg_map = {}
-            mode_map = {}
-            msg_count_map = {}
-            q = db.query(DbSession.id, DbSession.folder, DbSession.total_input_tokens, DbSession.total_output_tokens, DbSession.is_important, DbSession.created_at, DbSession.updated_at, DbSession.last_message_at, DbSession.mode, DbSession.message_count).filter(DbSession.archived == False)
+            q = (
+                db.query(DbSession)
+                .filter(DbSession.archived == False)
+                .order_by(DbSession.is_important.desc(), DbSession.updated_at.desc())
+            )
             q = owner_filter(q, DbSession, user)
-            rows = q.all()
-            for row in rows:
-                folder_map[row.id] = row.folder
-                token_map[row.id] = (row.total_input_tokens or 0) + (row.total_output_tokens or 0)
-                important_map[row.id] = row.is_important or False
-                created_map[row.id] = row.created_at.isoformat() if row.created_at else None
-                updated_map[row.id] = row.updated_at.isoformat() if row.updated_at else None
-                # Fall back to updated_at then created_at so sessions that
-                # predate the column (or have no messages) still sort sanely.
-                last_msg_map[row.id] = (
-                    row.last_message_at.isoformat() if row.last_message_at
-                    else (row.updated_at.isoformat() if row.updated_at
-                          else (row.created_at.isoformat() if row.created_at else None))
-                )
-                mode_map[row.id] = row.mode
-                msg_count_map[row.id] = row.message_count or 0
+            rows = [
+                row for row in q.all()
+                if not _is_hidden_session_name(row.name)
+            ]
             # Sessions with active documents that have content
             from sqlalchemy import func
             doc_session_ids = set(
@@ -334,26 +429,58 @@ def setup_session_routes(
                     GalleryImage, user)
                 .distinct().all()
             )
+
+            # Resolve saved routes without waiting for the frontend model catalog.
+            from core.database import ModelEndpoint
+            from src.endpoint_resolver import build_chat_url, normalize_base
+            endpoint_routes = {}
+            endpoint_query = owner_filter(db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), ModelEndpoint, user)
+            for endpoint in endpoint_query.all():
+                route_url = build_chat_url(normalize_base(endpoint.base_url or '')).rstrip('/')
+                endpoint_routes.setdefault(route_url, []).append(endpoint)
+            sessions = []
+            for s in rows:
+                if (
+                    (s.message_count or 0) <= 0
+                    and s.id not in doc_session_ids
+                    and s.id not in img_session_ids
+                    and s.id not in user_sessions
+                ):
+                    continue
+                # Fall back to updated_at then created_at so sessions that
+                # predate the column (or have no messages) still sort sanely.
+                last_message_at = (
+                    s.last_message_at.isoformat() if s.last_message_at
+                    else (s.updated_at.isoformat() if s.updated_at
+                          else (s.created_at.isoformat() if s.created_at else None))
+                )
+                matches = endpoint_routes.get((s.endpoint_url or '').rstrip('/'), [])
+                selected_endpoint = matches[0] if len(matches) == 1 else None
+                sessions.append({
+                    "id": s.id,
+                    "name": s.name,
+                    "model": _public_model(s.name, s.model),
+                    "endpoint_url": s.endpoint_url,
+                    "endpoint_id": selected_endpoint.id if selected_endpoint else None,
+                    "endpoint_name": selected_endpoint.name if selected_endpoint else None,
+                    "rag": s.rag,
+                    "archived": s.archived,
+                    "folder": s.folder,
+                    "cwd": s.cwd,
+                    "total_tokens": (s.total_input_tokens or 0) + (s.total_output_tokens or 0),
+                    "total_cost_usd": s.total_cost_usd or 0.0,
+                    "is_important": s.is_important or False,
+                    "created_at": s.created_at.isoformat() if s.created_at else None,
+                    "updated_at": s.updated_at.isoformat() if s.updated_at else None,
+                    "last_message_at": last_message_at,
+                    "has_documents": s.id in doc_session_ids,
+                    "has_images": s.id in img_session_ids,
+                    "mode": s.mode,
+                    "message_count": s.message_count or 0,
+                })
         finally:
             db.close()
 
-        sessions = [{"id": s.id, "name": s.name, "model": _public_model(s.name, s.model),
-                     "endpoint_url": s.endpoint_url, "rag": s.rag,
-                     "archived": s.archived, "folder": folder_map.get(s.id),
-                     "total_tokens": token_map.get(s.id, 0),
-                     "is_important": important_map.get(s.id, False),
-                     "created_at": created_map.get(s.id),
-                     "updated_at": updated_map.get(s.id),
-                     "last_message_at": last_msg_map.get(s.id),
-                     "has_documents": s.id in doc_session_ids,
-                     "has_images": s.id in img_session_ids,
-                     "mode": mode_map.get(s.id),
-                     "message_count": msg_count_map.get(s.id, 0)}
-                    for s in user_sessions.values()
-                    if not s.archived
-                    and (s.name or "").strip() not in ("Nobody", "Incognito")
-                    and (s.name or "").strip() not in _HIDDEN_SYSTEM_SESSION_NAMES]
-
         return sessions
     
     @router.post("/session", response_model=SessionResponse)
@@ -366,14 +493,10 @@ def setup_session_routes(
         skip_validation: str = Form(None),
         api_key: str = Form(""),
         endpoint_id: str = Form(""),
+        cwd: str = Form(None),
     ):
         skip_val = str(skip_validation).lower() == "true"
         user = effective_user(request)
-        _reject_delegated_session_options(
-            request,
-            skip_validation=skip_val,
-            api_key=api_key,
-        )
         endpoint_api_key = ""
         endpoint_base_url = ""
         _reject_raw_endpoint_url_for_non_admin(request, user, endpoint_id, endpoint_url)
@@ -466,6 +589,7 @@ def setup_session_routes(
             model=model_to_use,
             rag=str(rag).lower() == "true" if rag else False,
             owner=user,
+            cwd=cwd or None,
         )
         # Set auth headers for custom API-key endpoints
         resolved_key = request_api_key
@@ -490,7 +614,8 @@ def setup_session_routes(
             name=session.name,
             model=model_to_use,
             rag=str(rag).lower() == "true" if rag else False,
-            archived=False
+            archived=False,
+            cwd=session.cwd,
         )    
     @router.patch("/session/{sid}")
     def rename_session(
@@ -498,6 +623,7 @@ def setup_session_routes(
         name: str = Form(None), folder: str = Form(None),
         model: str = Form(None), endpoint_url: str = Form(None),
         endpoint_id: str = Form(None),
+        cwd: str = Form(None),
     ):
         _verify_session_owner(request, sid)
         try:
@@ -520,6 +646,19 @@ def setup_session_routes(
                     result["folder"] = folder if folder else None
             finally:
                 db.close()
+        if cwd is not None:
+            clean_cwd = cwd.strip() or None
+            db = SessionLocal()
+            try:
+                db_session = db.query(DbSession).filter(DbSession.id == sid).first()
+                if db_session:
+                    db_session.cwd = clean_cwd
+                    db_session.updated_at = utcnow_naive()
+                    db.commit()
+                    session.cwd = clean_cwd
+                    result["cwd"] = clean_cwd
+            finally:
+                db.close()
         # Switch model/endpoint mid-session
         if model is not None and endpoint_url is not None:
             user = effective_user(request)
@@ -598,11 +737,7 @@ def setup_session_routes(
         except (AttributeError, TypeError, ValueError) as exc:
             raise HTTPException(400, "Invalid message attachment metadata") from exc
         for m in messages:
-            sess.add_message(ChatMessage(
-                m["role"],
-                m["content"],
-                metadata=sanitize_client_message_metadata(m.get("metadata")),
-            ))
+            sess.add_message(ChatMessage(m["role"], m["content"], metadata=m.get("metadata")))
         session_manager.save_sessions()
         return {"ok": True, "count": len(messages)}
 
@@ -635,6 +770,8 @@ def setup_session_routes(
                     db.close()
 
                 if session_manager.delete_session(sid):
+                    from routes.chat_helpers import remove_session_sft_trace_rows
+                    remove_session_sft_trace_rows(effective_user(request), sid)
                     deleted_count += 1
             except Exception:
                 pass
@@ -659,6 +796,8 @@ def setup_session_routes(
 
             # Delete the session and all its messages
             if session_manager.delete_session(sid):
+                from routes.chat_helpers import remove_session_sft_trace_rows
+                remove_session_sft_trace_rows(effective_user(request), sid)
                 return {"status": "deleted"}
             else:
                 raise HTTPException(404, "Session not found")
@@ -944,8 +1083,6 @@ def setup_session_routes(
         model: str = Form("gpt-4o"),
         rag: str = Form(None)
     ):
-        if is_delegated_credential(request):
-            raise HTTPException(403, "This session type requires an interactive session")
         if not OPENAI_API_KEY:
             raise HTTPException(400, "Server missing OPENAI_API_KEY")
         sid = str(uuid.uuid4())
@@ -1074,11 +1211,19 @@ def setup_session_routes(
         if not session_manager.replace_messages(session_id, new_history):
             raise HTTPException(500, "Failed to save compacted history")
 
+        # Rough token estimate of the compacted history so clients can
+        # refresh their context-pressure display without waiting for the
+        # next turn's metrics event.
+        context_tokens_estimate = sum(
+            len(_message_text(m) or "") // 4 + 8 for m in new_history
+        )
+
         return {
             "ok": True,
             "summarized": len(older),
             "kept": len(recent),
             "message_count": len(new_history),
+            "context_tokens_estimate": context_tokens_estimate,
         }
 
     @router.post("/sessions/auto-sort")
@@ -1368,19 +1513,74 @@ def setup_session_routes(
         }
 
     @router.get("/session/{session_id}/context_info")
-    async def get_context_info(request: Request, session_id: str):
+    async def get_context_info(
+        request: Request,
+        session_id: str,
+        cwd: str | None = Query(default=None),
+    ):
         """Get the real context length for a session's model from the endpoint."""
         _verify_session_owner(request, session_id)
+        owner = effective_user(request)
         session = session_manager.get_session(session_id)
         if not session:
             raise HTTPException(404, "Session not found")
+        skills = _context_info_skill_inventory(skills_manager, owner=owner)
+        tools = _context_info_tool_inventory()
+        agents_md = _context_info_agents_md_inventory(cwd)
+        # Workspace visibility: lets the TUI answer "can the backend actually
+        # see this directory?" (mounted vs bridge-only) without probing.
+        from src.workspace_paths import backend_workspace_path, workspace_mount_pairs
+
+        _raw_cwd = str(cwd or getattr(session, "cwd", "") or "").strip()
+        _backend_cwd = backend_workspace_path(_raw_cwd)[:400] if _raw_cwd else ""
+        # Server-side tool policy: non-admin owners silently lose the computer
+        # tools (src/tool_security); surface that so the TUI can show it.
+        try:
+            from src.tool_security import blocked_tools_for_owner
+
+            _blocked = blocked_tools_for_owner(owner)
+        except Exception:
+            _blocked = set()
+        _computer = {"bash", "python", "read_file", "write_file", "host_shell"}
+        _policy = {
+            "computer_tools": "restricted" if _computer & _blocked else "full",
+            "reason": "non-admin owner" if _blocked else "single-user or admin",
+        }
+        _workspace = {
+            "backend_path": _backend_cwd,
+            "exists_in_backend": bool(_backend_cwd) and Path(_backend_cwd).is_dir(),
+            "mount_configured": bool(workspace_mount_pairs()),
+            "via_mount": bool(_raw_cwd) and backend_workspace_path(_raw_cwd) != _raw_cwd,
+        }
         if not session.endpoint_url or not session.model:
-            return {"context_length": None}
+            return {
+                "context_length": None,
+                "skills": skills,
+                "tools": tools,
+                "agents_md": agents_md,
+                "workspace": _workspace,
+                "tool_policy": _policy,
+            }
         try:
             from src.model_context import get_context_length
             ctx = get_context_length(session.endpoint_url, session.model)
-            return {"context_length": ctx, "model": session.model}
+            return {
+                "context_length": ctx,
+                "model": session.model,
+                "skills": skills,
+                "tools": tools,
+                "agents_md": agents_md,
+                "workspace": _workspace,
+                "tool_policy": _policy,
+            }
         except Exception:
-            return {"context_length": None}
+            return {
+                "context_length": None,
+                "skills": skills,
+                "tools": tools,
+                "agents_md": agents_md,
+                "workspace": _workspace,
+                "tool_policy": _policy,
+            }
 
     return router

routes/shell_routes.py

@@ -11,6 +11,7 @@ import shutil
 import subprocess
 import uuid
 import tempfile
+import time
 from collections import namedtuple
 from pathlib import Path
 from typing import Dict, Any
@@ -22,6 +23,7 @@ from src.host_docker_access import (
     running_in_container as _running_in_container,
 )
 from src.optional_deps import prepare_optional_dependency_import
+from src.auth_helpers import _auth_disabled
 
 # POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist
 # on Windows, so importing them unconditionally crashed app startup there
@@ -53,6 +55,11 @@ from core.platform_compat import (
 def _require_admin(request: Request):
     """Reject non-admin callers. Shell exec is admin-only — never expose to
     regular users; that's RCE-after-signup."""
+    # In the explicitly single-user, auth-disabled deployment the middleware
+    # does not attach a current user. AuthManager is still instantiated by the
+    # app, so checking only for its presence incorrectly returns 403 here.
+    if _auth_disabled():
+        return
     auth_manager = getattr(request.app.state, "auth_manager", None)
     if not auth_manager:
         # No auth at all — only safe in fully-trusted localhost dev mode
@@ -78,6 +85,13 @@ def _reject_cross_site(request: Request):
 _SSH_PORT_RE = re.compile(r"^\d{1,5}$")
 _SAFE_VENV_RE = re.compile(r"^[A-Za-z0-9_./~-]+$")
 
+# Dependency probes can involve several SSH/import checks. Keep the result
+# briefly so the Dependencies tab and a pre-launch check arriving together do
+# not repeat the same expensive work. Installation clears this cache.
+_PACKAGE_STATUS_CACHE: dict[tuple[str, ...], tuple[float, dict[str, Any]]] = {}
+_PACKAGE_STATUS_CACHE_TTL = 3.0
+_PACKAGE_STATUS_CACHE_MAX = 64
+
 
 def _ssh_base_argv(host: str, ssh_port: str | None) -> list[str]:
     """Build an ssh argv prefix for remote probes without local-shell parsing."""
@@ -204,6 +218,19 @@ def _package_installed_from_probe(name: str, probe: dict) -> bool:
             (dists.get("transformers") or modules.get("transformers", {}).get("real_module"))
             and (dists.get("torch") or modules.get("torch", {}).get("real_module"))
         )
+    if name == "office_docs":
+        return bool(
+            dists.get("markitdown")
+            or modules.get("markitdown", {}).get("real_module")
+            or dists.get("python-docx")
+            or modules.get("docx", {}).get("real_module")
+        )
+    if name == "psd_tools":
+        return bool(dists.get("psd-tools") or modules.get("psd_tools", {}).get("real_module"))
+    if name == "pymupdf":
+        return bool(dists.get("PyMuPDF") or modules.get("fitz", {}).get("real_module"))
+    if name == "libreoffice":
+        return bool(binaries.get("soffice") or binaries.get("libreoffice"))
     if name == "hf_transfer":
         return bool(
             dists.get("hf-transfer")
@@ -254,6 +281,28 @@ def _package_status_note(name: str, probe: dict) -> str:
         if _package_installed_from_probe(name, probe):
             return f"SAM object masks: transformers {dists.get('transformers', 'available')} with torch {dists.get('torch', 'available')}"
         return "SAM click/object mask selection needs transformers and torch."
+    if name == "office_docs":
+        if _package_installed_from_probe(name, probe):
+            if dists.get("markitdown"):
+                return f"Office document extraction: markitdown {dists['markitdown']}"
+            if dists.get("python-docx"):
+                return f"Word document extraction: python-docx {dists['python-docx']}"
+            return "Office document extraction available"
+        return "Office attachments need MarkItDown for full fidelity; DOCX has a basic built-in fallback."
+    if name == "psd_tools":
+        if _package_installed_from_probe(name, probe):
+            return f"PSD support: psd-tools {dists.get('psd-tools', 'available')}"
+        return "PSD files need psd-tools for layer/image parsing."
+    if name == "pymupdf":
+        if _package_installed_from_probe(name, probe):
+            return f"PDF forms/rendering: PyMuPDF {dists.get('PyMuPDF', 'available')}"
+        return "Advanced PDF open/render/form features need PyMuPDF."
+    if name == "libreoffice":
+        if binaries.get("soffice"):
+            return f"DOCX signable preview converter: {binaries['soffice']}"
+        if binaries.get("libreoffice"):
+            return f"DOCX signable preview converter: {binaries['libreoffice']}"
+        return "DOCX signing preview needs LibreOffice/soffice to convert Word files to PDF."
     if name == "mlx_lm":
         if _package_installed_from_probe(name, probe):
             return f"MLX LM {dists.get('mlx-lm', 'available')}"
@@ -399,16 +448,21 @@ dist_names={{
     'diffusers':['diffusers','torch'],
     'krea_diffusers':['diffusers','torch'],
     'sam_mask':['transformers','torch'],
-    'hf_transfer':['hf-transfer','hf_transfer'],
-}}
-bin_names={{
+    'office_docs':['markitdown','python-docx'],
+	    'psd_tools':['psd-tools'],
+	    'pymupdf':['PyMuPDF'],
+	    'libreoffice':[],
+	    'hf_transfer':['hf-transfer','hf_transfer'],
+	}}
+	bin_names={{
     'vllm':['vllm'],
     'llama_cpp':['llama-server'],
     'mflux':['mflux-generate-qwen', 'mflux-generate'],
-    'mlx_lama_swift':['odysseus-mlx-inpaint', 'mlx-lama-serve'],
-    'mlx_ddcolor_swift':['odysseus-mlx-colorize', 'mlx-ddcolor-serve'],
-    'tmux':['tmux'],
-}}
+	    'mlx_lama_swift':['odysseus-mlx-inpaint', 'mlx-lama-serve'],
+	    'mlx_ddcolor_swift':['odysseus-mlx-colorize', 'mlx-ddcolor-serve'],
+	    'libreoffice':['soffice', 'libreoffice'],
+	    'tmux':['tmux'],
+	}}
 
 def add_user_install_bins_to_path():
     candidates = []
@@ -457,6 +511,13 @@ def probe(n):
     mods = {{n: mod_status(n)}}
     if n == 'diffusers':
         mods['torch'] = mod_status('torch')
+    if n == 'office_docs':
+        mods['markitdown'] = mod_status('markitdown')
+        mods['docx'] = mod_status('docx')
+    if n == 'psd_tools':
+        mods['psd_tools'] = mod_status('psd_tools')
+    if n == 'pymupdf':
+        mods['fitz'] = mod_status('fitz')
     dists = dist_status(dist_names.get(n, [n]))
     bins = {{b: shutil.which(b) for b in bin_names.get(n, [])}}
     files = {{}}
@@ -1145,6 +1206,7 @@ def setup_shell_routes() -> APIRouter:
         "make":            {"debian": ["make"], "arch": ["make"], "fedora": ["make"], "alpine": ["make"], "suse": ["make"], "macos": []},
         "git":             {"debian": ["git"], "arch": ["git"], "fedora": ["git"], "alpine": ["git"], "suse": ["git"], "macos": ["git"]},
         "tmux":            {"debian": ["tmux"], "arch": ["tmux"], "fedora": ["tmux"], "alpine": ["tmux"], "suse": ["tmux"], "macos": ["tmux"]},
+        "libreoffice":     {"debian": ["libreoffice"], "arch": ["libreoffice-fresh"], "fedora": ["libreoffice"], "alpine": ["libreoffice"], "suse": ["libreoffice"], "macos": ["--cask", "libreoffice"]},
     }
     _BACKEND_EXTRAS = {
         "cuda":   {"debian": ["nvidia-cuda-toolkit"], "arch": ["cuda"], "fedora": ["cuda-toolkit"], "alpine": [], "suse": ["cuda"], "macos": []},
@@ -1206,13 +1268,16 @@ def setup_shell_routes() -> APIRouter:
         import sys
 
         platform_l = (platform or "").strip().lower()
-        model_hint_l = (model_hint or "").strip().lower()
-        has_krea_model = "krea" in model_hint_l
-        has_lama_mlx_model = any(
-            key in model_hint_l
-            for key in ("lama", "mi-gan", "migan", "inpainting-mlx")
+        package_cache_key = (
+            (host or "").strip(),
+            (ssh_port or "").strip(),
+            (venv or "").strip(),
+            (backend or "").strip().lower(),
+            platform_l,
         )
-        has_ddcolor_mlx_model = "ddcolor" in model_hint_l
+        cached_status = _PACKAGE_STATUS_CACHE.get(package_cache_key)
+        if cached_status and time.monotonic() - cached_status[0] < _PACKAGE_STATUS_CACHE_TTL:
+            return cached_status[1]
         _prepend_user_install_bins_to_path()
         importlib.invalidate_caches()
         try:
@@ -1396,6 +1461,13 @@ def setup_shell_routes() -> APIRouter:
                 "category": "Image",
                 "target": "local",
             },
+            {
+                "name": "psd_tools",
+                "pip": "psd-tools",
+                "desc": "Open Photoshop PSD files and inspect flattened/layered image data",
+                "category": "Image",
+                "target": "local",
+            },
             # ── Tools ──
             {
                 "name": "playwright",
@@ -1404,6 +1476,31 @@ def setup_shell_routes() -> APIRouter:
                 "category": "Tools",
                 "target": "local",
             },
+            {
+                "name": "office_docs",
+                "pip": "markitdown[docx,pptx,xlsx,xls]",
+                "desc": "Open Office attachments and documents (.docx, .pptx, .xlsx, .xls) as readable Markdown",
+                "category": "Tools",
+                "target": "local",
+            },
+            {
+                "name": "pymupdf",
+                "pip": "PyMuPDF",
+                "desc": "Advanced PDF opening, rendering, forms, annotations, and signatures",
+                "category": "Tools",
+                "target": "local",
+            },
+            {
+                "name": "libreoffice",
+                "pip": "",
+                "desc": "Convert DOCX attachments to signable PDF previews",
+                "category": "Tools",
+                "target": "local",
+                "kind": "system",
+                "system_prereqs": ["libreoffice"],
+                "install_cmd": "sudo apt install -y libreoffice || brew install --cask libreoffice",
+                "install_hint": "Install LibreOffice/soffice where Odysseus runs to open DOCX attachments as signable PDF previews. Without it, DOCX opens as readable Markdown.",
+            },
         ]
 
         # Most packages should not be installed through external means. Hence, set the default of the
@@ -1411,21 +1508,10 @@ def setup_shell_routes() -> APIRouter:
         for pkg in packages:
             pkg.setdefault("install_cmd", None)
             pkg.setdefault("update_cmd", None)
-        if not has_krea_model:
-            packages = [
-                p for p in packages
-                if p.get("name") not in {"krea_diffusers", "transformers"}
-            ]
-        if not has_lama_mlx_model:
-            packages = [
-                p for p in packages
-                if p.get("name") != "mlx_lama_swift"
-            ]
-        if not has_ddcolor_mlx_model:
-            packages = [
-                p for p in packages
-                if p.get("name") != "mlx_ddcolor_swift"
-            ]
+        # Keep the Image section complete. Dependency visibility is a product
+        # capability decision, not a substring test against a model id. Model
+        # catalogs may declare an explicit runtime package, while the generic
+        # backend preflight handles ordinary models.
         # Remote check: for remote-target packages, probe the selected server's
         # venv over SSH so a remote `pip install` actually reflects here.
         remote_status: dict = {}
@@ -1596,6 +1682,14 @@ def setup_shell_routes() -> APIRouter:
                         if IS_APPLE_SILICON
                         else "Requires a native Apple Silicon Mac with Apple Foundational Models support."
                     )
+                elif pkg["name"] == "libreoffice":
+                    soffice_path = shutil.which("soffice") or shutil.which("libreoffice")
+                    pkg["installed"] = soffice_path is not None
+                    pkg["status_note"] = (
+                        f"DOCX signable preview converter: {soffice_path}"
+                        if soffice_path
+                        else "DOCX signing preview needs LibreOffice/soffice."
+                    )
                 else:
                     pkg["installed"] = shutil.which(pkg["name"]) is not None
             elif pkg["name"] == "llama_cpp" and shutil.which("llama-server"):
@@ -1757,7 +1851,12 @@ def setup_shell_routes() -> APIRouter:
                 )
                 pkg["applicable"] = status.applicable
                 pkg["install_hint"] = status.install_hint
-        return {"packages": packages}
+        result = {"packages": packages}
+        if len(_PACKAGE_STATUS_CACHE) >= _PACKAGE_STATUS_CACHE_MAX:
+            oldest_key = min(_PACKAGE_STATUS_CACHE, key=lambda key: _PACKAGE_STATUS_CACHE[key][0])
+            _PACKAGE_STATUS_CACHE.pop(oldest_key, None)
+        _PACKAGE_STATUS_CACHE[package_cache_key] = (time.monotonic(), result)
+        return result
 
     @router.post("/api/cookbook/packages/install")
     async def install_package(request: Request):
@@ -1802,6 +1901,7 @@ def setup_shell_routes() -> APIRouter:
             *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
         )
         stdout, stderr = await proc.communicate()
+        _PACKAGE_STATUS_CACHE.clear()
         if proc.returncode == 0:
             return {"ok": True, "output": stdout.decode()[-200:]}
         return {"ok": False, "error": stderr.decode()[-300:]}
@@ -1824,7 +1924,7 @@ def setup_shell_routes() -> APIRouter:
         ssh_port = body.get("ssh_port")
         # Names users can request — must match canonical names used in the
         # deps catalog's `system_prereqs` field and on the System rows.
-        ALLOWED = {"cmake", "build-essential", "g++", "gcc", "git", "tmux", "make"}
+        ALLOWED = {"cmake", "build-essential", "g++", "gcc", "git", "tmux", "make", "libreoffice"}
         pkgs = [str(p).strip() for p in raw if str(p).strip() in ALLOWED]
         if not pkgs:
             return {"ok": False, "error": "no installable packages requested (allowlist: " + ", ".join(sorted(ALLOWED)) + ")"}
@@ -1854,7 +1954,15 @@ def setup_shell_routes() -> APIRouter:
                 else: out.append(n)
             return out
         def _brew(names):
-            return [n for n in names if n not in ("build-essential", "g++", "gcc", "make")]
+            out = []
+            for n in names:
+                if n in ("build-essential", "g++", "gcc", "make"):
+                    continue
+                if n == "libreoffice":
+                    out += ["--cask", "libreoffice"]
+                else:
+                    out.append(n)
+            return out
         # Build a single shell snippet that detects the package manager and
         # runs the right install. Non-interactive sudo (-n) only — if sudo
         # asks for a password the script reports it instead of hanging.
@@ -1920,6 +2028,7 @@ def setup_shell_routes() -> APIRouter:
             combined = err_txt or tail_out or f"exit code {proc.returncode}"
         else:
             combined = None
+        _PACKAGE_STATUS_CACHE.clear()
         return {
             "ok": ok,
             "exit_code": proc.returncode,

routes/task/task_routes.py

@@ -10,7 +10,7 @@ from typing import Optional, Dict, Any
 from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel
 
-from core.database import SessionLocal, ScheduledTask, TaskRun
+from core.database import SessionLocal, ScheduledTask, TaskRun, NotificationLog
 from core.constants import internal_api_base
 from src.auth_helpers import get_current_user
 from src.constants import DATA_DIR, EMAIL_URGENCY_CACHE_DIR
@@ -569,6 +569,57 @@ def setup_task_routes(task_scheduler) -> APIRouter:
         notes = task_scheduler.pop_notifications(owner=user)
         return {"notifications": notes}
 
+    @router.get("/notification-logs")
+    async def get_notification_logs(request: Request, limit: int = 200):
+        """Return persisted task notifications without consuming them."""
+        user = _owner(request)
+        if not user:
+            return {"notifications": []}
+        limit = max(1, min(int(limit or 200), 1000))
+        db = SessionLocal()
+        try:
+            rows = (db.query(NotificationLog)
+                    .filter(NotificationLog.owner == user)
+                    .order_by(NotificationLog.timestamp.desc())
+                    .limit(limit)
+                    .all())
+            return {"notifications": [
+                {
+                    "id": row.id,
+                    "task_name": row.task_name,
+                    "task_id": row.task_id,
+                    "status": row.status,
+                    "body": row.body,
+                    "timestamp": row.timestamp.isoformat() + "Z" if row.timestamp else None,
+                }
+                for row in rows
+            ]}
+        finally:
+            db.close()
+
+    @router.post("/notification-logs")
+    async def create_notification_log(request: Request):
+        """Persist an in-app toast so Settings can show notification history."""
+        user = _owner(request)
+        if not user:
+            raise HTTPException(401, "Authentication required")
+        body = await request.json()
+        message = str(body.get("body") or "").strip()[:2000]
+        if not message:
+            raise HTTPException(400, "Notification body required")
+        row = NotificationLog(
+            id=str(uuid.uuid4()), owner=user,
+            task_name=str(body.get("title") or "Odysseus")[:200],
+            status="error" if body.get("status") == "error" else "success",
+            body=message,
+        )
+        db = SessionLocal()
+        try:
+            db.add(row); db.commit()
+            return {"success": True}
+        finally:
+            db.close()
+
     @router.post("/{task_id}/clear-cache")
     async def clear_task_cache(request: Request, task_id: str):
         """Clear derived cache for one built-in task."""

routes/upload_routes.py

@@ -190,7 +190,8 @@ def setup_upload_routes(upload_handler):
             return None
         return session_id
 
-    def _promote_chat_image_to_gallery(meta: dict, owner: str | None, session_id: str | None = None) -> str | None:
+    def _promote_chat_image_to_gallery(meta: dict, owner: str | None, session_id: str | None = None,
+                                       gallery_id: str | None = None) -> str | None:
         """Make chat-uploaded images visible in Gallery without changing chat storage."""
         is_image_file = getattr(upload_handler, "is_image_file", None)
         if not callable(is_image_file):
@@ -205,6 +206,21 @@ def setup_upload_routes(upload_handler):
         db = SessionLocal()
         try:
             file_hash = meta.get("hash")
+            if gallery_id:
+                existing = db.query(GalleryImage).filter(
+                    GalleryImage.id == gallery_id,
+                    GalleryImage.is_active == True,  # noqa: E712
+                ).first()
+                if existing and (not owner or existing.owner == owner):
+                    image_dir = Path(GENERATED_IMAGES_DIR)
+                    image_dir.mkdir(parents=True, exist_ok=True)
+                    shutil.copy2(source_path, image_dir / existing.filename)
+                    existing.file_hash = file_hash
+                    existing.file_size = meta.get("size")
+                    existing.width = meta.get("width")
+                    existing.height = meta.get("height")
+                    db.commit()
+                    return existing.id
             if file_hash:
                 q = db.query(GalleryImage).filter(
                     GalleryImage.file_hash == file_hash,
@@ -259,6 +275,7 @@ def setup_upload_routes(upload_handler):
         request: Request,
         files: List[UploadFile] = File(...),
         session_id: Optional[str] = Form(None),
+        gallery_id: Optional[str] = Form(None),
     ):
         """Upload files with enhanced security and organization."""
         if not isinstance(session_id, str):
@@ -289,7 +306,7 @@ def setup_upload_routes(upload_handler):
             try:
                 owner = effective_user(request)
                 meta = upload_handler.save_upload(u, client_ip, owner=owner)
-                gallery_id = _promote_chat_image_to_gallery(meta, owner, session_id)
+                promoted_gallery_id = _promote_chat_image_to_gallery(meta, owner, session_id, gallery_id)
                 item = {
                     "id": meta["id"],
                     "name": meta["name"],
@@ -303,8 +320,8 @@ def setup_upload_routes(upload_handler):
                     "height": meta.get("height"),
                     "is_duplicate": meta.get("is_duplicate", False)
                 }
-                if gallery_id:
-                    item["gallery_id"] = gallery_id
+                if promoted_gallery_id:
+                    item["gallery_id"] = promoted_gallery_id
                 out.append(item)
             except HTTPException:
                 raise

routes/workspace_routes.py

@@ -82,4 +82,26 @@ def setup_workspace_routes():
         resolved = vet_workspace(path)
         return {"ok": resolved is not None, "path": resolved}
 
+    @router.get("/default")
+    def default_workspace(request: Request):
+        """Return the explicitly configured backend workspace, if usable.
+
+        WebUI has no local launch directory: it runs against this backend's
+        filesystem. An explicit default gives it the same zero-setup behavior
+        as TUI while keeping workspace access opt-in and server-vetted.
+        """
+        owner = get_current_user(request)
+        if not owner_is_admin_or_single_user(owner):
+            raise HTTPException(status_code=403, detail="Workspace default is admin-only")
+
+        configured = os.environ.get("ODYSSEUS_WORKSPACE_DEFAULT", "").strip()
+        if not configured:
+            return {"ok": False, "path": None}
+
+        from src.tool_execution import vet_workspace
+        from src.workspace_paths import backend_workspace_path
+
+        resolved = vet_workspace(backend_workspace_path(configured))
+        return {"ok": resolved is not None, "path": resolved}
+
     return router

scripts/add_hwfit_models.py

@@ -31,7 +31,43 @@ from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
 DATA_PATH = os.path.join(os.path.dirname(__file__), "..", "services", "hwfit", "data", "hf_models.json")
 DATA_PATH = os.path.abspath(DATA_PATH)
 
-AUTHORS = ["cyankiwi"]
+# Official / major model-provider orgs to refresh into the Cookbook catalog.
+# Keep this broad enough that new first-party releases appear after running the
+# updater, while avoiding a global HF scan that would pull in every community fork.
+AUTHORS = [
+    # Community quant provider we already use for AWQ/FP8 serving recipes.
+    "cyankiwi",
+    # Major first-party model providers.
+    "Qwen",
+    "deepseek-ai",
+    "zai-org",
+    "MiniMaxAI",
+    "moonshotai",
+    "mistralai",
+    "meta-llama",
+    "google",
+    "google-deepmind",
+    "microsoft",
+    "nvidia",
+    "CohereLabs",
+    "ai21labs",
+    "Tencent-Hunyuan",
+    "ibm-granite",
+    "tiiuae",
+    "01-ai",
+    "allenai",
+    "HuggingFaceTB",
+    "openai",
+]
+BROAD_AUTHORS_SKIP_FALLBACK_PROBES = {
+    # These orgs have hundreds/thousands of mixed-purpose repos. For them,
+    # catalog only entries that can be sized from cheap list metadata / repo
+    # names; do not block refreshes on per-repo config/safetensors downloads.
+    "google",
+    "microsoft",
+    "nvidia",
+    "allenai",
+}
 # Specific repos to add (in addition to the authors above). Optional explicit
 # overrides {repo: {field: value}} for things the name/metadata can't convey.
 EXTRA_REPOS = {
@@ -50,6 +86,21 @@ _GENERIC_TAGS = {
     "quantized", "chat",
 }
 
+_GEN_MODEL_PIPELINES = {
+    "text-generation",
+    "text2text-generation",
+    "image-text-to-text",
+    "text-generation-inference",
+    "conversational",
+}
+
+_GEN_MODEL_KEYWORDS = (
+    "llama", "gemma", "qwen", "deepseek", "glm", "chatglm", "minimax",
+    "kimi", "moonshot", "mistral", "mixtral", "codestral", "ministral",
+    "phi", "mai", "nemotron", "granite", "command", "aya", "jamba",
+    "hunyuan", "yi-", "yi_", "falcon", "olmo", "openai",
+)
+
 api = HfApi()
 
 
@@ -207,6 +258,8 @@ def _quant_from_name(name):
     n = name.lower()
     if "nvfp4" in n:
         return "NVFP4"
+    if re.search(r"(^|[-_/])bf16($|[-_/])", n):
+        return "BF16"
     if "mxfp4" in n:
         return "MXFP4"
     if re.search(r"(^|[-_/])nf4($|[-_/])", n):
@@ -248,7 +301,7 @@ def _arch_from_tags(tags):
     return ""
 
 
-def _entry_from_modelinfo(mi, overrides):
+def _entry_from_modelinfo(mi, overrides, *, probe_config=True, probe_safetensors=True):
     name = mi.id
     provider = name.split("/")[0]
     total, active = _parse_params(name)
@@ -272,7 +325,7 @@ def _entry_from_modelinfo(mi, overrides):
     # before safetensors so non-standard names still resolve without a
     # per-repo manual override in EXTRA_REPOS. Source repo first (works for
     # unquantized models) then the quantized parent via base_model:.
-    if total is None:
+    if total is None and probe_config:
         config_targets = [name]
         bm = _base_model_tag(getattr(mi, "tags", None))
         if bm and bm != name:
@@ -293,7 +346,7 @@ def _entry_from_modelinfo(mi, overrides):
     # therefore undercounts real parameter count by the same factor, which
     # then feeds a wrong `min_vram_gb` downstream. Sum per-dtype and unpack
     # the packed I32 tensors so the catalog stores the true param count.
-    if total is None:
+    if total is None and probe_safetensors:
         try:
             full = api.model_info(name, files_metadata=False)
             st = getattr(full, "safetensors", None)
@@ -322,7 +375,8 @@ def _entry_from_modelinfo(mi, overrides):
     created = getattr(mi, "created_at", None)
     rel = created.strftime("%Y-%m-%d") if created else datetime.utcnow().strftime("%Y-%m-%d")
     # Rough RAM/VRAM hints (fit.py recomputes the real requirement from params+quant).
-    _BPP = {"AWQ-4bit": 0.58, "GPTQ-Int4": 0.58, "mlx-4bit": 0.55, "mlx-6bit": 0.85,
+    _BPP = {"F16": 2.0, "BF16": 2.0,
+            "AWQ-4bit": 0.58, "GPTQ-Int4": 0.58, "mlx-4bit": 0.55, "mlx-6bit": 0.85,
             "AWQ-8bit": 1.1, "GPTQ-Int8": 1.1, "mlx-8bit": 1.1, "FP8": 1.1,
             "FP4": 0.58, "NVFP4": 0.58, "MXFP4": 0.58, "NF4": 0.58,
             "INT4": 0.58, "INT8": 1.1, "W4A16": 0.58, "W8A8": 1.1, "W8A16": 1.1,
@@ -360,6 +414,28 @@ def _entry_from_modelinfo(mi, overrides):
     return entry
 
 
+def _is_likely_catalog_model(mi):
+    """Cheap prefilter before config/safetensors probes.
+
+    Major HF orgs include thousands of encoder, CV, audio, adapter, and demo
+    repos. Cookbook's serve catalog is for generative models, so only do the
+    expensive config/model_info fallback for repos that already look relevant
+    from list_models(full=True) metadata.
+    """
+    name = str(getattr(mi, "id", "") or "")
+    if not name:
+        return False
+    # Size-bearing model names are usually exactly what we want (7B, 70B, A3B).
+    if _parse_params(name)[0]:
+        return True
+    pipeline = str(getattr(mi, "pipeline_tag", "") or "").lower()
+    if pipeline in _GEN_MODEL_PIPELINES:
+        return True
+    tags = " ".join(str(t).lower() for t in (getattr(mi, "tags", None) or []))
+    haystack = f"{name.lower()} {pipeline} {tags}"
+    return any(k in haystack for k in _GEN_MODEL_KEYWORDS)
+
+
 def main():
     with open(DATA_PATH, encoding="utf-8") as f:
         catalog = json.load(f)
@@ -377,8 +453,16 @@ def main():
         for mi in models:
             if mi.id in existing and not overwrite:
                 continue
+            if not _is_likely_catalog_model(mi):
+                continue
             ov = EXTRA_REPOS.get(mi.id)
-            entry = _entry_from_modelinfo(mi, ov)
+            skip_fallbacks = author in BROAD_AUTHORS_SKIP_FALLBACK_PROBES
+            entry = _entry_from_modelinfo(
+                mi,
+                ov,
+                probe_config=not skip_fallbacks,
+                probe_safetensors=not skip_fallbacks,
+            )
             if entry:
                 to_add[mi.id] = entry
 

scripts/analyze_odysseus_eval_targets.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Rank next Odysseus tool-router improvement targets from eval artifacts."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+
+def _load(path: Path) -> dict[str, Any]:
+    with path.open("r", encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def _metric(record: dict[str, Any], key: str, default: Any = None) -> Any:
+    metrics = record.get("metrics") or {}
+    return metrics.get(key, default)
+
+
+def _tool_rounds(record: dict[str, Any]) -> int:
+    metrics = record.get("metrics") or {}
+    usage = metrics.get("usage_buckets") or []
+    round_models = metrics.get("round_models") or []
+    if usage:
+        return len(usage)
+    if round_models:
+        return len(round_models)
+    snapshots = record.get("model_request_snapshots") or []
+    if snapshots:
+        return len(snapshots)
+    return 0
+
+
+def _is_infra_failure_error(error: dict[str, Any]) -> bool:
+    if not isinstance(error, dict):
+        return False
+    status = error.get("status")
+    text = " ".join(
+        str(error.get(key) or "")
+        for key in ("error", "message", "detail", "type")
+    ).lower()
+    if status in {502, 503, 504, 520, 521, 522, 523, 524}:
+        return True
+    return bool(
+        "cannot reach" in text
+        or "connection refused" in text
+        or "connection reset" in text
+        or "connect timeout" in text
+        or "read timeout" in text
+        or "unreachable" in text
+        or "cooldown active" in text
+        or "upstream protocol error" in text
+        or ("upstream" in text and "failed" in text)
+    )
+
+
+def _record_has_infra_error(record: dict[str, Any]) -> bool:
+    if record.get("infra_failure") is True:
+        return True
+    errors = list(record.get("stream_errors") or [])
+    stream_exception = record.get("stream_exception")
+    if isinstance(stream_exception, dict):
+        errors.append(stream_exception)
+    return any(_is_infra_failure_error(error) for error in errors)
+
+
+def _record_status(record: dict[str, Any]) -> str:
+    if _record_has_infra_error(record):
+        return "infra"
+    if not record.get("native_call_ok"):
+        return "routing"
+    if not record.get("command_contract_ok"):
+        return "contract"
+    if not record.get("tool_invocation_ok"):
+        return "invocation"
+    if not record.get("command_outcome_ok"):
+        return "outcome"
+    if not record.get("response_quality_ok"):
+        return "response"
+    if record.get("duplicate_textual_call"):
+        return "duplicate_text"
+    if record.get("repetitive_tool_call"):
+        return "repeat"
+    return "pass"
+
+
+def _first_output(record: dict[str, Any]) -> dict[str, Any]:
+    outputs = record.get("tool_outputs") or []
+    return outputs[0] if outputs else {}
+
+
+def _print_row(record: dict[str, Any]) -> None:
+    case = record.get("case")
+    status = _record_status(record)
+    first_tool = record.get("first_tool")
+    expected = record.get("expected_tool")
+    output = _first_output(record)
+    input_tokens = _metric(record, "input_tokens")
+    response_time = _metric(record, "response_time")
+    elapsed = record.get("elapsed_seconds")
+    rounds = _tool_rounds(record)
+    exit_code = output.get("exit_code")
+    print(
+        f"- {case}: status={status}, expected={expected}, first={first_tool}, "
+        f"rounds={rounds}, input={input_tokens}, response={response_time}s, "
+        f"elapsed={elapsed}s, exit={exit_code}"
+    )
+
+
+def _top(records: list[dict[str, Any]], key, limit: int) -> list[dict[str, Any]]:
+    return sorted(records, key=key, reverse=True)[:limit]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("artifact", type=Path)
+    parser.add_argument("--limit", type=int, default=12)
+    args = parser.parse_args()
+
+    artifact = _load(args.artifact)
+    records = list(artifact.get("records") or [])
+    infra = [record for record in records if _record_has_infra_error(record)]
+    evaluable = [record for record in records if not _record_has_infra_error(record)]
+    failed = [record for record in evaluable if _record_status(record) != "pass"]
+    slow = _top(
+        [record for record in evaluable if _metric(record, "response_time") is not None],
+        lambda record: float(_metric(record, "response_time", 0) or 0),
+        args.limit,
+    )
+    token_heavy = _top(
+        [record for record in evaluable if _metric(record, "input_tokens") is not None],
+        lambda record: int(_metric(record, "input_tokens", 0) or 0),
+        args.limit,
+    )
+    multi_round = _top(
+        [record for record in evaluable if _tool_rounds(record) > 1],
+        lambda record: (_tool_rounds(record), float(_metric(record, "response_time", 0) or 0)),
+        args.limit,
+    )
+
+    print(f"artifact: {args.artifact}")
+    print(f"model: {artifact.get('model')}")
+    print(f"cases: {artifact.get('cases', len(records))}")
+    print(f"infra: {len(infra)}")
+    print(f"evaluable: {len(evaluable)}")
+    print(f"failures: {len(failed)}")
+    print()
+
+    print("failures:")
+    if failed:
+        for record in failed:
+            _print_row(record)
+    else:
+        print("- none")
+    print()
+
+    print(f"slowest_{len(slow)}:")
+    for record in slow:
+        _print_row(record)
+    print()
+
+    print(f"token_heaviest_{len(token_heavy)}:")
+    for record in token_heavy:
+        _print_row(record)
+    print()
+
+    print(f"multi_round_{len(multi_round)}:")
+    if multi_round:
+        for record in multi_round:
+            _print_row(record)
+    else:
+        print("- none")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/assemble_sft_clean_corpus.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Assemble kept and validated repaired sessions into a clean SFT corpus."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import Counter, defaultdict
+from pathlib import Path
+from typing import Any
+
+
+def load_jsonl(path: Path) -> list[dict[str, Any]]:
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--trace", type=Path, required=True)
+    parser.add_argument("--verdicts", type=Path, required=True)
+    parser.add_argument("--repairs", type=Path, action="append", default=[])
+    parser.add_argument("--out-trace", type=Path, required=True)
+    parser.add_argument("--report", type=Path, required=True)
+    args = parser.parse_args()
+
+    source: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for row in load_jsonl(args.trace):
+        source[str(row.get("session_id") or "")].append(row)
+    verdicts = {str(row.get("session_id") or ""): row for row in load_jsonl(args.verdicts)}
+    repaired: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for path in args.repairs:
+        for row in load_jsonl(path):
+            repaired[str(row.get("session_id") or "")].append(row)
+
+    output: list[dict[str, Any]] = []
+    excluded: list[dict[str, Any]] = []
+    counts: Counter[str] = Counter()
+    for session_id in sorted(source):
+        verdict = verdicts.get(session_id)
+        decision = str((verdict or {}).get("verdict") or "missing")
+        if decision == "keep":
+            output.extend(source[session_id])
+            counts["kept"] += 1
+        elif decision == "repair" and repaired.get(session_id):
+            output.extend(repaired[session_id])
+            counts["repaired"] += 1
+        else:
+            counts["excluded"] += 1
+            excluded.append({
+                "session_id": session_id,
+                "verdict": decision,
+                "issues": (verdict or {}).get("issues") or [],
+                "repair_missing": decision == "repair" and session_id not in repaired,
+            })
+
+    args.out_trace.parent.mkdir(parents=True, exist_ok=True)
+    args.out_trace.write_text(
+        "\n".join(json.dumps(row, ensure_ascii=False) for row in output) + ("\n" if output else ""),
+        encoding="utf-8",
+    )
+    report = {
+        "source_sessions": len(source),
+        "output_sessions": counts["kept"] + counts["repaired"],
+        "output_turns": len(output),
+        "decisions": dict(counts),
+        "excluded": excluded,
+    }
+    args.report.parent.mkdir(parents=True, exist_ok=True)
+    args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({key: value for key, value in report.items() if key != "excluded"}, indent=2))
+
+
+if __name__ == "__main__":
+    main()

scripts/audit_email_sft_with_deepseek.py

@@ -0,0 +1,323 @@
+#!/usr/bin/env python3
+"""Audit recent Odysseus email SFT conversations with a DeepSeek judge."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sqlite3
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+DB = ROOT / "data" / "app.db"
+OUT_DIR = ROOT / "data" / "audits"
+
+
+EMAIL_RE = re.compile(
+    r"\b(email|emails|inbox|mailbox|attachment|attachments|draft|reply|archive|"
+    r"delete|spam|blocked|unblock|read|unread|favorite|done|contact)\b",
+    re.I,
+)
+
+
+def decrypt_secret(value: str) -> str:
+    if not value or not value.startswith("enc:"):
+        return value or ""
+    from cryptography.fernet import Fernet
+
+    key = (ROOT / "data" / ".app_key").read_bytes()
+    return Fernet(key).decrypt(value[len("enc:") :].encode("ascii")).decode("utf-8")
+
+
+def db() -> sqlite3.Connection:
+    con = sqlite3.connect(DB)
+    con.row_factory = sqlite3.Row
+    return con
+
+
+def deepseek_endpoint(con: sqlite3.Connection, endpoint_id: str | None = None, model: str | None = None) -> dict[str, str]:
+    if endpoint_id:
+        row = con.execute(
+            """
+            SELECT id, name, base_url, api_key, cached_models
+            FROM model_endpoints
+            WHERE id = ?
+              AND COALESCE(api_key, '') != ''
+            """,
+            (endpoint_id,),
+        ).fetchone()
+    else:
+        row = con.execute(
+        """
+        SELECT id, name, base_url, api_key, cached_models
+        FROM model_endpoints
+        WHERE is_enabled = 1
+          AND COALESCE(api_key, '') != ''
+          AND (lower(name) LIKE '%deepseek%' OR lower(id) LIKE '%deepseek%')
+        ORDER BY CASE WHEN lower(name) = 'deepseek' THEN 0 ELSE 1 END
+        LIMIT 1
+        """
+    ).fetchone()
+    if row is None:
+        raise RuntimeError("No enabled DeepSeek endpoint with an API key found in model_endpoints")
+    models = json.loads(row["cached_models"] or "[]")
+    selected = model or (models[0] if models else "deepseek-chat")
+    return {
+        "id": row["id"],
+        "name": row["name"],
+        "base_url": row["base_url"],
+        "api_key": decrypt_secret(row["api_key"] or ""),
+        "model": selected,
+    }
+
+
+def compact_tool_event(ev: dict[str, Any]) -> dict[str, Any]:
+    out = str(ev.get("output") or "")
+    return {
+        "tool": ev.get("tool"),
+        "command": ev.get("command"),
+        "output": out[:1200] + ("..." if len(out) > 1200 else ""),
+        "exit_code": ev.get("exit_code"),
+    }
+
+
+def session_payload(con: sqlite3.Connection, sid: str) -> dict[str, Any]:
+    s = con.execute(
+        "SELECT id, name, created_at, updated_at, message_count FROM sessions WHERE id = ?",
+        (sid,),
+    ).fetchone()
+    messages = []
+    for m in con.execute(
+        "SELECT role, content, metadata, timestamp FROM chat_messages WHERE session_id = ? ORDER BY timestamp, id",
+        (sid,),
+    ):
+        meta: dict[str, Any] = {}
+        if m["metadata"]:
+            try:
+                meta = json.loads(m["metadata"])
+            except json.JSONDecodeError:
+                meta = {}
+        content = m["content"] or ""
+        thinking = meta.get("thinking")
+        if isinstance(thinking, str) and len(thinking) > 1000:
+            thinking = thinking[:1000] + "..."
+        messages.append(
+            {
+                "role": m["role"],
+                "timestamp": m["timestamp"],
+                "content": content[:2500] + ("..." if len(content) > 2500 else ""),
+                "thinking": thinking,
+                "tool_events": [compact_tool_event(ev) for ev in meta.get("tool_events") or []],
+            }
+        )
+    docs = []
+    for d in con.execute(
+        """
+        SELECT id, title, language, current_content, source_email_uid, updated_at
+        FROM documents
+        WHERE session_id = ?
+        ORDER BY updated_at DESC
+        LIMIT 3
+        """,
+        (sid,),
+    ):
+        content = d["current_content"] or ""
+        docs.append(
+            {
+                "id": d["id"],
+                "title": d["title"],
+                "language": d["language"],
+                "source_email_uid": d["source_email_uid"],
+                "content": content[:1800] + ("..." if len(content) > 1800 else ""),
+            }
+        )
+    return {
+        "session": dict(s),
+        "messages": messages,
+        "open_documents": docs,
+    }
+
+
+def recent_email_sessions(con: sqlite3.Connection, owner: str, limit: int) -> list[str]:
+    rows = con.execute(
+        """
+        SELECT id
+        FROM sessions
+        WHERE owner = ?
+        ORDER BY updated_at DESC
+        LIMIT ?
+        """,
+        (owner, limit),
+    ).fetchall()
+    keep = []
+    for row in rows:
+        text = "\n".join(
+            r["content"] or ""
+            for r in con.execute("SELECT content FROM chat_messages WHERE session_id = ?", (row["id"],))
+        )
+        tools = "\n".join(
+            r["metadata"] or ""
+            for r in con.execute("SELECT metadata FROM chat_messages WHERE session_id = ?", (row["id"],))
+        )
+        if EMAIL_RE.search(text) or "mcp__email" in tools or "list_email" in tools:
+            keep.append(row["id"])
+    return keep
+
+
+def session_ids_from_results(path: Path) -> list[str]:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    rows = payload.get("results") if isinstance(payload, dict) else payload
+    if not isinstance(rows, list):
+        raise RuntimeError(f"Expected results list in {path}")
+    out: list[str] = []
+    for row in rows:
+        sid = str(row.get("session_id") or "").strip()
+        if sid and sid not in out:
+            out.append(sid)
+    return out
+
+
+def judge_prompt(batch: list[dict[str, Any]]) -> list[dict[str, str]]:
+    system = """You are auditing Odysseus email-agent conversations for SFT training quality.
+Return strict JSON only: {"results":[...]}.
+For every session, decide and copy back `session_id` and `session_name` from `session`.
+- verdict: keep, repair, or delete.
+- trainable_score: 0-100.
+- issues: short strings.
+- repairs: concrete edits needed, or [].
+- date_risk: none, low, medium, high.
+- thinking_trace_risk: none, low, medium, high.
+- rationale: one concise sentence.
+
+Important audit rules:
+- Keep only traces where user intent, tool calls, tool outputs, and final answer align.
+- Repair/delete if assistant claimed an email action without a corresponding tool event.
+- Repair/delete if it says tools are unavailable when email tools were actually needed/available.
+- Repair/delete repeated resend/stale-loop traces unless the bad branch is removed.
+- Repair/delete visible raw harness dumps, unpolished tool output, or synthetic/fake/SFT leaks in assistant/user message `content`.
+- Do not penalize raw text inside `tool_events.output` by itself. Tool outputs are allowed to be raw; only flag them when the assistant-facing final content also exposed the dump or when the tool result is semantically wrong.
+- Date-relative tasks are safe only if the trace includes a clear current date/timezone context or a tool query using explicit date bounds. Otherwise flag date_risk.
+- Thinking traces are usable only if they reflect correct tool choice and do not mention fake fixtures, harness bugs, stale injected data, or false tool unavailability.
+- Multi-intent user requests must satisfy all parts or be repair/delete.
+- Be strict: these are for training a model, not UI QA."""
+    user = json.dumps({"current_date": "2026-08-24", "timezone": "UTC", "sessions": batch}, ensure_ascii=False)
+    return [{"role": "system", "content": system}, {"role": "user", "content": user}]
+
+
+def call_judge(endpoint: dict[str, str], batch: list[dict[str, Any]]) -> dict[str, Any]:
+    payload = {
+        "model": endpoint["model"],
+        "messages": judge_prompt(batch),
+        "temperature": 0,
+        "max_tokens": 3500,
+        "response_format": {"type": "json_object"},
+    }
+    req = urllib.request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {endpoint['api_key']}",
+        },
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=75) as resp:
+        data = json.loads(resp.read().decode("utf-8"))
+    content = data["choices"][0]["message"]["content"]
+    if not isinstance(content, str) or not content.strip():
+        raise ValueError("Judge returned empty message content")
+    return json.loads(content)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--owner", default="sft_alex_creator")
+    ap.add_argument("--limit", type=int, default=140)
+    ap.add_argument("--batch-size", type=int, default=5)
+    ap.add_argument("--sleep", type=float, default=0.4)
+    ap.add_argument("--endpoint-id")
+    ap.add_argument("--model")
+    ap.add_argument("--results-file", type=Path, default=None, help="Audit exact session_ids from an overseer/eval actual_results.json")
+    args = ap.parse_args()
+
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+    con = db()
+    endpoint = deepseek_endpoint(con, endpoint_id=args.endpoint_id, model=args.model)
+    if args.results_file:
+        sids = session_ids_from_results(args.results_file)
+    else:
+        sids = recent_email_sessions(con, args.owner, args.limit)
+    stamp = time.strftime("%Y%m%d_%H%M%S")
+    out_jsonl = OUT_DIR / f"email_sft_deepseek_audit_{args.owner}_{stamp}.jsonl"
+    out_md = OUT_DIR / f"email_sft_deepseek_audit_{args.owner}_{stamp}.md"
+
+    all_results: list[dict[str, Any]] = []
+    for i in range(0, len(sids), args.batch_size):
+        batch_sids = sids[i : i + args.batch_size]
+        batch = [session_payload(con, sid) for sid in batch_sids]
+        for attempt in range(3):
+            try:
+                judged = call_judge(endpoint, batch)
+                break
+            except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:
+                if attempt == 2:
+                    raise
+                time.sleep(2 + attempt * 3)
+        results = judged.get("results", [])
+        for j, result in enumerate(results):
+            if j < len(batch):
+                result.setdefault("session_id", batch[j]["session"]["id"])
+                result.setdefault("session_name", batch[j]["session"]["name"])
+        with out_jsonl.open("a", encoding="utf-8") as f:
+            for result in results:
+                f.write(json.dumps(result, ensure_ascii=False) + "\n")
+        all_results.extend(results)
+        print(f"judged {min(i + args.batch_size, len(sids))}/{len(sids)}")
+        time.sleep(args.sleep)
+
+    counts: dict[str, int] = {}
+    for r in all_results:
+        counts[r.get("verdict", "unknown")] = counts.get(r.get("verdict", "unknown"), 0) + 1
+
+    lines = [
+        f"# Email SFT DeepSeek Audit: {args.owner}",
+        "",
+        f"- Sessions judged: {len(all_results)}",
+        f"- Source recent limit: {args.limit}",
+        f"- Endpoint: {endpoint.get('name')} ({endpoint.get('id')})",
+        f"- Model: {endpoint['model']}",
+        f"- Verdict counts: {json.dumps(counts, sort_keys=True)}",
+        "",
+        "## Repair/Delete Queue",
+        "",
+    ]
+    for r in all_results:
+        if r.get("verdict") == "keep":
+            continue
+        sid = r.get("session_id") or r.get("id") or r.get("session", {}).get("id")
+        name = r.get("session_name") or r.get("name") or ""
+        issues = ", ".join(r.get("issues") or [])
+        repairs = "; ".join(
+            item if isinstance(item, str) else json.dumps(item, ensure_ascii=False, sort_keys=True)
+            for item in (r.get("repairs") or [])
+        )
+        lines.append(f"- `{sid}` {name} -- **{r.get('verdict')}** score={r.get('trainable_score')} issues={issues} repairs={repairs}")
+    lines.extend(["", "## Keep Candidates", ""])
+    for r in all_results:
+        if r.get("verdict") != "keep":
+            continue
+        sid = r.get("session_id") or r.get("id") or r.get("session", {}).get("id")
+        name = r.get("session_name") or r.get("name") or ""
+        lines.append(f"- `{sid}` {name} -- score={r.get('trainable_score')} date={r.get('date_risk')} thinking={r.get('thinking_trace_risk')}")
+    out_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print(f"jsonl={out_jsonl}")
+    print(f"markdown={out_md}")
+
+
+if __name__ == "__main__":
+    main()

scripts/audit_historical_tool_routing.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Audit current capability routing against recorded historical tool turns.
+
+This is intentionally read-only: it never creates sessions or executes tools.
+Recorded assistant tool events provide the expected families; the current turn
+contract is evaluated with the original preceding conversation as history.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+from collections import Counter
+from pathlib import Path
+
+from src.turn_contract import FAMILY_TOOLS, canonical_tool, requested_capabilities
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_DB = Path("/home/pewds/odysseus-cookbook-fresh/data/app.db")
+DEFAULT_ANCHOR = "a37dcb3b-6864-4266-a115-f9e87aafd0eb"
+
+
+def tool_family(tool: str, command: object) -> set[str]:
+    name = canonical_tool(tool)
+    families = {family for family, tools in FAMILY_TOOLS.items() if name in tools}
+    # ui_control is a rendering/action bridge. Its command identifies the
+    # product family; do not label every such turn as the generic UI family.
+    if name == "ui_control":
+        text = str(command or "").lower()
+        if "email" in text:
+            return {"email"}
+        if "calendar" in text or "event" in text:
+            return {"calendar"}
+        if "note" in text:
+            return {"notes"}
+        if "document" in text or "editor" in text:
+            return {"documents"}
+    return families
+
+
+def metadata_tools(raw: str | None) -> set[str]:
+    try:
+        metadata = json.loads(raw or "{}")
+    except (TypeError, json.JSONDecodeError):
+        return set()
+    expected: set[str] = set()
+    for event in metadata.get("tool_events") or []:
+        expected.update(tool_family(event.get("tool", ""), event.get("command")))
+    return expected
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--db", type=Path, default=DEFAULT_DB)
+    parser.add_argument("--anchor", default=DEFAULT_ANCHOR)
+    parser.add_argument("--owner", default="sft_alex_creator")
+    parser.add_argument("--out", type=Path, default=ROOT / "reports/historical-routing-audit.json")
+    args = parser.parse_args()
+
+    con = sqlite3.connect(args.db)
+    con.row_factory = sqlite3.Row
+    anchor = con.execute("SELECT created_at FROM sessions WHERE id = ?", (args.anchor,)).fetchone()
+    if anchor is None:
+        raise SystemExit(f"Anchor session not found: {args.anchor}")
+    sessions = con.execute(
+        "SELECT id, name, created_at FROM sessions WHERE owner = ? AND created_at >= ? "
+        "ORDER BY created_at, id", (args.owner, anchor[0])
+    ).fetchall()
+
+    rows: list[dict] = []
+    seen: set[tuple] = set()
+    for session in sessions:
+        messages = con.execute(
+            "SELECT id, role, content, metadata, timestamp FROM chat_messages "
+            "WHERE session_id = ? ORDER BY timestamp, id", (session["id"],)
+        ).fetchall()
+        history: list[dict[str, str]] = []
+        for index, message in enumerate(messages):
+            role, content = message["role"], message["content"]
+            if role != "user":
+                history.append({"role": role, "content": content})
+                continue
+            following = next((m for m in messages[index + 1:] if m["role"] == "assistant"), None)
+            expected = metadata_tools(following["metadata"] if following else None)
+            if not expected:
+                history.append({"role": role, "content": content})
+                continue
+            key = (tuple((h["role"], h["content"].strip().lower()) for h in history), content.strip().lower(), tuple(sorted(expected)))
+            if key in seen:
+                history.append({"role": role, "content": content})
+                continue
+            seen.add(key)
+            actual = set(requested_capabilities(content, history))
+            missing = expected - actual
+            rows.append({
+                "session_id": session["id"], "session_name": session["name"],
+                "message_id": message["id"], "prompt": content,
+                "expected": sorted(expected), "actual": sorted(actual),
+                "missing": sorted(missing), "passed": not missing,
+            })
+            history.append({"role": role, "content": content})
+
+    failures = [row for row in rows if not row["passed"]]
+    report = {
+        "source_db": str(args.db), "anchor": args.anchor, "owner": args.owner,
+        "sessions_scanned": len(sessions), "labeled_unique_turns": len(rows),
+        "passed": len(rows) - len(failures), "failed": len(failures),
+        "accuracy": round((len(rows) - len(failures)) / len(rows), 6) if rows else None,
+        "missing_family_counts": dict(sorted(Counter(f for row in failures for f in row["missing"]).items())),
+        "failures": failures,
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    print(json.dumps({key: report[key] for key in ("sessions_scanned", "labeled_unique_turns", "passed", "failed", "accuracy", "missing_family_counts")}, indent=2))
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/audit_search_pipeline.py

@@ -0,0 +1,53 @@
+"""Read-only, reproducible provider probe. Prints JSON; never changes settings.
+
+Run with the application's Python from the repository root. Queries are public
+regressions plus unrelated controls. Coverage is diagnostic, not an accuracy score.
+"""
+import concurrent.futures
+import json
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import httpx
+from services.search.providers import _get_search_instance, _safesearch_for
+
+QUERIES = [
+    "What country has best meat",
+    "Sweden 78 year old British woman deportation Brexit residence application",
+    "Latest news in AI",
+    "Any latest info on quantum physics",
+    "What year did Ethiopia become independent",
+    "PostgreSQL transaction isolation documentation",
+    "Kyoto weather tomorrow",
+]
+ENGINES = ["bing", "mojeek", "presearch", "duckduckgo", "google", "bing news", "yep"]
+
+
+def probe(pair):
+    query, engine = pair
+    start = time.monotonic()
+    try:
+        response = httpx.get(
+            _get_search_instance() + "/search",
+            params={"q": query, "engines": engine, "format": "json",
+                    "language": "en", "safesearch": _safesearch_for("searxng")},
+            timeout=20,
+        )
+        response.raise_for_status()
+        data = response.json()
+        return {"query": query, "engine": engine,
+                "seconds": round(time.monotonic() - start, 2),
+                "unresponsive": data.get("unresponsive_engines", []),
+                "results": [{k: row.get(k) for k in (
+                    "title", "url", "content", "engines", "publishedDate"
+                )} for row in data.get("results", [])[:5]]}
+    except Exception as exc:
+        return {"query": query, "engine": engine, "error": type(exc).__name__}
+
+
+if __name__ == "__main__":
+    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
+        rows = list(pool.map(probe, [(q, e) for q in QUERIES for e in ENGINES]))
+    print(json.dumps(rows, ensure_ascii=False, indent=2))

scripts/audit_sft_corpus_with_deepseek.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python3
+"""Audit an Odysseus SFT JSONL corpus and use DeepSeek for semantic review."""
+
+from __future__ import annotations
+
+import argparse
+import collections
+import concurrent.futures
+import hashlib
+import json
+import random
+import re
+import sqlite3
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_TRACE = ROOT / "data" / "sft_traces" / "sft_alex_creator.jsonl"
+OUT_DIR = ROOT / "data" / "audits"
+
+LEAK_RE = re.compile(
+    r"fake-(?:sender|odysseus)|synthetic (?:sft|fixture)|safe for training|"
+    r"training traces?|you are a fish|prompt injection|harness (?:bug|issue|dump)",
+    re.I,
+)
+UNAVAILABLE_RE = re.compile(
+    r"(?:i (?:do not|don.t|cannot|can.t)|there(?: is|'s) no) .{0,55}"
+    r"(?:tool|access|email|calendar|memory|document|browser|shell)",
+    re.I,
+)
+RAW_DUMP_RE = re.compile(r"Here are your (?:emails|events) \(\d+\):", re.I)
+FAILURE_RE = re.compile(
+    r"(?:permission denied|requires? .{0,30}(?:dependency|package)|not configured|"
+    r"tool calls? failed|internal server error|traceback|timed out)",
+    re.I,
+)
+
+
+def decrypt_secret(value: str) -> str:
+    if not value or not value.startswith("enc:"):
+        return value or ""
+    from cryptography.fernet import Fernet
+
+    key = (ROOT / "data" / ".app_key").read_bytes()
+    return Fernet(key).decrypt(value[4:].encode("ascii")).decode("utf-8")
+
+
+def deepseek_endpoint(endpoint_id: str | None, model: str | None) -> dict[str, str]:
+    con = sqlite3.connect(ROOT / "data" / "app.db")
+    con.row_factory = sqlite3.Row
+    if endpoint_id:
+        row = con.execute(
+            "SELECT * FROM model_endpoints WHERE id=? AND COALESCE(api_key,'') != ''",
+            (endpoint_id,),
+        ).fetchone()
+    else:
+        row = con.execute(
+            """SELECT * FROM model_endpoints
+               WHERE is_enabled=1 AND COALESCE(api_key,'') != ''
+                 AND (lower(name) LIKE '%deepseek%' OR lower(id) LIKE '%deepseek%')
+               ORDER BY CASE WHEN lower(name)='deepseek' THEN 0 ELSE 1 END LIMIT 1"""
+        ).fetchone()
+    if row is None:
+        raise RuntimeError("No enabled DeepSeek endpoint with an API key")
+    models = json.loads(row["cached_models"] or "[]")
+    return {
+        "id": row["id"],
+        "name": row["name"],
+        "base_url": row["base_url"],
+        "api_key": decrypt_secret(row["api_key"]),
+        "model": model or (models[0] if models else "deepseek-chat"),
+    }
+
+
+def load_rows(path: Path) -> list[dict[str, Any]]:
+    rows = []
+    for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        if not line.strip():
+            continue
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError as exc:
+            rows.append({"_invalid_line": line_no, "_error": str(exc), "_raw": line[:500]})
+            continue
+        row["_line"] = line_no
+        rows.append(row)
+    return rows
+
+
+def live_session_ids(owner: str) -> set[str]:
+    con = sqlite3.connect(ROOT / "data" / "app.db")
+    try:
+        return {str(row[0]) for row in con.execute("SELECT id FROM sessions WHERE owner = ?", (owner,))}
+    finally:
+        con.close()
+
+
+def row_flags(row: dict[str, Any]) -> list[str]:
+    if "_invalid_line" in row:
+        return ["invalid_json"]
+    flags = []
+    user = str(row.get("user") or "")
+    assistant = str(row.get("assistant") or "")
+    thinking = str(row.get("thinking") or "")
+    visible = "\n".join((user, assistant, thinking))
+    events = row.get("tool_events") or []
+    if not user.strip() or not assistant.strip():
+        flags.append("missing_user_or_assistant")
+    if LEAK_RE.search(visible):
+        flags.append("fixture_or_harness_leak")
+    if UNAVAILABLE_RE.search(assistant):
+        flags.append("possible_false_tool_unavailability")
+    if RAW_DUMP_RE.search(assistant):
+        flags.append("raw_harness_style_answer")
+    if any(FAILURE_RE.search(str(ev.get("output") or "")) for ev in events):
+        flags.append("tool_failure_present")
+    if events and not assistant.strip():
+        flags.append("tool_call_without_final_answer")
+    if len(row.get("round_texts") or []) > 2:
+        nonempty = [str(x).strip() for x in row.get("round_texts") or [] if str(x).strip()]
+        if len(nonempty) > 1 and len(set(nonempty)) < len(nonempty):
+            flags.append("repeated_round_text")
+    return flags
+
+
+def compact_row(row: dict[str, Any]) -> dict[str, Any]:
+    def clip(value: Any, size: int) -> str:
+        text = str(value or "")
+        return text[:size] + ("..." if len(text) > size else "")
+
+    return {
+        "line": row.get("_line"),
+        "message_id": row.get("message_id"),
+        "user": clip(row.get("user"), 1200),
+        "assistant": clip(row.get("assistant"), 2200),
+        "thinking": clip(row.get("thinking"), 1600),
+        "flags": row_flags(row),
+        "tools": [
+            {
+                "tool": ev.get("tool"),
+                "command": clip(ev.get("command"), 700),
+                "output": clip(ev.get("output"), 1100),
+                "exit_code": ev.get("exit_code"),
+            }
+            for ev in (row.get("tool_events") or [])
+        ],
+    }
+
+
+def _parse_json_message(message: dict[str, Any]) -> dict[str, Any]:
+    content = str(message.get("content") or message.get("reasoning_content") or "").strip()
+    content = re.sub(r"^```(?:json)?\s*|\s*```$", "", content, flags=re.I | re.S).strip()
+    if not content.startswith("{"):
+        match = re.search(r"\{.*\}", content, flags=re.S)
+        if match:
+            content = match.group(0)
+    if not content:
+        raise ValueError("DeepSeek returned empty content and reasoning_content")
+    return json.loads(content)
+
+
+def judge(endpoint: dict[str, str], sessions: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    system = """You are a strict SFT corpus auditor for a general tool-using agent.
+Return JSON only as {"results":[...]}. Return exactly one result per session.
+Each result: session_id, verdict (keep|repair|delete), score (0-100), issues (strings), repairs (specific strings), and coverage_notes.
+
+Judge the complete behavior and whether the response is a good speaking-style target. Keep only when intent, reasoning, tool selection, arguments, tool outputs, state changes, follow-ups, and final answers agree, and the visible answer is concise, natural, and synthesized for the user. Repair means a coherent trace can be fixed by removing/replacing specific turns or text. Delete means the trajectory teaches a materially wrong strategy or is too corrupted.
+
+Flag false tool-unavailability claims, repeated answers/turns, stale resend branches, missing requested actions, success claims without successful tool evidence, malformed tool arguments, raw harness dumps presented as the answer, fixture/SFT/harness/prompt-injection discussion, incorrect relative dates/timezones, unsafe destructive actions, needless tools, tool loops, and thinking that contradicts the final action. Also mark repair when the final answer mechanically echoes tool output, repeats metadata the user did not request, narrates internal routing, asks needless follow-up questions, or is substantially more verbose than needed. A failed tool call is acceptable only when the assistant handles it correctly and does not teach a bad workaround. Do not penalize raw formatting that exists only inside tool output. For multi-intent prompts, every requested part must be handled. Be conservative because these traces train both tool strategy and response style."""
+    payload = {
+        "model": endpoint["model"],
+        "messages": [
+            {"role": "system", "content": system},
+            {"role": "user", "content": json.dumps({"audit_date": "2026-08-30", "timezone": "UTC", "sessions": sessions}, ensure_ascii=False)},
+        ],
+        "temperature": 0,
+        "max_tokens": 12000,
+        "response_format": {"type": "json_object"},
+    }
+    req = urllib.request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode(),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=120) as response:
+        result = json.loads(response.read().decode())
+    results = _parse_json_message(result["choices"][0]["message"])["results"]
+    expected_ids = [str(session.get("session_id") or "") for session in sessions]
+    actual_ids = [str(item.get("session_id") or "") for item in results]
+    if len(results) != len(sessions) or sorted(actual_ids) != sorted(expected_ids):
+        raise ValueError(
+            f"DeepSeek verdict IDs do not match batch: expected={expected_ids!r} actual={actual_ids!r}"
+        )
+    by_id = {str(item["session_id"]): item for item in results}
+    return [by_id[session_id] for session_id in expected_ids]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--trace", type=Path, default=DEFAULT_TRACE)
+    parser.add_argument("--live-owner", help="Only audit traced sessions still present in app.db for this owner")
+    parser.add_argument("--endpoint-id")
+    parser.add_argument("--model", default="deepseek-v4-flash")
+    parser.add_argument("--sample-per-tool", type=int, default=2)
+    parser.add_argument("--max-sessions", type=int, default=260)
+    parser.add_argument("--all-sessions", action="store_true", help="Semantically review every session in scope")
+    parser.add_argument("--exclude-verdicts", type=Path, help="Skip session IDs already present in this verdict JSONL")
+    parser.add_argument("--batch-size", type=int, default=4)
+    parser.add_argument("--workers", type=int, default=6)
+    parser.add_argument("--seed", type=int, default=17)
+    parser.add_argument("--skip-deepseek", action="store_true")
+    args = parser.parse_args()
+
+    rows = load_rows(args.trace)
+    if args.live_owner:
+        live_ids = live_session_ids(args.live_owner)
+        rows = [row for row in rows if str(row.get("session_id") or "") in live_ids]
+    sessions: dict[str, list[dict[str, Any]]] = collections.defaultdict(list)
+    tools: collections.Counter[str] = collections.Counter()
+    models: collections.Counter[str] = collections.Counter()
+    flag_counts: collections.Counter[str] = collections.Counter()
+    duplicate_ids: collections.Counter[str] = collections.Counter()
+    content_hashes: collections.defaultdict[str, list[dict[str, Any]]] = collections.defaultdict(list)
+    for row in rows:
+        sid = str(row.get("session_id") or f"invalid-line-{row.get('_invalid_line')}")
+        sessions[sid].append(row)
+        models[str((row.get("metadata") or {}).get("model") or "unknown")] += 1
+        duplicate_ids[str(row.get("message_id") or "missing")] += 1
+        digest = hashlib.sha256(json.dumps([row.get("user"), row.get("assistant"), row.get("tool_events")], sort_keys=True, default=str).encode()).hexdigest()
+        content_hashes[digest].append(row)
+        for flag in row_flags(row):
+            flag_counts[flag] += 1
+        for event in row.get("tool_events") or []:
+            tools[str(event.get("tool") or "unknown")] += 1
+
+    suspicious = {sid for sid, turns in sessions.items() if any(row_flags(row) for row in turns)}
+    by_tool: dict[str, list[str]] = collections.defaultdict(list)
+    for sid, turns in sessions.items():
+        for tool in {str(e.get("tool")) for row in turns for e in row.get("tool_events") or [] if e.get("tool")}:
+            by_tool[tool].append(sid)
+    rng = random.Random(args.seed)
+    if args.all_sessions:
+        selected = set(sessions)
+    else:
+        selected = set(suspicious)
+        for tool, candidates in sorted(by_tool.items()):
+            pool = sorted(set(candidates) - selected)
+            selected.update(rng.sample(pool, min(args.sample_per_tool, len(pool))))
+        selected = set(sorted(selected)[: args.max_sessions])
+    if args.exclude_verdicts:
+        reviewed = {
+            str(json.loads(line).get("session_id") or "")
+            for line in args.exclude_verdicts.read_text(encoding="utf-8").splitlines()
+            if line.strip()
+        }
+        selected.difference_update(reviewed)
+
+    stamp = time.strftime("%Y%m%d_%H%M%S")
+    out = OUT_DIR / f"sft_corpus_deepseek_audit_{stamp}"
+    out.mkdir(parents=True, exist_ok=True)
+    deterministic = {
+        "trace": str(args.trace),
+        "live_owner": args.live_owner,
+        "turns": len(rows),
+        "sessions": len(sessions),
+        "tool_counts": dict(tools.most_common()),
+        "model_counts": dict(models.most_common()),
+        "flag_counts": dict(flag_counts.most_common()),
+        "suspicious_sessions": len(suspicious),
+        "duplicate_message_ids": {k: v for k, v in duplicate_ids.items() if v > 1},
+        "exact_duplicate_rows": sum(len(v) - 1 for v in content_hashes.values() if len(v) > 1),
+        "deepseek_selected_sessions": len(selected),
+        "all_sessions": args.all_sessions,
+        "excluded_verdicts": str(args.exclude_verdicts) if args.exclude_verdicts else None,
+    }
+    (out / "coverage.json").write_text(json.dumps(deterministic, indent=2), encoding="utf-8")
+    with (out / "deterministic_repair_queue.jsonl").open("w", encoding="utf-8") as handle:
+        for sid in sorted(suspicious):
+            handle.write(json.dumps({"session_id": sid, "flags": sorted({f for r in sessions[sid] for f in row_flags(r)}), "lines": [r.get("_line") for r in sessions[sid]]}) + "\n")
+
+    judged: list[dict[str, Any]] = []
+    if not args.skip_deepseek:
+        endpoint = deepseek_endpoint(args.endpoint_id, args.model)
+        chosen = sorted(selected)
+        batches = []
+        for start in range(0, len(chosen), args.batch_size):
+            ids = chosen[start : start + args.batch_size]
+            batch = [{"session_id": sid, "name": sessions[sid][0].get("session_name"), "turns": [compact_row(r) for r in sessions[sid]]} for sid in ids]
+            batches.append((start, batch))
+
+        def run_batch(item: tuple[int, list[dict[str, Any]]]) -> tuple[int, list[dict[str, Any]]]:
+            start, batch = item
+            for attempt in range(3):
+                try:
+                    results = judge(endpoint, batch)
+                    return start, results
+                except (urllib.error.URLError, TimeoutError, KeyError, ValueError, json.JSONDecodeError) as exc:
+                    if attempt == 2:
+                        raise RuntimeError(f"DeepSeek batch failed at {start}: {exc}") from exc
+                    time.sleep(3 + attempt * 4)
+            raise AssertionError("unreachable")
+
+        completed = 0
+        ordered: dict[int, list[dict[str, Any]]] = {}
+        with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool:
+            futures = [pool.submit(run_batch, item) for item in batches]
+            for future in concurrent.futures.as_completed(futures):
+                start, results = future.result()
+                ordered[start] = results
+                completed += len(results)
+                print(f"deepseek {completed}/{len(chosen)}", flush=True)
+        for start in sorted(ordered):
+            judged.extend(ordered[start])
+        with (out / "deepseek_verdicts.jsonl").open("w", encoding="utf-8") as handle:
+            for result in judged:
+                handle.write(json.dumps(result, ensure_ascii=False) + "\n")
+
+    verdicts = collections.Counter(str(row.get("verdict") or "unknown") for row in judged)
+    report = [
+        "# SFT Corpus Audit", "",
+        f"- Trace: `{args.trace}`", f"- Turns: {len(rows)}", f"- Sessions: {len(sessions)}",
+        f"- Tools represented: {len(tools)}", f"- Suspicious sessions (deterministic): {len(suspicious)}",
+        f"- Exact duplicate rows: {deterministic['exact_duplicate_rows']}",
+        f"- DeepSeek sessions reviewed: {len(judged)}", f"- DeepSeek verdicts: `{dict(verdicts)}`", "",
+        "## Deterministic Flags", "",
+    ]
+    report.extend(f"- {name}: {count}" for name, count in flag_counts.most_common())
+    report.extend(["", "## Lowest-Coverage Tools", ""])
+    report.extend(f"- `{tool}`: {count}" for tool, count in sorted(tools.items(), key=lambda x: (x[1], x[0]))[:20])
+    report.extend(["", "## DeepSeek Repair/Delete Queue", ""])
+    for row in judged:
+        if row.get("verdict") == "keep":
+            continue
+        report.append(f"- `{row.get('session_id')}` **{row.get('verdict')}** score={row.get('score')}: {'; '.join(row.get('issues') or [])}")
+    (out / "report.md").write_text("\n".join(report) + "\n", encoding="utf-8")
+    print(f"output={out}")
+
+
+if __name__ == "__main__":
+    main()

scripts/audit_typo_tool_routing.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Build and score deterministic typo variants of real labeled tool prompts."""
+
+from __future__ import annotations
+
+import argparse, hashlib, json, re, sqlite3
+from collections import Counter
+from pathlib import Path
+
+from src.turn_contract import requested_capabilities
+
+DB = Path("/home/pewds/odysseus-cookbook-fresh/data/app.db")
+ANCHOR = "a37dcb3b-6864-4266-a115-f9e87aafd0eb"
+TRIGGERS = {
+    "calendar": ("calendar", "event", "meeting", "appointment", "agenda"),
+    "notes": ("note", "notes", "checklist", "groceries"),
+    "tasks": ("task", "tasks", "todo", "reminder"),
+    "skills": ("skill", "skills"),
+    "memory": ("memory", "memories", "remember", "forget"),
+    "documents": ("document", "documents", "doc", "editor"),
+    "email": ("email", "emails", "inbox", "mail", "spam"),
+    "search_browser": ("search", "web", "browse", "browser", "website", "youtube"),
+    "shell_files": ("file", "files", "folder", "directory", "shell", "terminal", "workspace", "bash", "python"),
+    "cookbook_admin": ("cookbook", "endpoint", "model", "server", "download", "settings"),
+}
+TOOL_FAMILY = {
+    "manage_calendar": "calendar", "manage_notes": "notes", "manage_tasks": "tasks",
+    "manage_skills": "skills", "manage_memory": "memory", "search_chats": "memory",
+    "manage_documents": "documents", "create_document": "documents", "edit_document": "documents",
+    "update_document": "documents", "suggest_document": "documents",
+    "list_email_accounts": "email", "list_emails": "email", "search_emails": "email",
+    "read_email": "email", "send_email": "email", "reply_to_email": "email", "draft_email": "email",
+    "web_search": "search_browser", "web_fetch": "search_browser", "private_browser": "search_browser",
+    "youtube_tool": "search_browser", "search_hf_models": "search_browser",
+    "bash": "shell_files", "python": "shell_files", "read_file": "shell_files", "write_file": "shell_files",
+    "list_models": "cookbook_admin", "list_served_models": "cookbook_admin", "serve_model": "cookbook_admin",
+    "stop_served_model": "cookbook_admin", "list_cookbook_servers": "cookbook_admin", "manage_endpoints": "cookbook_admin",
+}
+NEIGHBOR = {"a":"s","e":"r","i":"o","o":"p","s":"d","t":"y","r":"t","l":"k","n":"m","m":"n","d":"f","c":"v","b":"n","w":"e","f":"g","g":"h","h":"j","p":"o","k":"l","v":"b","u":"i"}
+
+def variants(word: str) -> list[tuple[str,str]]:
+    i = max(1, min(len(word)-2, len(word)//2))
+    out = [("delete", word[:i]+word[i+1:]), ("duplicate", word[:i]+word[i]+word[i:])]
+    if i+1 < len(word): out.append(("transpose", word[:i]+word[i+1]+word[i]+word[i+2:]))
+    repl = NEIGHBOR.get(word[i].lower(), "x")
+    out.append(("neighbor", word[:i]+repl+word[i+1:]))
+    if len(word) >= 6: out.append(("split", word[:i]+" "+word[i:]))
+    return out
+
+def expected_family(metadata: str | None) -> str | None:
+    try: events = json.loads(metadata or "{}").get("tool_events") or []
+    except json.JSONDecodeError: return None
+    families = []
+    for event in events:
+        tool = str(event.get("tool") or "").rsplit("__",1)[-1]
+        if TOOL_FAMILY.get(tool): families.append(TOOL_FAMILY[tool])
+    return families[0] if families and len(set(families)) == 1 else None
+
+def main() -> int:
+    ap=argparse.ArgumentParser(); ap.add_argument("--db",type=Path,default=DB); ap.add_argument("--out",type=Path,required=True); ap.add_argument("--per-family",type=int,default=20); a=ap.parse_args()
+    con=sqlite3.connect(a.db); con.row_factory=sqlite3.Row
+    t0=con.execute("select created_at from sessions where id=?",(ANCHOR,)).fetchone()[0]
+    sessions=con.execute("select id from sessions where owner='sft_alex_creator' and created_at>=? order by created_at,id",(t0,)).fetchall()
+    seeds={f:[] for f in TRIGGERS}
+    for s in sessions:
+        ms=con.execute("select role,content,metadata from chat_messages where session_id=? order by timestamp,id",(s[0],)).fetchall(); history=[]
+        for i,m in enumerate(ms):
+            if m['role']!='user': history.append({'role':m['role'],'content':m['content']}); continue
+            nxt=next((x for x in ms[i+1:] if x['role']=='assistant'),None); fam=expected_family(nxt['metadata'] if nxt else None)
+            if fam and len(seeds[fam])<a.per_family and fam in requested_capabilities(m['content'],history):
+                hit=next((w for w in TRIGGERS[fam] if re.search(r'\b'+re.escape(w)+r'\b',m['content'],re.I)),None)
+                if hit: seeds[fam].append((s[0],m['content'],tuple(history),hit))
+            history.append({'role':'user','content':m['content']})
+    rows=[]
+    for fam,items in seeds.items():
+        for sid,prompt,history,word in items:
+            for kind,bad in variants(word):
+                changed=re.sub(r'\b'+re.escape(word)+r'\b',bad,prompt,count=1,flags=re.I)
+                actual=sorted(requested_capabilities(changed,history)); digest=hashlib.sha256((sid+changed).encode()).hexdigest()
+                rows.append({'split':'blind' if int(digest[:2],16)<64 else 'dev','family':fam,'source_session':sid,'mutation':kind,'prompt':changed,'actual':actual,'passed':fam in actual})
+    summary = {}
+    for split in ('dev', 'blind'):
+        selected = [row for row in rows if row['split'] == split]
+        passed = sum(row['passed'] for row in selected)
+        exact = sum(row['actual'] == [row['family']] for row in selected)
+        wrong = sum(bool(row['actual']) and row['family'] not in row['actual']
+                    and row['actual'] != ['unknown'] for row in selected)
+        abstained = sum(not row['actual'] or row['actual'] == ['unknown'] for row in selected)
+        summary[split] = {
+            'total': len(selected), 'passed': passed,
+            'accuracy': round(passed / len(selected), 6) if selected else None,
+            'exact': exact, 'exact_accuracy': round(exact / len(selected), 6) if selected else None,
+            'wrong_family': wrong,
+            'wrong_family_rate': round(wrong / len(selected), 6) if selected else None,
+            'abstained': abstained,
+        }
+    report={'summary':summary,'family_failures':dict(Counter(r['family'] for r in rows if not r['passed'])),'rows':rows}
+    a.out.write_text(json.dumps(report,indent=2,ensure_ascii=False)+'\n'); print(json.dumps({'summary':summary,'family_failures':report['family_failures']},indent=2)); return 0
+if __name__=='__main__': raise SystemExit(main())

scripts/build_odysseus_everyday_deepseek_heldout_cases.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from core.database import ModelEndpoint, SessionLocal
+
+DEFAULT_OUT = REPO_ROOT / "data/evals/ody_everyday_deepseek_heldout_v1_20260821/cases.json"
+
+
+FAMILIES: dict[str, dict[str, Any]] = {
+    "notes_create": {
+        "count": 3,
+        "instruction": "Personal note creation requests. The prompt must ask to add/create/save a note with the exact marker as the note title and a short body.",
+        "case": {
+            "kind": "note",
+            "marker": "__MARKER__",
+            "expect_first_tool": "manage_notes",
+            "must_mutate": "note_created",
+        },
+        "default_user": "Add a note titled __MARKER__ saying buy oats after school pickup",
+    },
+    "tasks_recurring": {
+        "count": 3,
+        "instruction": "Recurring reminder/automation requests involving email/search words. The correct behavior is to create a scheduled task, not run the inner action now. Include exact marker as the task name.",
+        "case": {
+            "kind": "task",
+            "marker": "__MARKER__",
+            "expect_first_tool": "manage_tasks",
+            "must_mutate": "task_created",
+        },
+        "default_user": "Every morning at 7:30, remind me to review the latest inbox email. Name it __MARKER__",
+    },
+    "calendar_create": {
+        "count": 2,
+        "instruction": "Calendar create requests for tomorrow at 7pm, with exact marker as title. Keep tomorrow/7pm so the existing state check applies.",
+        "case": {
+            "kind": "calendar",
+            "marker": "__MARKER__",
+            "expect_first_tool": "manage_calendar",
+            "must_mutate": "calendar_created_2026_08_22_19",
+        },
+        "default_user": "Add dinner tomorrow at 7pm titled __MARKER__",
+    },
+    "calendar_move": {
+        "count": 2,
+        "instruction": "Calendar move requests. Ask to move the event with exact marker to 8pm tomorrow.",
+        "case": {
+            "kind": "calendar",
+            "marker": "__MARKER__",
+            "precreate_calendar_event": {
+                "summary": "__MARKER__",
+                "dtstart": "2026-08-22T19:00:00",
+                "dtend": "2026-08-22T20:00:00",
+            },
+            "expect_first_tool": "manage_calendar",
+            "must_mutate": "calendar_moved_2026_08_22_20",
+        },
+        "default_user": "Move my calendar event __MARKER__ to 8pm tomorrow",
+    },
+    "calendar_delete": {
+        "count": 2,
+        "instruction": "Calendar delete requests. Ask to delete/remove/cancel the existing event with exact marker as the name.",
+        "case": {
+            "kind": "calendar",
+            "marker": "__MARKER__",
+            "precreate_calendar_event": {
+                "summary": "__MARKER__",
+                "dtstart": "2026-08-22T13:00:00",
+                "dtend": "2026-08-22T14:00:00",
+            },
+            "expect_first_tool": "manage_calendar",
+            "must_mutate": "calendar_deleted",
+        },
+        "default_user": "Delete the calendar event named __MARKER__",
+    },
+    "email_latest": {
+        "count": 3,
+        "instruction": "Personal inbox/latest email requests. They must clearly refer to the user's own email, not public web search.",
+        "case": {
+            "kind": "email",
+            "expect_first_tool_any": ["mcp__email__list_emails", "list_emails"],
+            "forbidden_tools": ["web_search", "web_fetch"],
+            "must_answer_any": ["From:", "UID", "Booking.com", "latest email"],
+        },
+        "default_user": "What's my latest emails",
+    },
+    "web_synthesis": {
+        "count": 3,
+        "instruction": "Public web lookup requests about why snails bubble/foam. The prompt should require lookup and explanation, not just links.",
+        "case": {
+            "kind": "web",
+            "expect_first_tool": "web_search",
+            "forbidden_repeat_tools": ["web_search"],
+            "must_answer_any": ["mucus", "foam", "bubble"],
+            "must_answer_any_2": ["stress", "irritant", "predator", "moisture", "defense"],
+            "forbidden_final": ["Here are links for that topic", "WEB SEARCH RESULTS", "```sources"],
+        },
+        "default_user": "Look up why snails bubble up sometimes",
+    },
+    "draft_active_email": {
+        "count": 3,
+        "instruction": "Active email compose draft edit requests. The prompt must ask to write/update the open draft and include the phrase '8am works'.",
+        "case": {
+            "kind": "draft",
+            "active_document": {
+                "title": "Everyday email draft probe",
+                "language": "email",
+                "content": (
+                    "To: test@example.com\n"
+                    "Subject: Re: Test manual draft\n"
+                    "In-Reply-To: <manual@example.com>\n"
+                    "References: <manual@example.com>\n"
+                    "X-Source-UID: 999999\n"
+                    "---\n\n"
+                    "---------- Previous message ----------\n"
+                    "Can you confirm the meeting time?\n"
+                ),
+            },
+            "expect_first_tool_any": ["update_document", "edit_document"],
+            "forbidden_tools": ["manage_calendar", "web_search", "mcp__email__list_emails", "mcp__email__read_email"],
+            "must_mutate": "document_contains_8am",
+        },
+        "default_user": "Write a response to it saying 8am works for me",
+    },
+}
+
+
+def deepseek_endpoint() -> dict[str, str]:
+    db = SessionLocal()
+    try:
+        row = (
+            db.query(ModelEndpoint)
+            .filter(ModelEndpoint.name.ilike("%deepseek%"), ModelEndpoint.is_enabled == True)  # noqa: E712
+            .order_by(ModelEndpoint.updated_at.desc())
+            .first()
+        )
+        if row is None or not row.api_key:
+            raise RuntimeError("no enabled DeepSeek endpoint with API key")
+        return {
+            "name": row.name,
+            "base_url": row.base_url,
+            "api_key": row.api_key,
+            "cached_models": row.cached_models or "",
+        }
+    finally:
+        db.close()
+
+
+def call_deepseek(endpoint: dict[str, str], prompt: str) -> dict[str, Any]:
+    model = "deepseek-chat"
+    try:
+        cached = json.loads(endpoint["cached_models"] or "[]")
+        if cached:
+            model = cached[0]
+    except json.JSONDecodeError:
+        pass
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": "Return strict JSON only. No markdown."},
+            {"role": "user", "content": prompt},
+        ],
+        "temperature": 0.7,
+        "max_tokens": 3000,
+    }
+    req = request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=90) as resp:
+        body = json.loads(resp.read().decode("utf-8"))
+    content = body["choices"][0]["message"]["content"]
+    content = re.sub(r"^```(?:json)?\s*|\s*```$", "", content.strip(), flags=re.I | re.S)
+    parsed = json.loads(content)
+    return {"model": model, "content": parsed}
+
+
+def valid_user(family: str, text: Any) -> bool:
+    if not isinstance(text, str):
+        return False
+    lowered = text.lower()
+    if family in {"notes_create", "tasks_recurring", "calendar_create", "calendar_move", "calendar_delete"} and "__MARKER__" not in text:
+        return False
+    if family == "calendar_create" and ("tomorrow" not in lowered or "7" not in lowered):
+        return False
+    if family == "calendar_move" and ("tomorrow" not in lowered or "8" not in lowered):
+        return False
+    if family == "draft_active_email" and "8am works" not in lowered:
+        return False
+    return 6 <= len(text.split()) <= 32
+
+
+def build_cases(generated: dict[str, Any]) -> list[dict[str, Any]]:
+    cases: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for family, spec in FAMILIES.items():
+        prompts = generated.get(family, [])
+        if not isinstance(prompts, list):
+            prompts = []
+        prompts = [item for item in prompts if valid_user(family, item)]
+        prompts.append(spec["default_user"])
+        chosen: list[str] = []
+        for prompt in prompts:
+            key = prompt.lower()
+            if key in seen:
+                continue
+            seen.add(key)
+            chosen.append(prompt)
+            if len(chosen) >= spec["count"]:
+                break
+        while len(chosen) < spec["count"]:
+            chosen.append(spec["default_user"])
+        for idx, user in enumerate(chosen):
+            case = dict(spec["case"])
+            case.update({"id": f"deepseek_{family}_{idx:02d}", "user": user, "deepseek_family": family})
+            cases.append(case)
+    return cases
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+    args = parser.parse_args()
+
+    prompt = {
+        "task": "Generate held-out everyday Odysseus tool-use eval prompts.",
+        "date_context": "Current date is 2026-08-21 Asia/Tokyo; tomorrow is 2026-08-22.",
+        "requirements": [
+            "Return JSON object only.",
+            "Keys must be exactly the family names provided.",
+            "Each value is a list of natural user prompts.",
+            "For marker families, include the literal placeholder __MARKER__ exactly once.",
+            "Do not copy the default prompt; produce paraphrases.",
+            "Keep prompts short and realistic.",
+        ],
+        "families": {name: {"count": spec["count"], "instruction": spec["instruction"], "default": spec["default_user"]} for name, spec in FAMILIES.items()},
+    }
+    endpoint = deepseek_endpoint()
+    started = time.time()
+    response = call_deepseek(endpoint, json.dumps(prompt, ensure_ascii=False))
+    cases = build_cases(response["content"])
+    payload = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "generator": "build_odysseus_everyday_deepseek_heldout_cases.py",
+        "provider": "DeepSeek",
+        "model": response["model"],
+        "elapsed_seconds": round(time.time() - started, 3),
+        "families": {name: spec["count"] for name, spec in FAMILIES.items()},
+        "raw_generated": response["content"],
+        "cases": cases,
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"out": str(args.out), "cases": len(cases), "model": response["model"], "elapsed_seconds": payload["elapsed_seconds"]}, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_everyday_deepseek_heldout_v3_cases.py

@@ -0,0 +1,353 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from core.database import ModelEndpoint, SessionLocal
+
+
+DEFAULT_OUT = REPO_ROOT / "data/evals/ody_everyday_deepseek_heldout_v3_20260821/cases.json"
+
+
+FAMILIES: dict[str, dict[str, Any]] = {
+    "negative_email_concept": {
+        "count": 3,
+        "instruction": "Text-only questions about what email/inbox/reply concepts mean. Do not ask to access the user's mailbox.",
+        "case": {
+            "kind": "negative_email",
+            "expect_no_tool": True,
+            "must_answer_any": ["email", "message", "reply", "inbox"],
+            "forbidden_tools": ["web_search", "mcp__email__list_emails", "mcp__email__read_email"],
+        },
+        "default_user": "What does replying to an email mean? Don't open my inbox.",
+    },
+    "negative_calendar_concept": {
+        "count": 3,
+        "instruction": "Text-only calendar questions that explicitly do not ask to create/update/delete events.",
+        "case": {
+            "kind": "negative_calendar",
+            "expect_no_tool": True,
+            "must_answer_any": ["calendar", "event", "invite", "schedule"],
+            "forbidden_tools": ["manage_calendar"],
+        },
+        "default_user": "What is a calendar invite? Don't add anything.",
+    },
+    "negative_web_no_lookup": {
+        "count": 3,
+        "instruction": "Text-only web/search concept prompts that explicitly say not to search or look anything up.",
+        "case": {
+            "kind": "negative_web",
+            "expect_no_tool": True,
+            "must_answer_any": ["search", "web", "pages", "results"],
+            "forbidden_tools": ["web_search"],
+        },
+        "default_user": "Explain what search results are without searching.",
+    },
+    "notes_create": {
+        "count": 4,
+        "instruction": "Personal note creation requests. Include literal __MARKER__ exactly once as the note title and a short body.",
+        "case": {
+            "kind": "note",
+            "marker": "__MARKER__",
+            "expect_first_tool": "manage_notes",
+            "must_mutate": "note_created",
+        },
+        "default_user": "Save a note titled __MARKER__ with body pick up dry cleaning",
+    },
+    "tasks_recurring": {
+        "count": 4,
+        "instruction": "Recurring reminder/automation requests that mention email/search/web/inbox words. Correct behavior is scheduled task creation, not doing the inner action immediately. Include __MARKER__ exactly once as task name.",
+        "case": {
+            "kind": "task",
+            "marker": "__MARKER__",
+            "expect_first_tool": "manage_tasks",
+            "forbidden_tools": ["web_search", "mcp__email__list_emails"],
+            "must_mutate": "task_created",
+        },
+        "default_user": "Create a recurring task named __MARKER__ to check my inbox every morning at 7:30",
+    },
+    "calendar_create": {
+        "count": 4,
+        "instruction": "Calendar create requests for tomorrow at 7pm. Include __MARKER__ exactly once as title/name.",
+        "case": {
+            "kind": "calendar",
+            "marker": "__MARKER__",
+            "expect_first_tool": "manage_calendar",
+            "must_mutate": "calendar_created_2026_08_22_19",
+        },
+        "default_user": "Put __MARKER__ on my calendar tomorrow at 7pm",
+    },
+    "calendar_move": {
+        "count": 4,
+        "instruction": "Calendar move/reschedule requests for an existing event. Include __MARKER__ exactly once and move it to 8pm tomorrow.",
+        "case": {
+            "kind": "calendar",
+            "marker": "__MARKER__",
+            "precreate_calendar_event": {
+                "summary": "__MARKER__",
+                "dtstart": "2026-08-22T19:00:00",
+                "dtend": "2026-08-22T20:00:00",
+            },
+            "expect_first_tool": "manage_calendar",
+            "forbidden_tools": ["manage_tasks"],
+            "must_mutate": "calendar_moved_2026_08_22_20",
+        },
+        "default_user": "Reschedule __MARKER__ to tomorrow at 8pm",
+    },
+    "calendar_delete": {
+        "count": 4,
+        "instruction": "Calendar delete/remove/cancel requests for an existing event by title/name. Include __MARKER__ exactly once.",
+        "case": {
+            "kind": "calendar",
+            "marker": "__MARKER__",
+            "precreate_calendar_event": {
+                "summary": "__MARKER__",
+                "dtstart": "2026-08-22T13:00:00",
+                "dtend": "2026-08-22T14:00:00",
+            },
+            "expect_first_tool": "manage_calendar",
+            "must_mutate": "calendar_deleted",
+        },
+        "default_user": "Cancel the calendar event titled __MARKER__",
+    },
+    "email_latest": {
+        "count": 4,
+        "instruction": "Personal latest/recent inbox requests. They must refer to the user's own email and must not sound like public web search.",
+        "case": {
+            "kind": "email",
+            "expect_first_tool_any": ["mcp__email__list_emails", "list_emails"],
+            "forbidden_tools": ["web_search", "web_fetch"],
+            "must_answer_any": ["From:", "UID", "latest email", "email"],
+        },
+        "default_user": "Show me the latest thing in my inbox.",
+    },
+    "web_synthesis": {
+        "count": 4,
+        "instruction": "Public web lookup requests about why snails bubble/foam. Must require lookup plus a concise explanation, not just links.",
+        "case": {
+            "kind": "web",
+            "expect_first_tool": "web_search",
+            "forbidden_repeat_tools": ["web_search"],
+            "must_answer_any": ["mucus", "foam", "bubble"],
+            "must_answer_any_2": ["stress", "irritant", "predator", "moisture", "defense"],
+            "forbidden_final": ["Here are links for that topic", "WEB SEARCH RESULTS", "```sources"],
+        },
+        "default_user": "Find out why snails foam up and explain the reason.",
+    },
+    "draft_active_email": {
+        "count": 4,
+        "instruction": "Active email compose draft edit requests. Ask to write/update the open/current/active draft, and include phrase '8am works'. Do not ask to send.",
+        "case": {
+            "kind": "draft",
+            "active_document": {
+                "title": "Everyday email draft probe",
+                "language": "email",
+                "content": (
+                    "To: test@example.com\n"
+                    "Subject: Re: Test manual draft\n"
+                    "In-Reply-To: <manual@example.com>\n"
+                    "References: <manual@example.com>\n"
+                    "X-Source-UID: 999999\n"
+                    "---\n\n"
+                    "---------- Previous message ----------\n"
+                    "Can you confirm the meeting time?\n"
+                ),
+            },
+            "expect_first_tool_any": ["update_document", "edit_document"],
+            "forbidden_tools": ["manage_calendar", "web_search", "mcp__email__list_emails", "mcp__email__read_email"],
+            "must_mutate": "document_contains_8am",
+        },
+        "default_user": "In the active email draft, write that 8am works for me.",
+        "fallback_users": [
+            "In the active email draft, write that 8am works for me.",
+            "Update the open email draft to say 8am works.",
+            "Add to the current draft that 8am works for me.",
+            "Write back in the active draft that 8am works.",
+        ],
+    },
+}
+
+
+def deepseek_endpoint() -> dict[str, str]:
+    db = SessionLocal()
+    try:
+        row = (
+            db.query(ModelEndpoint)
+            .filter(
+                ModelEndpoint.name.ilike("%deepseek%"),
+                ModelEndpoint.is_enabled == True,  # noqa: E712
+                ModelEndpoint.api_key.isnot(None),
+                ModelEndpoint.api_key != "",
+            )
+            .order_by(ModelEndpoint.updated_at.desc())
+            .first()
+        )
+        if row is None or not row.api_key:
+            raise RuntimeError("no enabled DeepSeek endpoint with API key")
+        return {
+            "name": row.name,
+            "base_url": row.base_url,
+            "api_key": row.api_key,
+            "cached_models": row.cached_models or "",
+        }
+    finally:
+        db.close()
+
+
+def call_deepseek(endpoint: dict[str, str], prompt: str) -> dict[str, Any]:
+    model = "deepseek-chat"
+    try:
+        cached = json.loads(endpoint["cached_models"] or "[]")
+        if cached:
+            model = cached[0]
+    except json.JSONDecodeError:
+        pass
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": "Return strict JSON only. No markdown or commentary."},
+            {"role": "user", "content": prompt},
+        ],
+        "temperature": 0.85,
+        "max_tokens": 5000,
+    }
+    req = request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=90) as resp:
+        body = json.loads(resp.read().decode("utf-8"))
+    content = body["choices"][0]["message"]["content"]
+    cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", (content or "").strip(), flags=re.I | re.S)
+    if not cleaned.startswith("{"):
+        match = re.search(r"\{.*\}", cleaned, flags=re.S)
+        if match:
+            cleaned = match.group(0)
+    try:
+        parsed = json.loads(cleaned)
+    except json.JSONDecodeError as exc:
+        raise RuntimeError(f"DeepSeek response was not JSON: {cleaned[:1000]!r}") from exc
+    return {"model": model, "content": parsed}
+
+
+def valid_user(family: str, text: Any) -> bool:
+    if not isinstance(text, str):
+        return False
+    lowered = text.lower()
+    marker_family = family in {
+        "notes_create",
+        "tasks_recurring",
+        "calendar_create",
+        "calendar_move",
+        "calendar_delete",
+    }
+    if marker_family and text.count("__MARKER__") != 1:
+        return False
+    if family == "calendar_create" and ("tomorrow" not in lowered or "7" not in lowered):
+        return False
+    if family == "calendar_move" and ("tomorrow" not in lowered or "8" not in lowered):
+        return False
+    if family == "draft_active_email" and "8am works" not in lowered:
+        return False
+    if family.startswith("negative_") and any(word in lowered for word in ("open my", "show me my", "latest", "create", "delete", "remove", "schedule it")):
+        return False
+    return 5 <= len(text.split()) <= 34
+
+
+def build_cases(generated: dict[str, Any]) -> list[dict[str, Any]]:
+    cases: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for family, spec in FAMILIES.items():
+        prompts = generated.get(family, [])
+        if not isinstance(prompts, list):
+            prompts = []
+        prompts = [item for item in prompts if valid_user(family, item)]
+        prompts.append(spec["default_user"])
+        chosen: list[str] = []
+        for prompt in prompts:
+            key = prompt.lower()
+            if key in seen:
+                continue
+            seen.add(key)
+            chosen.append(prompt)
+            if len(chosen) >= spec["count"]:
+                break
+        fallback_users = spec.get("fallback_users") or [spec["default_user"]]
+        fallback_idx = 0
+        while len(chosen) < spec["count"]:
+            fallback = fallback_users[fallback_idx % len(fallback_users)]
+            fallback_idx += 1
+            key = fallback.lower()
+            if key in seen and len(fallback_users) > 1:
+                continue
+            seen.add(key)
+            chosen.append(fallback)
+        for idx, user in enumerate(chosen):
+            case = dict(spec["case"])
+            case.update({"id": f"deepseek_v3_{family}_{idx:02d}", "user": user, "deepseek_family": family})
+            cases.append(case)
+    return cases
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+    args = parser.parse_args()
+
+    prompt = {
+        "task": "Generate broader held-out everyday Odysseus tool-use eval prompts.",
+        "date_context": "Current date is 2026-08-21 Asia/Tokyo; tomorrow is 2026-08-22.",
+        "requirements": [
+            "Return JSON object only.",
+            "Keys must be exactly the family names provided.",
+            "Each value is a list of natural user prompts.",
+            "Generate at least count+3 prompts per family so validation can discard weak ones.",
+            "For marker families, include literal placeholder __MARKER__ exactly once.",
+            "Do not copy the default prompt; produce realistic paraphrases with varied syntax.",
+            "Avoid multi-intent prompts; each prompt should test one requested action.",
+        ],
+        "families": {
+            name: {
+                "count": spec["count"],
+                "instruction": spec["instruction"],
+                "default": spec["default_user"],
+            }
+            for name, spec in FAMILIES.items()
+        },
+    }
+    endpoint = deepseek_endpoint()
+    started = time.time()
+    response = call_deepseek(endpoint, json.dumps(prompt, ensure_ascii=False))
+    cases = build_cases(response["content"])
+    payload = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "generator": "build_odysseus_everyday_deepseek_heldout_v3_cases.py",
+        "provider": "DeepSeek",
+        "model": response["model"],
+        "elapsed_seconds": round(time.time() - started, 3),
+        "families": {name: spec["count"] for name, spec in FAMILIES.items()},
+        "raw_generated": response["content"],
+        "cases": cases,
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"out": str(args.out), "cases": len(cases), "model": response["model"], "elapsed_seconds": payload["elapsed_seconds"]}, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_realistic_web_synthesis_rows.py

@@ -0,0 +1,361 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import sqlite3
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_ACTUALS = [
+    REPO_ROOT / "data/evals/ody_search_teacher_pipeline_20260821/deepseek_actual/actual_results.json",
+    REPO_ROOT / "data/evals/ody_v57_quick_live_search_cases_20260821/v59_run_20260821_2042/actual_results.json",
+]
+DEFAULT_OUT_DIR = Path("/home/pewds/odysseus-finetune/data/teacher_live_gaps/odysseus_v60_realistic_verbose_web_synthesis_20260821")
+
+WEB_TOOLS = {"web_search", "web_fetch"}
+FORBIDDEN_FINAL_RE = re.compile(
+    r"WEB SEARCH RESULTS|```sources|\b\d+\s+Web sources\b|from the search results|results indicate|returned snippets|top results|i searched",
+    re.IGNORECASE,
+)
+
+TOOL_SCHEMAS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "web_search",
+            "description": "Search the public web for source-backed information.",
+            "parameters": {
+                "type": "object",
+                "properties": {"query": {"type": "string"}},
+                "required": ["query"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "web_fetch",
+            "description": "Fetch a specific URL when search snippets do not contain enough evidence.",
+            "parameters": {
+                "type": "object",
+                "properties": {"url": {"type": "string"}},
+                "required": ["url"],
+            },
+        },
+    },
+]
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def load_teacher_endpoint(db_path: Path, model: str | None) -> dict[str, str]:
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    try:
+        row = conn.execute(
+            """
+            SELECT base_url, api_key, cached_models
+            FROM model_endpoints
+            WHERE is_enabled = 1
+              AND api_key IS NOT NULL
+              AND api_key != ''
+              AND (lower(name) LIKE '%deepseek%' OR lower(id) LIKE '%deepseek%')
+            ORDER BY updated_at DESC
+            LIMIT 1
+            """
+        ).fetchone()
+    finally:
+        conn.close()
+    if row is None:
+        raise RuntimeError("no enabled DeepSeek endpoint with API key found in app DB")
+    selected_model = model
+    if not selected_model:
+        cached = json.loads(row["cached_models"] or "[]")
+        selected_model = cached[0] if cached else "deepseek-v4-flash"
+    return {"base_url": row["base_url"], "api_key": row["api_key"], "model": selected_model}
+
+
+def call_json(endpoint: dict[str, str], payload: dict[str, Any]) -> dict[str, Any]:
+    body = {
+        "model": endpoint["model"],
+        "messages": [
+            {
+                "role": "system",
+                "content": (
+                    "Return strict JSON only. You are creating SFT final answers for web tool traces. "
+                    "Do not include chain-of-thought or prose outside JSON."
+                ),
+            },
+            {"role": "user", "content": json.dumps(payload, ensure_ascii=False)},
+        ],
+        "temperature": 0.2,
+        "max_tokens": 900,
+        "response_format": {"type": "json_object"},
+    }
+    req = request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(body).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=180) as resp:
+        parsed = json.loads(resp.read().decode("utf-8"))
+    text = str(parsed["choices"][0]["message"].get("content") or "").strip()
+    text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL).strip()
+    return json.loads(text)
+
+
+def normalize_args(tool: str, args: Any) -> dict[str, Any]:
+    if isinstance(args, dict):
+        return args
+    if isinstance(args, str):
+        stripped = args.strip()
+        if stripped.startswith("{"):
+            try:
+                parsed = json.loads(stripped)
+                if isinstance(parsed, dict):
+                    return parsed
+            except json.JSONDecodeError:
+                pass
+        return {"query": stripped} if tool == "web_search" else {"url": stripped}
+    return {}
+
+
+def compact_tool_output(text: str, max_chars: int = 3000) -> str:
+    text = re.sub(r"\r\n?", "\n", text or "").strip()
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    if len(text) <= max_chars:
+        return text
+    sources = ""
+    if text.startswith("```sources"):
+        end = text.find("```", 3)
+        if end != -1:
+            sources = text[: end + 3].strip()
+    summary_match = re.search(r"SEARCH RESULTS SUMMARY:\n[-]+\n(?P<body>.*?)(?:\n={10,}|\Z)", text, re.DOTALL)
+    summary = summary_match.group("body").strip() if summary_match else ""
+    fetched_match = re.search(r"FETCHED PAGE CONTENT:\n[-]+\n(?P<body>.*?)(?:\n={10,}|\Z)", text, re.DOTALL)
+    fetched = fetched_match.group("body").strip() if fetched_match else ""
+    chunks = [chunk for chunk in [sources, summary[:1600], fetched[:900]] if chunk]
+    compact = "\n\n".join(chunks).strip()
+    if not compact:
+        compact = text[:max_chars].rstrip()
+    return compact[:max_chars].rstrip()
+
+
+def load_results(paths: list[Path]) -> list[dict[str, Any]]:
+    out: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for path in paths:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+        for result in payload.get("results") or []:
+            key = f"{path}:{result.get('id')}"
+            if key in seen:
+                continue
+            seen.add(key)
+            result = dict(result)
+            result["_source_path"] = str(path)
+            out.append(result)
+    return out
+
+
+def load_teacher_finals(path: Path | None) -> dict[str, str]:
+    if path is None:
+        return {}
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    finals: dict[str, str] = {}
+    for item in payload.get("edits") or []:
+        if not item.get("accepted"):
+            continue
+        edited = item.get("edited") or {}
+        final = str(edited.get("final") or "").strip()
+        if final and not FORBIDDEN_FINAL_RE.search(final):
+            finals[str(item.get("id"))] = final
+    return finals
+
+
+def usable_web_steps(result: dict[str, Any], max_tools: int) -> list[dict[str, Any]]:
+    calls = result.get("tool_calls") or []
+    outputs = result.get("tool_outputs") or []
+    steps: list[dict[str, Any]] = []
+    for idx, call in enumerate(calls):
+        tool = call.get("tool") or call.get("name")
+        if tool not in WEB_TOOLS:
+            continue
+        if idx >= len(outputs):
+            continue
+        output = outputs[idx]
+        if output.get("tool") and output.get("tool") not in WEB_TOOLS:
+            continue
+        args = normalize_args(tool, call.get("args"))
+        if tool == "web_search" and not args.get("query"):
+            continue
+        if tool == "web_fetch" and not args.get("url"):
+            continue
+        content = compact_tool_output(str(output.get("output") or ""))
+        if not content:
+            continue
+        steps.append({"tool": tool, "args": args, "output": content})
+        if len(steps) >= max_tools:
+            break
+    return steps
+
+
+def teacher_final(endpoint: dict[str, str], result: dict[str, Any], steps: list[dict[str, Any]]) -> dict[str, Any]:
+    prompt = {
+        "task": "Write the assistant's final answer after these web tool calls.",
+        "current_date": "2026-08-21",
+        "user": result.get("user") or "",
+        "prior_turns": result.get("prior_turns") or [],
+        "tool_steps": steps,
+        "bad_actual_final": result.get("final_answer") or "",
+        "requirements": [
+            "Return JSON with should_train boolean, final string, and reason string.",
+            "Use the tool evidence to answer the user's actual question directly.",
+            "If snippets are insufficient for a precise value, say the best supported answer and the uncertainty briefly.",
+            "Do not say 'from the search results', 'results indicate', 'snippets', 'I searched', or list sources.",
+            "Do not copy raw snippets. Synthesize.",
+            "Keep the final to 1-4 short sentences.",
+            "If this request should not have searched, set should_train=false.",
+        ],
+    }
+    return call_json(endpoint, prompt)
+
+
+def build_row(result: dict[str, Any], steps: list[dict[str, Any]], final: str) -> dict[str, Any] | None:
+    final = re.sub(r"\s+", " ", final).strip()
+    if not final or len(final) > 900 or FORBIDDEN_FINAL_RE.search(final):
+        return None
+    messages: list[dict[str, Any]] = []
+    for turn in result.get("prior_turns") or []:
+        if isinstance(turn, dict) and turn.get("user"):
+            messages.append({"role": "user", "content": str(turn["user"])})
+            if turn.get("assistant"):
+                messages.append({"role": "assistant", "content": str(turn["assistant"])})
+    messages.append({"role": "user", "content": result.get("user") or ""})
+    for idx, step in enumerate(steps):
+        call_id = f"call_{result.get('id', 'web')}_{idx}"
+        messages.append({
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [{
+                "id": call_id,
+                "type": "function",
+                "function": {
+                    "name": step["tool"],
+                    "arguments": json.dumps(step["args"], separators=(",", ":"), ensure_ascii=True),
+                },
+            }],
+        })
+        messages.append({"role": "tool", "tool_call_id": call_id, "content": step["output"]})
+    messages.append({"role": "assistant", "content": final})
+    row = {
+        "messages": messages,
+        "tools": TOOL_SCHEMAS,
+        "generator": "odysseus_realistic_verbose_web_synthesis_teacher",
+        "metadata": {
+            "source_result_id": result.get("id"),
+            "source_path": result.get("_source_path"),
+            "source_pass": result.get("pass"),
+            "actual_final": result.get("final_answer") or "",
+        },
+    }
+    row["uuid"] = stable_id("ody_v60_realistic_web_synthesis", row)
+    return row
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--actual", type=Path, action="append", default=[])
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT_DIR)
+    parser.add_argument("--db", type=Path, default=REPO_ROOT / "data/app.db")
+    parser.add_argument("--teacher-model", default="")
+    parser.add_argument("--teacher-edits", type=Path)
+    parser.add_argument("--max-cases", type=int, default=180)
+    parser.add_argument("--max-tools", type=int, default=3)
+    args = parser.parse_args()
+
+    paths = args.actual or DEFAULT_ACTUALS
+    final_by_id = load_teacher_finals(args.teacher_edits)
+    endpoint = None if final_by_id else load_teacher_endpoint(args.db, args.teacher_model or None)
+    results = load_results(paths)
+    candidates = []
+    for result in results:
+        if result.get("kind") != "web":
+            continue
+        steps = usable_web_steps(result, args.max_tools)
+        if steps and steps[0]["tool"] == "web_search":
+            candidates.append((result, steps))
+    candidates = candidates[: args.max_cases]
+
+    rows: list[dict[str, Any]] = []
+    audits: list[dict[str, Any]] = []
+    for result, steps in candidates:
+        try:
+            if result.get("id") in final_by_id:
+                edited = {
+                    "should_train": True,
+                    "final": final_by_id[str(result.get("id"))],
+                    "reason": "reused existing teacher-edited final",
+                }
+            else:
+                assert endpoint is not None
+                edited = teacher_final(endpoint, result, steps)
+            row = None
+            if edited.get("should_train") is True:
+                row = build_row(result, steps, str(edited.get("final") or ""))
+            accepted = row is not None
+            if accepted:
+                rows.append(row)
+            audits.append({
+                "id": result.get("id"),
+                "source_path": result.get("_source_path"),
+                "accepted": accepted,
+                "tool_count": len(steps),
+                "actual_final": result.get("final_answer") or "",
+                "teacher": edited,
+            })
+        except Exception as exc:
+            audits.append({"id": result.get("id"), "source_path": result.get("_source_path"), "accepted": False, "error": repr(exc)})
+        print(json.dumps({"processed": len(audits), "accepted": len(rows), "id": result.get("id")}), flush=True)
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, row in enumerate(rows):
+        (val if idx % 10 == 9 else train).append(row)
+    for name, subset in [("all.jsonl", rows), ("train.jsonl", train), ("val.jsonl", val)]:
+        (args.out_dir / name).write_text("".join(json.dumps(row, ensure_ascii=True) + "\n" for row in subset), encoding="utf-8")
+    (args.out_dir / "audit.json").write_text(json.dumps({"audit": audits}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    manifest = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "source_actuals": [str(path) for path in paths],
+        "candidate_cases": len(candidates),
+        "accepted_sft_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "max_tools": args.max_tools,
+        "goal": "train direct synthesis after realistic verbose web_search/web_fetch outputs",
+        "files": {
+            "train": str(args.out_dir / "train.jsonl"),
+            "val": str(args.out_dir / "val.jsonl"),
+            "all": str(args.out_dir / "all.jsonl"),
+            "audit": str(args.out_dir / "audit.json"),
+        },
+    }
+    (args.out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(manifest, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_search_teacher_edited_rows.py

@@ -0,0 +1,267 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_ACTUAL = REPO_ROOT / "data/evals/ody_search_teacher_pipeline_20260821/deepseek_actual/actual_results.json"
+DEFAULT_OUT_DIR = Path("/home/pewds/odysseus-finetune/data/teacher_live_gaps/odysseus_v58_teacher_edited_search_traces_20260821")
+
+WEB_TOOLS = {"web_search", "web_fetch"}
+SOURCE_DUMP_RE = re.compile(r"WEB SEARCH RESULTS|```sources|\b\d+\s+Web sources\b", re.IGNORECASE)
+META_FINAL_RE = re.compile(r"\b(the user asked|the user is asking|tool evidence|i should answer)\b", re.IGNORECASE)
+
+TOOL_SCHEMAS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "web_search",
+            "description": "Search the public web for source-backed information.",
+            "parameters": {
+                "type": "object",
+                "properties": {"query": {"type": "string"}},
+                "required": ["query"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "web_fetch",
+            "description": "Fetch a specific URL when search snippets do not contain enough evidence.",
+            "parameters": {
+                "type": "object",
+                "properties": {"url": {"type": "string"}},
+                "required": ["url"],
+            },
+        },
+    },
+]
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def call_json(base_url: str, api_key: str, model: str, payload: dict[str, Any]) -> dict[str, Any]:
+    body = {
+        "model": model,
+        "messages": [
+            {
+                "role": "system",
+                "content": (
+                    "Return strict JSON only. You are editing tool-use traces for SFT. "
+                    "Do not include chain-of-thought or prose outside JSON."
+                ),
+            },
+            {"role": "user", "content": json.dumps(payload, ensure_ascii=False)},
+        ],
+        "temperature": 0.25,
+        "max_tokens": 2200,
+        "response_format": {"type": "json_object"},
+    }
+    req = request.Request(
+        base_url.rstrip("/") + "/chat/completions",
+        data=json.dumps(body).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=180) as resp:
+        parsed = json.loads(resp.read().decode("utf-8"))
+    text = str(parsed["choices"][0]["message"].get("content") or "").strip()
+    text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL).strip()
+    return json.loads(text)
+
+
+def summarize_outputs(result: dict[str, Any]) -> list[dict[str, Any]]:
+    outputs = []
+    for idx, output in enumerate(result.get("tool_outputs") or []):
+        text = str(output.get("output") or "")
+        outputs.append({
+            "tool": output.get("tool"),
+            "output_head": text[:1800],
+            "output_tail": text[-800:] if len(text) > 1800 else "",
+            "exit_code": output.get("exit_code"),
+            "call_args": (result.get("tool_calls") or [{}])[idx].get("args") if idx < len(result.get("tool_calls") or []) else None,
+        })
+    return outputs
+
+
+def needs_teacher_edit(result: dict[str, Any]) -> bool:
+    final = str(result.get("final_answer") or "")
+    tools = result.get("tool_names") or []
+    failures = result.get("failures") or []
+    if result.get("kind") != "web":
+        return False
+    if not tools or tools[0] != "web_search":
+        return True
+    if any(tool not in WEB_TOOLS for tool in tools):
+        return True
+    if len(tools) > 3:
+        return True
+    if SOURCE_DUMP_RE.search(final) or META_FINAL_RE.search(final):
+        return True
+    if len(final.split()) < 8:
+        return True
+    if failures:
+        return True
+    return False
+
+
+def teacher_edit(endpoint: dict[str, str], result: dict[str, Any]) -> dict[str, Any]:
+    prompt = {
+        "task": "Edit this failed/weak Odysseus web tool trace into one minimal correct SFT trace.",
+        "current_date": "2026-08-21",
+        "user": result.get("user"),
+        "prior_turns": result.get("prior_turns") or [],
+        "actual_tool_calls": result.get("tool_calls") or [],
+        "actual_tool_outputs": summarize_outputs(result),
+        "actual_final": result.get("final_answer") or "",
+        "failures": result.get("failures") or [],
+        "requirements": [
+            "Return JSON with should_train boolean, reason string, trace array, and final string.",
+            "If the user request is evergreen/simple and should not search, set should_train=false.",
+            "For search-worthy requests, trace must contain 1 to 3 tool steps.",
+            "Each trace step must have tool, args, and output.",
+            "Allowed tools are only web_search and web_fetch.",
+            "web_search args must be an object like {\"query\":\"...\"}. The query must preserve the important nouns, requested property, location, time, and follow-up context.",
+            "Use web_fetch only after a search when snippets are insufficient and include a plausible URL from the search evidence.",
+            "The output field should be concise synthetic tool evidence, not a huge raw dump. It must contain enough evidence to justify the final.",
+            "The final must answer directly in 1-4 sentences. No source dumps. No 'the user asked'.",
+            "Do not hardcode this exact test; infer the general correct behavior from the request.",
+        ],
+    }
+    return call_json(endpoint["base_url"], endpoint["api_key"], endpoint["model"], prompt)
+
+
+def build_row(result: dict[str, Any], edited: dict[str, Any]) -> dict[str, Any] | None:
+    if edited.get("should_train") is not True:
+        return None
+    trace = edited.get("trace")
+    final = re.sub(r"\s+", " ", str(edited.get("final") or "")).strip()
+    if not isinstance(trace, list) or not trace or len(trace) > 3:
+        return None
+    if not final or SOURCE_DUMP_RE.search(final) or META_FINAL_RE.search(final) or len(final) > 1200:
+        return None
+    messages: list[dict[str, Any]] = [{"role": "user", "content": result.get("user") or ""}]
+    for idx, step in enumerate(trace):
+        if not isinstance(step, dict):
+            return None
+        tool = str(step.get("tool") or "")
+        if tool not in WEB_TOOLS:
+            return None
+        args = step.get("args") or {}
+        if isinstance(args, str):
+            try:
+                args = json.loads(args)
+            except json.JSONDecodeError:
+                args = {"query": args} if tool == "web_search" else {"url": args}
+        if tool == "web_search" and not str(args.get("query") or "").strip():
+            return None
+        if tool == "web_fetch" and not str(args.get("url") or "").strip():
+            return None
+        output = str(step.get("output") or "").strip()
+        if not output or len(output) > 1800:
+            output = output[:1800].rstrip()
+        call_id = f"call_{result.get('id', 'trace')}_{idx}"
+        messages.append({
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [{
+                "id": call_id,
+                "type": "function",
+                "function": {"name": tool, "arguments": json.dumps(args, separators=(",", ":"), ensure_ascii=True)},
+            }],
+        })
+        messages.append({"role": "tool", "tool_call_id": call_id, "content": output})
+    messages.append({"role": "assistant", "content": final})
+    row = {
+        "messages": messages,
+        "tools": TOOL_SCHEMAS,
+        "generator": "odysseus_deepseek_teacher_edited_search_trace",
+        "metadata": {
+            "source_result_id": result.get("id"),
+            "source_pass": result.get("pass"),
+            "actual_tool_names": result.get("tool_names") or [],
+            "teacher_reason": edited.get("reason") or "",
+        },
+    }
+    row["uuid"] = stable_id("ody_v58_teacher_edited_search", row)
+    return row
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--actual", type=Path, default=DEFAULT_ACTUAL)
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT_DIR)
+    parser.add_argument("--base-url", default=os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1"))
+    parser.add_argument("--model", default=os.environ.get("DEEPSEEK_TEACHER_MODEL", "deepseek-chat"))
+    parser.add_argument("--api-key", default=os.environ.get("DEEPSEEK_API_KEY", ""))
+    parser.add_argument("--max-cases", type=int, default=120)
+    args = parser.parse_args()
+    if not args.api_key:
+        raise RuntimeError("DEEPSEEK_API_KEY is required")
+    payload = json.loads(args.actual.read_text(encoding="utf-8"))
+    endpoint = {"base_url": args.base_url, "api_key": args.api_key, "model": args.model}
+    candidates = [result for result in payload.get("results") or [] if needs_teacher_edit(result)]
+    candidates = candidates[: args.max_cases]
+    rows: list[dict[str, Any]] = []
+    edits: list[dict[str, Any]] = []
+    for result in candidates:
+        try:
+            edited = teacher_edit(endpoint, result)
+            row = build_row(result, edited)
+            accepted = row is not None
+            if accepted:
+                rows.append(row)
+            edits.append({
+                "id": result.get("id"),
+                "user": result.get("user"),
+                "accepted": accepted,
+                "actual_tool_names": result.get("tool_names") or [],
+                "actual_final": result.get("final_answer") or "",
+                "edited": edited,
+            })
+        except Exception as exc:
+            edits.append({"id": result.get("id"), "user": result.get("user"), "accepted": False, "error": repr(exc)})
+        print(json.dumps({"processed": len(edits), "accepted": len(rows), "id": result.get("id")}), flush=True)
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, row in enumerate(rows):
+        (val if idx % 8 == 7 else train).append(row)
+    for name, subset in [("all.jsonl", rows), ("train.jsonl", train), ("val.jsonl", val)]:
+        (args.out_dir / name).write_text("".join(json.dumps(row, ensure_ascii=True) + "\n" for row in subset), encoding="utf-8")
+    (args.out_dir / "edits.json").write_text(json.dumps({"edits": edits}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    manifest = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "source_actual_results": str(args.actual),
+        "candidate_cases": len(candidates),
+        "accepted_sft_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "allowed_tools": sorted(WEB_TOOLS),
+        "files": {
+            "train": str(args.out_dir / "train.jsonl"),
+            "val": str(args.out_dir / "val.jsonl"),
+            "all": str(args.out_dir / "all.jsonl"),
+            "edits": str(args.out_dir / "edits.json"),
+        },
+    }
+    (args.out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(manifest, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_tool_efficiency_sft_slice.py

@@ -0,0 +1,447 @@
+#!/usr/bin/env python3
+"""Build a small targeted Odysseus tool-router SFT slice.
+
+This slice targets current measured gaps rather than broad tool coverage:
+
+- one-call manage_memory add;
+- one-call manage_memory add inside CRUD follow-through;
+- clean manage_tasks create schema;
+- contextual web_search follow-up after a normal answer;
+- no-tool chat boundaries.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+
+MANAGE_MEMORY_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "manage_memory",
+        "description": "Manage saved memories: list, add, edit, delete, or search.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {"type": "string", "enum": ["list", "add", "edit", "delete", "search"]},
+                "text": {"type": "string"},
+                "memory_id": {"type": "string"},
+                "category": {"type": "string", "enum": ["fact", "event", "contact", "preference"]},
+            },
+            "required": ["action"],
+        },
+    },
+}
+
+MANAGE_TASKS_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "manage_tasks",
+        "description": "Manage scheduled or recurring background tasks.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {"type": "string", "enum": ["list", "create", "edit", "delete", "pause", "resume"]},
+                "task_id": {"type": "string"},
+                "name": {"type": "string"},
+                "prompt": {"type": "string"},
+                "task_type": {"type": "string", "enum": ["llm", "research", "action"]},
+                "schedule": {"type": "string"},
+                "scheduled_time": {"type": "string"},
+                "output_target": {"type": "string"},
+            },
+            "required": ["action"],
+        },
+    },
+}
+
+WEB_SEARCH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "web_search",
+        "description": "Search the web for current or source-backed information.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "query": {"type": "string"},
+                "time_filter": {"type": "string", "enum": ["day", "week", "month", "year"]},
+            },
+            "required": ["query"],
+        },
+    },
+}
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def tool_call(name: str, arguments: dict[str, Any], suffix: str) -> dict[str, Any]:
+    return {
+        "id": f"call_{suffix}",
+        "type": "function",
+        "function": {
+            "name": name,
+            "arguments": json.dumps(arguments, separators=(",", ":"), ensure_ascii=True),
+        },
+    }
+
+
+def memory_rows() -> list[dict[str, Any]]:
+    markers = [
+        ("Remember this temporary eval fact: {text}.", "fact"),
+        ("Save this about me: {text}.", "fact"),
+        ("Store this preference: {text}.", "preference"),
+        ("Add this to memory: {text}.", "fact"),
+        ("Add to memory that {text}.", "fact"),
+        ("Please remember: {text}.", "fact"),
+        ("Save this as a memory: {text}.", "fact"),
+        ("Keep this in saved memory: {text}.", "fact"),
+        ("Can you remember this for later: {text}.", "fact"),
+        ("Put this in memory: {text}.", "fact"),
+        ("Make a memory that says {text}.", "fact"),
+        ("I want you to remember that {text}.", "fact"),
+        ("Save this preference for me: {text}.", "preference"),
+        ("Add a saved fact: {text}.", "fact"),
+    ]
+    facts = [
+        "I prefer concise travel checklists",
+        "My current project is organizing public domain art references",
+        "I like calendar summaries grouped by day",
+        "My preferred invoice label is Tsuki admin",
+        "I want model eval notes kept short",
+        "I use Runpod for temporary H100 training jobs",
+        "I prefer source links when asking for websites",
+        "My document drafts should stay in markdown",
+        "short eval probes should use temporary fixture markers",
+        "tool add calls should include the memory text immediately",
+        "memory cleanup should be checked after CRUD evals",
+        "adapter comparisons should record both correctness and efficiency",
+        "I prefer benchmark summaries to include artifact paths",
+        "I want Odysseus tool tests to report input tokens",
+        "I prefer LAN testing before blaming model latency",
+        "I like public domain art links from official sources",
+        "I want temporary eval memories deleted after tests",
+        "I prefer compact prompts for Qwen tool-router evals",
+        "I track LoRA quality by correctness and tool efficiency",
+        "I want web-link followups to use search when URLs are requested",
+        "I prefer no-tool answers for general knowledge reminders",
+        "I want memory add calls to avoid validation retries",
+    ]
+    rows: list[dict[str, Any]] = []
+    for i, text in enumerate(facts):
+        template, category = markers[i % len(markers)]
+        user = template.format(text=text)
+        args = {"action": "add", "text": text, "category": category}
+        call = tool_call("manage_memory", args, f"memory_add_{i}")
+        row = {
+            "messages": [
+                {"role": "user", "content": user},
+                {"role": "assistant", "content": "", "tool_calls": [call]},
+                {"role": "tool", "tool_call_id": call["id"], "content": f"Memory added: [{category}] {text}"},
+                {"role": "assistant", "content": "Done."},
+            ],
+            "tools": [MANAGE_MEMORY_TOOL],
+            "generator": "targeted_efficiency_static_v1",
+            "metadata": {
+                "category": "memory_one_call_add",
+                "target_issue": "avoid_incomplete_manage_memory_add_first_call",
+                "expected_tool_calls": 1,
+            },
+        }
+        row["uuid"] = stable_id("ody_eff_memory", row)
+        rows.append(row)
+    return rows
+
+
+def memory_crud_rows() -> list[dict[str, Any]]:
+    specs = [
+        (
+            "ODY-EVAL-CRUD-MEMORY-FLOW alpha checkpoint",
+            "ODY-EVAL-CRUD-MEMORY-FLOW beta checkpoint",
+            "fact",
+        ),
+        (
+            "I prefer one paragraph status updates for model evals",
+            "I prefer concise bullet status updates for model evals",
+            "preference",
+        ),
+        (
+            "My current benchmark focus is Odysseus tool-call efficiency",
+            "My current benchmark focus is memory add one-call efficiency",
+            "fact",
+        ),
+        (
+            "I use temporary memory fixtures during harness tests",
+            "I delete temporary memory fixtures after harness tests",
+            "fact",
+        ),
+        (
+            "I want saved memory changes to avoid retry tool calls",
+            "I want saved memory add calls to include text immediately",
+            "preference",
+        ),
+        (
+            "Runpod H100 jobs should be tracked in short notes",
+            "Runpod H100 jobs should be tracked with adapter and eval paths",
+            "fact",
+        ),
+    ]
+    rows: list[dict[str, Any]] = []
+    for i, (alpha, beta, category) in enumerate(specs):
+        memory_id = f"mem_eff_{i:02d}"
+        add_call = tool_call(
+            "manage_memory",
+            {"action": "add", "text": alpha, "category": category},
+            f"memory_crud_add_{i}",
+        )
+        edit_call = tool_call(
+            "manage_memory",
+            {"action": "edit", "memory_id": memory_id, "text": beta},
+            f"memory_crud_edit_{i}",
+        )
+        delete_call = tool_call(
+            "manage_memory",
+            {"action": "delete", "memory_id": memory_id},
+            f"memory_crud_delete_{i}",
+        )
+        row = {
+            "messages": [
+                {"role": "user", "content": f"Remember this temporary eval fact: {alpha}."},
+                {"role": "assistant", "content": "", "tool_calls": [add_call]},
+                {
+                    "role": "tool",
+                    "tool_call_id": add_call["id"],
+                    "content": f"Memory added: [{category}] {alpha}\nMemory id: {memory_id}",
+                },
+                {"role": "assistant", "content": "Done."},
+                {"role": "user", "content": f"Update that memory to say {beta}."},
+                {"role": "assistant", "content": "", "tool_calls": [edit_call]},
+                {
+                    "role": "tool",
+                    "tool_call_id": edit_call["id"],
+                    "content": f"Memory updated: {beta}\nMemory id: {memory_id}",
+                },
+                {"role": "assistant", "content": "Updated."},
+                {"role": "user", "content": "Delete that memory."},
+                {"role": "assistant", "content": "", "tool_calls": [delete_call]},
+                {
+                    "role": "tool",
+                    "tool_call_id": delete_call["id"],
+                    "content": f"Memory '{memory_id}' deleted",
+                },
+                {"role": "assistant", "content": "Deleted."},
+            ],
+            "tools": [MANAGE_MEMORY_TOOL],
+            "generator": "targeted_efficiency_static_v2",
+            "metadata": {
+                "category": "memory_crud_one_call_followthrough",
+                "target_issue": "avoid_incomplete_manage_memory_add_first_call_in_crud_context",
+                "expected_tool_calls_per_turn": [1, 1, 1],
+            },
+        }
+        row["uuid"] = stable_id("ody_eff_memory_crud", row)
+        rows.append(row)
+    return rows
+
+
+def task_rows() -> list[dict[str, Any]]:
+    specs = [
+        ("Daily email triage checkpoint", "Summarize unread important email each morning.", "daily", "09:00"),
+        ("Weekly invoice reminder", "Remind me to review open invoices every Monday.", "weekly", "08:30"),
+        ("Runpod spend check", "Check the Runpod budget note and remind me if follow-up is needed.", "daily", "18:00"),
+        ("Calendar prep", "Prepare a short next-day calendar summary.", "daily", "20:00"),
+        ("Research queue sweep", "Review saved research tasks and list blockers.", "weekly", "10:00"),
+        ("Document cleanup reminder", "Remind me to tidy stale editor documents.", "weekly", "16:00"),
+    ]
+    rows: list[dict[str, Any]] = []
+    for i, (name, prompt, schedule, scheduled_time) in enumerate(specs):
+        user = f"Create a scheduled task named {name} that runs {schedule} at {scheduled_time} UTC and has prompt: {prompt}"
+        args = {
+            "action": "create",
+            "name": name,
+            "prompt": prompt,
+            "task_type": "llm",
+            "schedule": schedule,
+            "scheduled_time": scheduled_time,
+            "output_target": "chat",
+        }
+        call = tool_call("manage_tasks", args, f"task_create_{i}")
+        row = {
+            "messages": [
+                {"role": "user", "content": user},
+                {"role": "assistant", "content": "", "tool_calls": [call]},
+                {"role": "tool", "tool_call_id": call["id"], "content": f"Task created: {name}"},
+                {"role": "assistant", "content": "Task created."},
+            ],
+            "tools": [MANAGE_TASKS_TOOL],
+            "generator": "targeted_efficiency_static_v1",
+            "metadata": {
+                "category": "task_create_clean_schema",
+                "target_issue": "avoid_loose_task_create_fields",
+                "expected_tool_calls": 1,
+            },
+        }
+        row["uuid"] = stable_id("ody_eff_task", row)
+        rows.append(row)
+    return rows
+
+
+def web_followup_rows() -> list[dict[str, Any]]:
+    first_answers = [
+        (
+            "What are some good sites for public domain art?",
+            "Good public domain art sources include Wikimedia Commons, The Met Open Access, Rijksmuseum Rijksstudio, Smithsonian Open Access, and the Library of Congress.",
+            "send links",
+            "public domain art Wikimedia Commons Met Open Access Rijksmuseum Smithsonian Library of Congress official links",
+        ),
+        (
+            "What are good places to find old maps online?",
+            "Good places include the Library of Congress, David Rumsey Map Collection, Wikimedia Commons, and Old Maps Online.",
+            "sned links for those",
+            "old maps Library of Congress David Rumsey Wikimedia Commons Old Maps Online official links",
+        ),
+        (
+            "Where can I find free classical music recordings?",
+            "Try Musopen, Wikimedia Commons audio, Internet Archive, and IMSLP for public domain scores and recordings.",
+            "for the websites",
+            "free classical music recordings Musopen Wikimedia Commons Internet Archive IMSLP official links",
+        ),
+        (
+            "What are reliable sources for public domain books?",
+            "Project Gutenberg, Internet Archive, HathiTrust, Standard Ebooks, and Open Library are useful starting points.",
+            "send the links",
+            "public domain books Project Gutenberg Internet Archive HathiTrust Standard Ebooks Open Library official links",
+        ),
+    ]
+    rows: list[dict[str, Any]] = []
+    for i, (user1, answer1, user2, query) in enumerate(first_answers):
+        call = tool_call("web_search", {"query": query}, f"web_followup_{i}")
+        row = {
+            "messages": [
+                {"role": "user", "content": user1},
+                {"role": "assistant", "content": answer1},
+                {"role": "user", "content": user2},
+                {"role": "assistant", "content": "", "tool_calls": [call]},
+                {
+                    "role": "tool",
+                    "tool_call_id": call["id"],
+                    "content": "Search results include official source pages with URLs.",
+                },
+                {"role": "assistant", "content": "Here are the official links."},
+            ],
+            "tools": [WEB_SEARCH_TOOL],
+            "generator": "targeted_efficiency_static_v1",
+            "metadata": {
+                "category": "contextual_web_followup",
+                "target_issue": "use_web_search_for_links_followup_after_general_answer",
+                "expected_tool_calls": 1,
+            },
+        }
+        row["uuid"] = stable_id("ody_eff_web", row)
+        rows.append(row)
+    return rows
+
+
+def no_tool_rows() -> list[dict[str, Any]]:
+    pairs = [
+        ("hi", "Hi! How can I help?"),
+        ("who are you?", "I am Odysseus, your assistant."),
+        ("what does VAT stand for?", "VAT stands for value-added tax."),
+        ("where is Sweden on a map?", "Sweden is in Northern Europe, on the Scandinavian Peninsula between Norway and Finland."),
+        ("thanks", "You're welcome."),
+        ("what is public domain art?", "Public domain art is artwork whose copyright has expired, was waived, or never applied, so it can generally be reused freely."),
+        ("do you remember what VAT stands for?", "VAT stands for value-added tax."),
+        ("what should I remember before buying public domain art?", "Check the source, copyright status, license notes, and whether the site marks the work as public domain."),
+        ("remind me what Sweden is bordered by", "Sweden borders Norway to the west and Finland to the east."),
+        ("what does it mean to remember something in a computer?", "It means storing data or information so it can be retrieved later."),
+        ("do you remember what HTML stands for?", "HTML stands for HyperText Markup Language."),
+        ("remind me what CPU means", "CPU means central processing unit."),
+        ("can you remind me where Italy is?", "Italy is in Southern Europe, extending into the Mediterranean Sea."),
+        ("what should I remember when choosing a password?", "Use a long unique password, avoid reuse, and store it in a password manager."),
+        ("remember in Python, what does len do?", "In Python, len returns the number of items in an object such as a string, list, or dictionary."),
+        ("what did I ask you to remember in this sentence: remember to check licenses?", "You asked about the phrase 'remember to check licenses'; it means to keep license checks in mind."),
+        ("does remember always mean save to memory?", "No. It can mean recall, explain, or keep something in mind; only explicit save/store requests need memory tools."),
+        ("remind me what public domain means", "Public domain means a work is not restricted by copyright and can generally be used freely."),
+        ("what should I remember before using AI-generated images?", "Check licensing, provenance, consent issues, and whether the output matches your intended use."),
+        ("do you remember the capital of Sweden?", "The capital of Sweden is Stockholm."),
+        ("remind me how to spell calendar", "Calendar is spelled c-a-l-e-n-d-a-r."),
+        ("what does a computer memory store?", "Computer memory stores data and instructions for use by programs."),
+    ]
+    rows: list[dict[str, Any]] = []
+    for user, assistant in pairs:
+        row = {
+            "messages": [
+                {"role": "user", "content": user},
+                {"role": "assistant", "content": assistant},
+            ],
+            "tools": [MANAGE_MEMORY_TOOL, MANAGE_TASKS_TOOL, WEB_SEARCH_TOOL],
+            "generator": "targeted_efficiency_static_v1",
+            "metadata": {
+                "category": "no_tool_boundary",
+                "target_issue": "avoid_overcalling_tools_on_general_chat",
+                "expected_tool_calls": 0,
+            },
+        }
+        row["uuid"] = stable_id("ody_eff_boundary", row)
+        rows.append(row)
+    return rows
+
+
+def split_rows(rows: list[dict[str, Any]], val_every: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, row in enumerate(rows):
+        (val if idx % val_every == val_every - 1 else train).append(row)
+    return train, val
+
+
+def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("".join(json.dumps(row, ensure_ascii=True) + "\n" for row in rows), encoding="utf-8")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--out-dir",
+        default="/home/pewds/odysseus-finetune/data/targeted_efficiency/odysseus_tool_efficiency_v1_20260820",
+    )
+    parser.add_argument("--val-every", type=int, default=5)
+    args = parser.parse_args()
+
+    rows = memory_rows() + memory_crud_rows() + task_rows() + web_followup_rows() + no_tool_rows()
+    train, val = split_rows(rows, args.val_every)
+    out_dir = Path(args.out_dir)
+    write_jsonl(out_dir / "train.jsonl", train)
+    write_jsonl(out_dir / "val.jsonl", val)
+    write_jsonl(out_dir / "all.jsonl", rows)
+    manifest = {
+        "name": out_dir.name,
+        "total_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "source_eval": "/home/pewds/odysseus-cookbook-fresh/data/evals/qwen35_9b_v44_memory_onecall_efficiency_20260820_202333.json",
+        "categories": {
+            category: sum(1 for row in rows if row["metadata"]["category"] == category)
+            for category in sorted({row["metadata"]["category"] for row in rows})
+        },
+        "acceptance_target": (
+            "memory_add_one_call_efficiency should reach 2/2 efficiency; "
+            "memory_crud_followthrough should reach 3/3 correctness and 3/3 efficiency; "
+            "memory_add_wording_variants_efficiency should reach 6/6 correctness and 6/6 efficiency; "
+            "memory_no_tool_boundary should reach 4/4 no-tool correctness; "
+            "full contextual correctness should remain 42/42 or better."
+        ),
+    }
+    (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
+    print(json.dumps(manifest, indent=2, ensure_ascii=True))
+
+
+if __name__ == "__main__":
+    main()

scripts/build_odysseus_v54_live_gap_teacher_sft.py

@@ -0,0 +1,557 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import contextlib
+import hashlib
+import json
+import os
+import re
+import sqlite3
+import sys
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_OUT = Path("/home/pewds/odysseus-finetune/data/teacher_live_gaps/odysseus_v54_live_gap_teacher_20260821")
+DEFAULT_EVAL_OUT = REPO_ROOT / "data/evals/ody_v54_live_gap_teacher_heldout_20260821/cases.json"
+
+
+WEB_SEARCH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "web_search",
+        "description": "Search the web for current or source-backed information.",
+        "parameters": {
+            "type": "object",
+            "properties": {"query": {"type": "string"}},
+            "required": ["query"],
+        },
+    },
+}
+
+
+CALENDAR_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "manage_calendar",
+        "description": "Create, update, list, and delete calendar events.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {"type": "string"},
+                "summary": {"type": "string"},
+                "dtstart": {"type": "string"},
+                "dtend": {"type": "string"},
+            },
+            "required": ["action"],
+        },
+    },
+}
+
+
+FAMILIES: list[dict[str, Any]] = [
+    {
+        "name": "web_synthesis_animal_foam",
+        "train_count": 48,
+        "heldout_count": 12,
+        "instruction": (
+            "Public web lookup questions about animals producing foam, bubbles, froth, or mucus. "
+            "The assistant must search once with specific biological terms and then synthesize a concise cause/explanation. "
+            "Rows should include snails often, but also a few other small animal examples. Final answers must mention the relevant mechanism, "
+            "not dump links or say evidence is insufficient when the simulated evidence is enough."
+        ),
+    },
+    {
+        "name": "web_retry_after_weak_results",
+        "train_count": 24,
+        "heldout_count": 8,
+        "instruction": (
+            "The first web_search result is weak, dictionary-like, or off-topic. The assistant should make one improved web_search "
+            "with better scientific/current terms, then synthesize the answer. Focus on failures where a generic query found dictionary/noise."
+        ),
+    },
+    {
+        "name": "calendar_ambiguous_time_boundary",
+        "train_count": 16,
+        "heldout_count": 6,
+        "instruction": (
+            "Calendar requests with relative dates and ambiguous times. If the user says 8pm/8 PM/evening at 8, create or update 20:00. "
+            "If the user only says 'at 8' without AM/PM or context, ask a short clarification instead of guessing 8pm."
+        ),
+    },
+]
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def clean_text(value: Any) -> str:
+    return re.sub(r"\s+", " ", str(value or "")).strip()
+
+
+def clean_terms(value: Any) -> list[str]:
+    if isinstance(value, str):
+        text = clean_text(value)
+        return [text] if text else []
+    if isinstance(value, list):
+        return [clean_text(item) for item in value if clean_text(item)]
+    return []
+
+
+def tool_call(name: str, arguments: dict[str, Any], suffix: str) -> dict[str, Any]:
+    return {
+        "id": f"call_{suffix}",
+        "type": "function",
+        "function": {
+            "name": name,
+            "arguments": json.dumps(arguments, separators=(",", ":"), ensure_ascii=True),
+        },
+    }
+
+
+def deepseek_endpoint() -> dict[str, str]:
+    api_key = os.environ.get("DEEPSEEK_API_KEY", "").strip()
+    if api_key:
+        return {
+            "name": "env-deepseek",
+            "base_url": os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1"),
+            "api_key": api_key,
+            "cached_models": os.environ.get("DEEPSEEK_MODEL", "deepseek-chat"),
+        }
+    db_path = REPO_ROOT / "data/app.db"
+    conn = sqlite3.connect(str(db_path))
+    try:
+        conn.row_factory = sqlite3.Row
+        row = conn.execute(
+            """
+            SELECT name, base_url, api_key, cached_models
+            FROM model_endpoints
+            WHERE lower(name) LIKE '%deepseek%'
+              AND COALESCE(is_enabled, 0) = 1
+              AND COALESCE(api_key, '') != ''
+            ORDER BY updated_at DESC
+            LIMIT 1
+            """
+        ).fetchone()
+        if not row:
+            raise RuntimeError("no enabled DeepSeek endpoint with API key")
+        return {
+            "name": row["name"],
+            "base_url": row["base_url"],
+            "api_key": row["api_key"],
+            "cached_models": row["cached_models"] or "",
+        }
+    finally:
+        conn.close()
+
+
+def call_deepseek(endpoint: dict[str, str], prompt: dict[str, Any], max_tokens: int = 8000) -> dict[str, Any]:
+    model = "deepseek-chat"
+    try:
+        cached = json.loads(endpoint.get("cached_models") or "[]")
+        if cached:
+            model = cached[0]
+    except json.JSONDecodeError:
+        if endpoint.get("cached_models"):
+            model = endpoint["cached_models"]
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": "Return strict JSON only. No markdown or commentary."},
+            {"role": "user", "content": json.dumps(prompt, ensure_ascii=False)},
+        ],
+        "temperature": 0.65,
+        "max_tokens": max_tokens,
+    }
+    req = request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=120) as resp:
+        body = json.loads(resp.read().decode("utf-8"))
+    content = body["choices"][0]["message"]["content"]
+    cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", (content or "").strip(), flags=re.I | re.S)
+    if not cleaned.startswith("{"):
+        match = re.search(r"\{.*\}", cleaned, flags=re.S)
+        if match:
+            cleaned = match.group(0)
+    return {"model": model, "content": json.loads(cleaned)}
+
+
+def teacher_prompt(family: dict[str, Any], count: int, batch: int) -> dict[str, Any]:
+    return {
+        "task": "Generate Odysseus SFT specs for live tool-use gaps.",
+        "current_state": {
+            "model": "qwen35-9b-tool-router-v53-web-repair",
+            "live_gap_eval": "DeepSeek-heldout v3 rescored 38/41",
+            "real_failures": [
+                "Web search often searches but returns a weak snippet dump instead of a concise explanation.",
+                "If search evidence is weak/noisy, the route should search again with better terms instead of giving up or dumping links.",
+                "Calendar generated heldout contained an ambiguous 'tomorrow at 8' case; do not teach that bare 8 means 8pm.",
+            ],
+        },
+        "family": family["name"],
+        "count": count,
+        "batch": batch,
+        "family_instruction": family["instruction"],
+        "requirements": [
+            "Return JSON object with key rows: list.",
+            "Return exactly count rows.",
+            "Every row must have user and final.",
+            "Web rows need ideal_query, evidence, query_must_include, answer_must_include.",
+            "Retry rows also need bad_query and bad_evidence.",
+            "Calendar rows need calendar_args for tool rows or no_tool=true for clarification rows.",
+            "Use varied casual wording and typos, but do not include private names, email addresses, or secrets.",
+            "Final answers must be concise and user-facing.",
+            "Never include raw source blocks, WEB SEARCH RESULTS, or link dumps in final.",
+        ],
+        "target_examples_not_to_copy": [
+            "Look up why snails produce foam and give me a short explanation.",
+            "Why do snails make foam? Check online and explain briefly.",
+            "Search the web for the reason snails bubble up, then summarize it concisely.",
+            "Move EVENT to tomorrow at 8 PM.",
+            "Move EVENT to tomorrow at 8.",
+        ],
+    }
+
+
+def valid_spec(family: str, item: Any) -> bool:
+    if not isinstance(item, dict):
+        return False
+    user = clean_text(item.get("user"))
+    final = clean_text(item.get("final"))
+    if len(user.split()) < 4 or len(user) > 240 or not final:
+        return False
+    if any(bad in final for bad in ("WEB SEARCH RESULTS", "```sources", "Here are links")):
+        return False
+    if family.startswith("web_"):
+        if not clean_text(item.get("ideal_query")):
+            return False
+        if family == "web_retry_after_weak_results" and not clean_text(item.get("bad_query")):
+            return False
+    if family == "calendar_ambiguous_time_boundary":
+        if item.get("no_tool"):
+            return bool(re.search(r"\b(?:am|pm|morning|evening|clarify|which)\b", final, re.I))
+        args = item.get("calendar_args")
+        if not isinstance(args, dict):
+            return False
+        action = str(args.get("action") or "").lower()
+        if action not in {"create_event", "update_event"}:
+            return False
+        return bool(args.get("summary") and args.get("dtstart") and args.get("dtend"))
+    return True
+
+
+def deterministic_calendar_specs() -> list[dict[str, Any]]:
+    tool_specs = [
+        ("move the meeting to tomorrow at 8 PM", "update_event", "meeting", "2026-08-23T20:00:00", "Done. The meeting is moved to tomorrow at 8:00 PM."),
+        ("reschedule dinner to tomorrow at 8 in the evening", "update_event", "dinner", "2026-08-23T20:00:00", "Done. Dinner is rescheduled to tomorrow at 8:00 PM."),
+        ("shift the appointment to tomorrow at 8 PM", "update_event", "appointment", "2026-08-23T20:00:00", "Done. The appointment is moved to tomorrow at 8:00 PM."),
+        ("schedule a call for Friday at 8 PM", "create_event", "Call", "2026-08-28T20:00:00", "Scheduled the call for Friday at 8:00 PM."),
+        ("add lunch with Sam next Monday at 8pm", "create_event", "Lunch with Sam", "2026-08-24T20:00:00", "Scheduled lunch with Sam for next Monday at 8:00 PM."),
+        ("book dinner Friday evening at 8", "create_event", "Dinner", "2026-08-28T20:00:00", "Scheduled dinner for Friday at 8:00 PM."),
+        ("move the party to tomorrow evening at 8", "update_event", "party", "2026-08-23T20:00:00", "Done. The party is moved to tomorrow at 8:00 PM."),
+        ("change my workout event to tomorrow at 8pm", "update_event", "workout", "2026-08-23T20:00:00", "Done. The workout is moved to tomorrow at 8:00 PM."),
+    ]
+    specs: list[dict[str, Any]] = []
+    for user, action, summary, start, final in tool_specs:
+        hour = int(start[11:13]) + 1
+        specs.append({
+            "user": user,
+            "calendar_args": {
+                "action": action,
+                "summary": summary,
+                "dtstart": start,
+                "dtend": start[:11] + f"{hour:02d}" + start[13:],
+            },
+            "tool_result": "AI: Calendar updated.",
+            "final": final,
+        })
+    for user in [
+        "move meeting to tomorrow at 8",
+        "can u move my workout to tmrw at 8?",
+        "book dinner for Friday at 8?",
+        "shift the appointment to tomorrow at 8",
+        "move the event to tomorrow at 8",
+        "reschedule lunch next Monday at 8",
+        "change the appointment to the day after tomorrow at 8",
+        "push the call to Friday at 8",
+        "put the dentist appointment tomorrow at 8",
+        "move my calendar event to 8 tomorrow",
+        "schedule dinner at 8",
+        "set the meeting for 8 tomorrow",
+        "can we do the appointment at 8",
+        "change it to 8",
+    ]:
+        specs.append({
+            "user": user,
+            "no_tool": True,
+            "final": "Do you mean 8 AM or 8 PM?",
+        })
+    return specs
+
+
+def build_sft_row(family: str, idx: int, spec: dict[str, Any], split: str) -> dict[str, Any]:
+    user = clean_text(spec["user"])
+    final = clean_text(spec["final"])
+    messages: list[dict[str, Any]] = [{"role": "user", "content": user}]
+    tools: list[dict[str, Any]] = []
+    expected_calls = 0
+    if family == "web_retry_after_weak_results":
+        bad = tool_call("web_search", {"query": clean_text(spec["bad_query"])}, f"{family}_{idx}_bad")
+        good = tool_call("web_search", {"query": clean_text(spec["ideal_query"])}, f"{family}_{idx}_good")
+        messages.extend([
+            {"role": "assistant", "content": "", "tool_calls": [bad]},
+            {"role": "tool", "tool_call_id": bad["id"], "content": clean_text(spec.get("bad_evidence"))},
+            {"role": "assistant", "content": "", "tool_calls": [good]},
+            {"role": "tool", "tool_call_id": good["id"], "content": clean_text(spec.get("evidence"))},
+            {"role": "assistant", "content": final},
+        ])
+        tools = [WEB_SEARCH_TOOL]
+        expected_calls = 2
+    elif family.startswith("web_"):
+        call = tool_call("web_search", {"query": clean_text(spec["ideal_query"])}, f"{family}_{idx}")
+        messages.extend([
+            {"role": "assistant", "content": "", "tool_calls": [call]},
+            {"role": "tool", "tool_call_id": call["id"], "content": clean_text(spec.get("evidence"))},
+            {"role": "assistant", "content": final},
+        ])
+        tools = [WEB_SEARCH_TOOL]
+        expected_calls = 1
+    elif family == "calendar_ambiguous_time_boundary" and spec.get("no_tool"):
+        messages.append({"role": "assistant", "content": final})
+    else:
+        args = dict(spec["calendar_args"])
+        call = tool_call("manage_calendar", args, f"{family}_{idx}")
+        messages.extend([
+            {"role": "assistant", "content": "", "tool_calls": [call]},
+            {"role": "tool", "tool_call_id": call["id"], "content": clean_text(spec.get("tool_result")) or "AI: Calendar updated."},
+            {"role": "assistant", "content": final},
+        ])
+        tools = [CALENDAR_TOOL]
+        expected_calls = 1
+
+    row = {
+        "messages": messages,
+        "tools": tools,
+        "generator": "deepseek_teacher_v54_live_gap",
+        "metadata": {
+            "category": family,
+            "split": split,
+            "expected_tool_calls": expected_calls,
+            "query_must_include": clean_terms(spec.get("query_must_include")),
+            "answer_must_include": clean_terms(spec.get("answer_must_include")),
+            "source_failures": [
+                "deepseek_v3_web_synthesis_00",
+                "deepseek_v3_web_synthesis_03",
+                "deepseek_v3_calendar_move_02",
+            ],
+        },
+    }
+    row["uuid"] = stable_id("ody_v54_live_gap", row)
+    return row
+
+
+def build_eval_case(family: str, idx: int, spec: dict[str, Any]) -> dict[str, Any]:
+    case: dict[str, Any] = {
+        "id": f"v54_live_gap_{family}_{idx:02d}",
+        "kind": "calendar" if family == "calendar_ambiguous_time_boundary" else "web",
+        "user": clean_text(spec["user"]),
+        "deepseek_family": family,
+        "forbidden_final": ["WEB SEARCH RESULTS", "```sources", "Here are links for that topic"],
+    }
+    if family.startswith("web_"):
+        user_lower = case["user"].lower()
+        if re.search(r"\b(?:search|look\s+up|check\s+online|web|find\s+out|google)\b", user_lower):
+            case["expect_first_tool"] = "web_search"
+        if family != "web_retry_after_weak_results":
+            case["max_web_searches"] = 1
+        if family == "web_synthesis_animal_foam":
+            case["must_answer_any"] = ["mucus", "foam", "bubble", "froth", "slime"]
+            case["must_answer_any_2"] = [
+                "stress",
+                "defense",
+                "irritat",
+                "moisture",
+                "predator",
+                "protect",
+                "osmosis",
+                "salt",
+            ]
+        else:
+            terms = clean_terms(spec.get("answer_must_include"))
+            expanded: list[str] = []
+            for term in terms:
+                expanded.extend(part.strip() for part in re.split(r"[,/]| or ", term) if part.strip())
+            if expanded:
+                case["must_answer_any"] = expanded[:8]
+    elif spec.get("no_tool"):
+        case["expect_no_tool"] = True
+        case["forbidden_tools"] = ["manage_calendar"]
+        case["must_answer_any"] = ["AM", "PM", "morning", "evening", "clarify", "which"]
+    else:
+        case["expect_first_tool"] = "manage_calendar"
+    return case
+
+
+def split_rows(rows: list[dict[str, Any]], val_every: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, row in enumerate(rows):
+        (val if idx % val_every == val_every - 1 else train).append(row)
+    return train, val
+
+
+def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("".join(json.dumps(row, ensure_ascii=True) + "\n" for row in rows), encoding="utf-8")
+
+
+def file_sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT)
+    parser.add_argument("--eval-out", type=Path, default=DEFAULT_EVAL_OUT)
+    parser.add_argument("--val-every", type=int, default=6)
+    args = parser.parse_args()
+
+    endpoint = deepseek_endpoint()
+    started = time.time()
+    previous_manifest = args.out_dir / "manifest.json"
+    model = ""
+    if previous_manifest.exists():
+        with contextlib.suppress(Exception):
+            model = str(json.loads(previous_manifest.read_text(encoding="utf-8")).get("model") or "")
+    if not model:
+        model = "deepseek-chat"
+    raw: dict[str, Any] = {}
+    rows: list[dict[str, Any]] = []
+    heldout: list[dict[str, Any]] = []
+    seen_users: set[str] = set()
+
+    for family in FAMILIES:
+        needed = family["train_count"] + family["heldout_count"]
+        generated: list[dict[str, Any]] = []
+        valid: list[dict[str, Any]] = []
+        cache_path = args.out_dir / f"raw_{family['name']}.json"
+        cache_path.parent.mkdir(parents=True, exist_ok=True)
+        if family["name"] == "calendar_ambiguous_time_boundary":
+            generated = deterministic_calendar_specs()
+            valid = [item for item in generated if valid_spec(family["name"], item)]
+            cache_path.write_text(
+                json.dumps({"family": family["name"], "rows": generated, "source": "deterministic_schema_valid"}, ensure_ascii=False, indent=2) + "\n",
+                encoding="utf-8",
+            )
+        elif cache_path.exists():
+            cached = json.loads(cache_path.read_text(encoding="utf-8"))
+            generated = cached.get("rows", []) if isinstance(cached, dict) else []
+            valid = [item for item in generated if valid_spec(family["name"], item)]
+        for batch in range(1, 16):
+            if len(valid) >= needed:
+                break
+            response = call_deepseek(endpoint, teacher_prompt(family, min(18, needed + 4), batch))
+            model = response["model"]
+            batch_rows = response["content"].get("rows", [])
+            if isinstance(batch_rows, list):
+                generated.extend(batch_rows)
+                valid = [item for item in generated if valid_spec(family["name"], item)]
+                cache_path.write_text(
+                    json.dumps({"family": family["name"], "rows": generated}, ensure_ascii=False, indent=2) + "\n",
+                    encoding="utf-8",
+                )
+        raw[family["name"]] = generated
+        train_count = 0
+        heldout_count = 0
+        for item in valid:
+            key = clean_text(item["user"]).lower()
+            if key in seen_users:
+                continue
+            seen_users.add(key)
+            if train_count < family["train_count"]:
+                rows.append(build_sft_row(family["name"], train_count, item, "train_or_val"))
+                train_count += 1
+            elif heldout_count < family["heldout_count"]:
+                heldout.append(build_eval_case(family["name"], heldout_count, item))
+                heldout_count += 1
+            if train_count >= family["train_count"] and heldout_count >= family["heldout_count"]:
+                break
+        if train_count < family["train_count"] or heldout_count < family["heldout_count"]:
+            raise RuntimeError(
+                f"{family['name']} valid rows short: train {train_count}/{family['train_count']}, "
+                f"heldout {heldout_count}/{family['heldout_count']}"
+            )
+
+    train, val = split_rows(rows, args.val_every)
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    write_jsonl(args.out_dir / "train.jsonl", train)
+    write_jsonl(args.out_dir / "val.jsonl", val)
+    write_jsonl(args.out_dir / "all.jsonl", rows)
+    (args.out_dir / "raw_teacher.json").write_text(json.dumps(raw, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+    args.eval_out.parent.mkdir(parents=True, exist_ok=True)
+    eval_payload = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "generator": Path(__file__).name,
+        "provider": endpoint["name"],
+        "model": model,
+        "source": "V53 DeepSeek-heldout live-gap failures",
+        "cases": heldout,
+    }
+    args.eval_out.write_text(json.dumps(eval_payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+
+    manifest = {
+        "name": args.out_dir.name,
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "provider": endpoint["name"],
+        "model": model,
+        "elapsed_seconds": round(time.time() - started, 3),
+        "total_sft_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "heldout_cases": len(heldout),
+        "categories": {family["name"]: sum(1 for row in rows if row["metadata"]["category"] == family["name"]) for family in FAMILIES},
+        "heldout_categories": {family["name"]: sum(1 for case in heldout if case["deepseek_family"] == family["name"]) for family in FAMILIES},
+        "source_eval": "data/evals/ody_everyday_deepseek_heldout_v53_current_20260821_1508_dynamic_calendar_rescored/actual_results.json",
+        "acceptance_target": (
+            "Train as a narrow V54 top-up only after reviewing rows. Promote only if V54 passes live-hard, "
+            "DeepSeek-heldout rescored cases, V54 live-gap heldout, and old CRUD regression."
+        ),
+        "files": {
+            "train": str(args.out_dir / "train.jsonl"),
+            "val": str(args.out_dir / "val.jsonl"),
+            "all": str(args.out_dir / "all.jsonl"),
+            "raw_teacher": str(args.out_dir / "raw_teacher.json"),
+            "heldout_eval": str(args.eval_out),
+        },
+    }
+    for key, value in list(manifest["files"].items()):
+        manifest[f"{key}_sha256"] = file_sha256(Path(value))
+    (args.out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+
+    print(json.dumps({
+        "out_dir": str(args.out_dir),
+        "eval_out": str(args.eval_out),
+        "total_sft_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "heldout_cases": len(heldout),
+        "categories": manifest["categories"],
+        "heldout_categories": manifest["heldout_categories"],
+        "model": model,
+    }, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_v55_web_synthesis_teacher_sft.py

@@ -0,0 +1,519 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import contextlib
+import hashlib
+import json
+import os
+import random
+import re
+import sqlite3
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_OUT = Path("/home/pewds/odysseus-finetune/data/teacher_live_gaps/odysseus_v55_web_synthesis_teacher_20260821")
+DEFAULT_EVAL_OUT = REPO_ROOT / "data/evals/ody_v55_web_synthesis_teacher_heldout_20260821/cases.json"
+
+
+WEB_SEARCH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "web_search",
+        "description": "Search the web for current or source-backed information.",
+        "parameters": {
+            "type": "object",
+            "properties": {"query": {"type": "string"}},
+            "required": ["query"],
+        },
+    },
+}
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def clean(value: Any) -> str:
+    return re.sub(r"\s+", " ", str(value or "")).strip()
+
+
+def tool_call(name: str, arguments: dict[str, Any], suffix: str) -> dict[str, Any]:
+    return {
+        "id": f"call_{suffix}",
+        "type": "function",
+        "function": {
+            "name": name,
+            "arguments": json.dumps(arguments, separators=(",", ":"), ensure_ascii=True),
+        },
+    }
+
+
+def source_block(query: str, rows: list[tuple[str, str]]) -> str:
+    lines = [
+        "```sources",
+        *[f"[{idx}] {title}\n    https://example.test/{idx}" for idx, (title, _snippet) in enumerate(rows, start=1)],
+        "```",
+        "",
+        "======================================================================",
+        "WEB SEARCH RESULTS AND FETCHED CONTENT",
+        f"Query: {query}",
+        f"Searched {len(rows)} results, fetched {len(rows)} pages",
+        "======================================================================",
+        "",
+        "SEARCH RESULTS SUMMARY:",
+        "--------------------------------------------------",
+    ]
+    for idx, (title, snippet) in enumerate(rows, start=1):
+        lines.extend([f"[{idx}] {title}", f"    URL: https://example.test/{idx}", f"    Snippet: {snippet}", ""])
+    return "\n".join(lines).strip()
+
+
+ANCHORS: list[dict[str, Any]] = [
+    {
+        "family": "animal_foam_synthesis",
+        "topic": "sea cucumber defensive foam/sticky secretions",
+        "users": [
+            "why do sea creatures like sea cucumbers produce foam?",
+            "why do sea cucumbers shoot out sticky foamy stuff?",
+            "what is the foam/stringy stuff sea cucumbers produce for?",
+        ],
+        "query": "sea cucumber sticky foam mucus defense cuvierian tubules predators",
+        "rows": [
+            ("Sea cucumber defense", "Sea cucumbers use chemical defenses and can eject sticky Cuvierian tubules to tangle or deter predators."),
+            ("Cuvierian tubules", "Some sea cucumbers expel sticky mucus-like threads from the Cuvierian organ as a defensive response."),
+            ("Marine animal mucus", "Foam or froth around marine animals is often mucus or secretions mixed with water and air during stress or defense."),
+        ],
+        "final": "Sea cucumbers do it mainly as a defense response. What looks like foam is usually sticky mucus-like secretions or Cuvierian tubules mixed with water and air, used to distract, tangle, or deter predators.",
+        "answer_any": ["mucus", "foam", "sticky", "tubules"],
+        "answer_any_2": ["defense", "predator", "protect", "stress"],
+    },
+    {
+        "family": "animal_foam_synthesis",
+        "topic": "slug bubbles when touched",
+        "users": [
+            "search and tell me why some slugs bubble up when touched",
+            "why do slugs foam when you touch them?",
+            "look up why slugs bubble when disturbed",
+        ],
+        "query": "slug bubble foam mucus when touched stress defense predator",
+        "rows": [
+            ("Slug mucus", "Slugs produce mucus over their bodies; when disturbed, air can mix with the mucus and make bubbles or foam."),
+            ("Slug defense", "Many slugs secrete thicker or stickier mucus when threatened, which can deter predators and protect the body."),
+            ("Gastropod stress response", "Handling, salt, dryness, or irritation can increase mucus secretion in slugs and snails."),
+        ],
+        "final": "Slugs bubble when touched because disturbance can trigger extra mucus, and air gets mixed into it as foam. It is mostly a stress or defense response that helps protect the slug and discourage predators.",
+        "answer_any": ["mucus", "foam", "bubble", "slime"],
+        "answer_any_2": ["stress", "defense", "predator", "protect"],
+    },
+    {
+        "family": "animal_foam_synthesis",
+        "topic": "snail froth around body",
+        "users": [
+            "why do snails make froth around themselves?",
+            "what is the froth that comes out of snails?",
+            "why does my garden snail have white foam around it?",
+        ],
+        "query": "snail froth foam mucus stress irritation defense moisture",
+        "rows": [
+            ("Why snails bubble", "Land snails can release bubbly mucus when stressed, irritated, handled, exposed to salt, or drying out."),
+            ("Snail slime", "Snail slime is mucus; it helps movement, moisture retention, and protection from irritation."),
+            ("Foam in snails", "The froth is mucus mixed with air, usually a defensive or stress response rather than a separate substance."),
+        ],
+        "final": "The froth is snail mucus mixed with air. Snails can make it when stressed, irritated, handled, too dry, or exposed to salt or chemicals; it helps protect them and retain moisture.",
+        "answer_any": ["mucus", "foam", "froth", "bubble"],
+        "answer_any_2": ["stress", "defense", "irritat", "moisture", "salt"],
+    },
+    {
+        "family": "public_fact_synthesis",
+        "topic": "vaccines immune system",
+        "users": [
+            "how do vaccines work? give me a quick explanation",
+            "look up how vaccines work and summarize it simply",
+            "quickly explain how vaccines train your immune system",
+        ],
+        "query": "vaccines work immune system antigen antibodies memory cells",
+        "rows": [
+            ("How vaccines work", "Vaccines expose the immune system to a harmless antigen or instructions to make one, so the body can practice recognizing it."),
+            ("Immune memory", "After vaccination, immune cells and antibodies can respond faster if the real pathogen appears later."),
+            ("CDC vaccine basics", "Vaccines help the body build protection without having to get the disease first."),
+        ],
+        "final": "Vaccines show your immune system a harmless antigen, or instructions to make one, so it can build antibodies and memory cells. Later, if the real germ shows up, your immune system recognizes it faster.",
+        "answer_any": ["immune system", "antigen", "antibodies"],
+        "answer_any_2": ["memory", "recognizes", "protection"],
+    },
+    {
+        "family": "public_fact_synthesis",
+        "topic": "biggest animal ever",
+        "users": [
+            "whats the biggest animal ever?",
+            "look up the biggest animal ever and answer short",
+            "what is the largest animal that ever lived?",
+        ],
+        "query": "largest biggest animal ever blue whale weight 200 tons",
+        "rows": [
+            ("Largest animals", "The blue whale is generally considered the largest animal ever known to have lived."),
+            ("Blue whale size", "Blue whales can reach about 30 meters long and weigh roughly 180 to 200 tonnes."),
+            ("Biggest animal ever", "No dinosaur currently beats the blue whale for overall body mass."),
+        ],
+        "final": "The blue whale is the biggest animal ever known. The largest individuals can reach about 30 meters long and roughly 180-200 tonnes.",
+        "answer_any": ["blue whale"],
+        "answer_any_2": ["200", "ton", "30 meters"],
+    },
+    {
+        "family": "public_fact_synthesis",
+        "topic": "onions crying chemistry",
+        "users": [
+            "why do onions make you cry?",
+            "look up why chopping onions makes eyes water",
+            "what chemical in onions makes people cry?",
+        ],
+        "query": "onions make you cry syn-propanethial-S-oxide irritates eyes lachrymatory factor",
+        "rows": [
+            ("Onion lachrymatory factor", "Cutting onions releases enzymes that form syn-Propanethial-S-oxide, a volatile lachrymatory factor."),
+            ("Why onions cause tears", "The gas reaches the eyes, irritates nerve endings, and the eyes produce tears to flush it away."),
+            ("Onion chemistry", "Sulfur compounds in onion cells mix after cutting and create the eye-irritating vapor."),
+        ],
+        "final": "Cut onions produce syn-propanethial-S-oxide, a volatile sulfur compound. It irritates your eyes, so your tear glands make tears to dilute and wash it away.",
+        "answer_any": ["syn-propanethial-S-oxide", "sulfur"],
+        "answer_any_2": ["irritates", "eyes", "tears"],
+    },
+    {
+        "family": "public_fact_synthesis",
+        "topic": "wifi simple explanation",
+        "users": [
+            "how does wifi work? short explanation",
+            "look up how wifi works and explain simply",
+            "what does a wifi router actually do?",
+        ],
+        "query": "how wifi works radio waves router 2.4 GHz 5 GHz devices",
+        "rows": [
+            ("How Wi-Fi works", "Wi-Fi uses radio waves to send data between a router/access point and devices."),
+            ("Wi-Fi frequencies", "Common Wi-Fi bands include 2.4 GHz and 5 GHz, with newer systems also using 6 GHz."),
+            ("Wireless networking", "The router connects local wireless devices to the internet and manages packets between them."),
+        ],
+        "final": "Wi-Fi sends data by radio waves between your device and a router. The router talks to devices on bands like 2.4 GHz or 5 GHz, then forwards that traffic to and from the internet.",
+        "answer_any": ["radio waves"],
+        "answer_any_2": ["router", "2.4 GHz", "5 GHz"],
+    },
+    {
+        "family": "public_fact_synthesis",
+        "topic": "speed of light",
+        "users": [
+            "what is the speed of light? quick",
+            "look up the exact speed of light",
+            "speed of light in vacuum please",
+        ],
+        "query": "speed of light vacuum 299792458 meters per second",
+        "rows": [
+            ("Speed of light", "The speed of light in vacuum is exactly 299,792,458 meters per second."),
+            ("Physical constant c", "The defined value of c is 299,792,458 m/s."),
+            ("Light speed", "In everyday terms, light travels about 300,000 kilometers per second in vacuum."),
+        ],
+        "final": "In vacuum, the speed of light is exactly 299,792,458 meters per second, about 300,000 km/s.",
+        "answer_any": ["299", "792", "458"],
+        "answer_any_2": ["meters per second", "km/s", "vacuum"],
+    },
+]
+
+
+BAD_QUERY_ROWS = [
+    ("official links", "Official link directory", "A URL shortener and link directory; it does not answer the user's question."),
+    ("scientific links", "Scientific link collection", "Generic source list with no answer details."),
+    ("why", "WHY | English meaning", "Dictionary entry for the word why, unrelated to the user's topic."),
+    ("biggest", "BIGGEST | English meaning", "Dictionary entry for the word biggest, not an answer."),
+    ("Wikipedia Python packaging packaging.python.org PyPI pip setuptools build", "Python Packaging User Guide", "Python package publishing docs; unrelated to the user's question."),
+]
+
+
+def deepseek_endpoint() -> dict[str, str] | None:
+    api_key = os.environ.get("DEEPSEEK_API_KEY", "").strip()
+    if api_key:
+        return {
+            "name": "env-deepseek",
+            "base_url": os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1"),
+            "api_key": api_key,
+            "cached_models": os.environ.get("DEEPSEEK_MODEL", "deepseek-chat"),
+        }
+    db_path = REPO_ROOT / "data/app.db"
+    conn = sqlite3.connect(str(db_path))
+    try:
+        conn.row_factory = sqlite3.Row
+        row = conn.execute(
+            """
+            SELECT name, base_url, api_key, cached_models
+            FROM model_endpoints
+            WHERE lower(name) LIKE '%deepseek%'
+              AND COALESCE(is_enabled, 0) = 1
+              AND COALESCE(api_key, '') != ''
+            ORDER BY updated_at DESC
+            LIMIT 1
+            """
+        ).fetchone()
+        if not row:
+            return None
+        return {
+            "name": row["name"],
+            "base_url": row["base_url"],
+            "api_key": row["api_key"],
+            "cached_models": row["cached_models"] or "deepseek-chat",
+        }
+    finally:
+        conn.close()
+
+
+def call_deepseek(endpoint: dict[str, str], prompt: dict[str, Any]) -> list[dict[str, str]]:
+    model = "deepseek-chat"
+    with contextlib.suppress(Exception):
+        cached = json.loads(endpoint.get("cached_models") or "[]")
+        if isinstance(cached, list) and cached:
+            model = cached[0]
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": "Return strict JSON only. No markdown."},
+            {"role": "user", "content": json.dumps(prompt, ensure_ascii=False)},
+        ],
+        "temperature": 0.55,
+        "max_tokens": 5000,
+    }
+    req = request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=120) as resp:
+        body = json.loads(resp.read().decode("utf-8"))
+    content = body["choices"][0]["message"]["content"]
+    cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", clean(content), flags=re.I | re.S)
+    if not cleaned.startswith("{"):
+        match = re.search(r"\{.*\}", cleaned, flags=re.S)
+        if match:
+            cleaned = match.group(0)
+    parsed = json.loads(cleaned)
+    rows = parsed.get("rows", [])
+    return [row for row in rows if isinstance(row, dict)]
+
+
+def teacher_variants(anchor: dict[str, Any], count: int, endpoint: dict[str, str] | None) -> list[dict[str, str]]:
+    fallback: list[dict[str, str]] = []
+    prefixes = ["", "quick: ", "can you search this: ", "look this up and summarize: "]
+    for idx in range(count):
+        user = prefixes[idx % len(prefixes)] + anchor["users"][idx % len(anchor["users"])]
+        fallback.append({"user": user, "final": anchor["final"]})
+    if endpoint is None:
+        return fallback
+    prompt = {
+        "task": "Generate varied SFT phrasings for a web-search tool-use model.",
+        "count": count,
+        "topic": anchor["topic"],
+        "source_failure": "Current model searches, then dumps snippets instead of synthesizing a concise answer.",
+        "requirements": [
+            "Return JSON object with rows list.",
+            "Each row has user and final only.",
+            "User should be casual and varied; some can include typos.",
+            "Final must be concise, direct, and answer from evidence.",
+            "Final must not mention snippets, sources, WEB SEARCH RESULTS, or links.",
+            "Do not include private names, emails, secrets, or exact API keys.",
+        ],
+        "ideal_query": anchor["query"],
+        "evidence": [snippet for _title, snippet in anchor["rows"]],
+        "must_include_one_of": anchor["answer_any"],
+        "must_include_one_of_second_group": anchor["answer_any_2"],
+        "example_final_style": anchor["final"],
+    }
+    with contextlib.suppress(Exception):
+        rows = call_deepseek(endpoint, prompt)
+        valid = []
+        for row in rows:
+            user = clean(row.get("user"))
+            final = clean(row.get("final"))
+            if len(user.split()) >= 3 and final and not re.search(r"WEB SEARCH RESULTS|```sources|links?", final, re.I):
+                valid.append({"user": user, "final": final})
+        if len(valid) >= max(3, count // 2):
+            return (valid + fallback)[:count]
+    return fallback
+
+
+def row(category: str, messages: list[dict[str, Any]], expected_calls: int, anchor: dict[str, Any], source_ids: list[str]) -> dict[str, Any]:
+    item = {
+        "messages": messages,
+        "tools": [WEB_SEARCH_TOOL] if expected_calls else [],
+        "generator": "deepseek_teacher_v55_web_synthesis",
+        "metadata": {
+            "category": category,
+            "split": "train_or_val",
+            "expected_tool_calls": expected_calls,
+            "query_must_include": anchor["query"].split()[:5],
+            "answer_must_include": anchor["answer_any"] + anchor["answer_any_2"],
+            "source_case_ids": source_ids,
+        },
+    }
+    item["uuid"] = stable_id("ody_v55_web_synth", item)
+    return item
+
+
+def build_rows(endpoint: dict[str, str] | None, per_anchor: int, retry_per_anchor: int) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    raw: dict[str, Any] = {"provider": endpoint["name"] if endpoint else "deterministic_fallback", "anchors": []}
+    source_ids = [
+        "v54_live_gap_web_synthesis_animal_foam_01",
+        "v54_live_gap_web_synthesis_animal_foam_04",
+        "v54_live_gap_web_synthesis_animal_foam_05",
+        "v54_live_gap_web_synthesis_animal_foam_10",
+        "v54_live_gap_web_retry_after_weak_results_00",
+        "v54_live_gap_web_retry_after_weak_results_01",
+        "v54_live_gap_web_retry_after_weak_results_05",
+    ]
+    for anchor_idx, anchor in enumerate(ANCHORS):
+        variants = teacher_variants(anchor, per_anchor, endpoint)
+        raw["anchors"].append({"topic": anchor["topic"], "rows": variants})
+        for idx, variant in enumerate(variants):
+            call = tool_call("web_search", {"query": anchor["query"]}, f"synth_{anchor_idx}_{idx}")
+            messages = [
+                {"role": "user", "content": variant["user"]},
+                {"role": "assistant", "content": "", "tool_calls": [call]},
+                {"role": "tool", "tool_call_id": call["id"], "content": source_block(anchor["query"], anchor["rows"])},
+                {"role": "assistant", "content": variant["final"]},
+            ]
+            rows.append(row("web_compress_noisy_results", messages, 1, anchor, source_ids))
+        for idx in range(retry_per_anchor):
+            bad_query, title, snippet = BAD_QUERY_ROWS[(anchor_idx + idx) % len(BAD_QUERY_ROWS)]
+            first = tool_call("web_search", {"query": bad_query}, f"retry_{anchor_idx}_{idx}_bad")
+            second = tool_call("web_search", {"query": anchor["query"]}, f"retry_{anchor_idx}_{idx}_good")
+            messages = [
+                {"role": "user", "content": anchor["users"][idx % len(anchor["users"])]},
+                {"role": "assistant", "content": "", "tool_calls": [first]},
+                {"role": "tool", "tool_call_id": first["id"], "content": source_block(bad_query, [(title, snippet)])},
+                {"role": "assistant", "content": "", "tool_calls": [second]},
+                {"role": "tool", "tool_call_id": second["id"], "content": source_block(anchor["query"], anchor["rows"])},
+                {"role": "assistant", "content": anchor["final"]},
+            ]
+            rows.append(row("web_retry_bad_query_then_synthesize", messages, 2, anchor, source_ids))
+    return rows, raw
+
+
+def build_eval_cases() -> list[dict[str, Any]]:
+    cases: list[dict[str, Any]] = []
+    for idx, anchor in enumerate(ANCHORS):
+        cases.append({
+            "id": f"v55_web_synthesis_anchor_{idx:02d}",
+            "kind": "web",
+            "user": anchor["users"][0],
+            "forbidden_final": ["WEB SEARCH RESULTS", "```sources", "Here are links", "not enough clear evidence"],
+            "must_answer_any": anchor["answer_any"],
+            "must_answer_any_2": anchor["answer_any_2"],
+            "max_web_searches": 2,
+        })
+    for idx, anchor in enumerate(ANCHORS[:5]):
+        cases.append({
+            "id": f"v55_web_retry_anchor_{idx:02d}",
+            "kind": "web",
+            "user": "search properly and answer: " + anchor["users"][1],
+            "expect_first_tool": "web_search",
+            "forbidden_query_any": ["official links", "scientific links", "python packaging", "dictionary"],
+            "must_answer_any": anchor["answer_any"],
+            "must_answer_any_2": anchor["answer_any_2"],
+            "forbidden_final": ["WEB SEARCH RESULTS", "```sources", "Here are links", "not enough clear evidence"],
+            "max_web_searches": 2,
+        })
+    return cases
+
+
+def split_rows(rows: list[dict[str, Any]], val_every: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, item in enumerate(rows):
+        (val if idx % val_every == val_every - 1 else train).append(item)
+    return train, val
+
+
+def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("".join(json.dumps(item, ensure_ascii=True) + "\n" for item in rows), encoding="utf-8")
+
+
+def file_sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT)
+    parser.add_argument("--eval-out", type=Path, default=DEFAULT_EVAL_OUT)
+    parser.add_argument("--per-anchor", type=int, default=14)
+    parser.add_argument("--retry-per-anchor", type=int, default=4)
+    parser.add_argument("--val-every", type=int, default=6)
+    parser.add_argument("--seed", type=int, default=55)
+    args = parser.parse_args()
+
+    started = time.time()
+    rng = random.Random(args.seed)
+    endpoint = deepseek_endpoint()
+    rows, raw = build_rows(endpoint, args.per_anchor, args.retry_per_anchor)
+    rng.shuffle(rows)
+    train, val = split_rows(rows, args.val_every)
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    write_jsonl(args.out_dir / "train.jsonl", train)
+    write_jsonl(args.out_dir / "val.jsonl", val)
+    write_jsonl(args.out_dir / "all.jsonl", rows)
+    (args.out_dir / "raw_teacher.json").write_text(json.dumps(raw, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+    eval_cases = build_eval_cases()
+    args.eval_out.parent.mkdir(parents=True, exist_ok=True)
+    args.eval_out.write_text(
+        json.dumps(
+            {
+                "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                "generator": Path(__file__).name,
+                "source": "V54 live heldout failures where search ran but final synthesis missed answer terms.",
+                "cases": eval_cases,
+            },
+            ensure_ascii=True,
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    categories = sorted({item["metadata"]["category"] for item in rows})
+    manifest = {
+        "name": args.out_dir.name,
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "provider": raw["provider"],
+        "elapsed_seconds": round(time.time() - started, 3),
+        "total_sft_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "heldout_cases": len(eval_cases),
+        "categories": {category: sum(1 for item in rows if item["metadata"]["category"] == category) for category in categories},
+        "source_eval": "data/evals/ody_v54_live_gap_topup_gate_20260821_1555_queryguard2/live_gap_heldout/actual_results.json",
+        "source_case_ids": rows[0]["metadata"]["source_case_ids"] if rows else [],
+        "acceptance_target": (
+            "V55 must pass user-reported web 3/3, V54 live-gap heldout, V55 synthesis heldout, "
+            "and old CRUD regression before replacing V53/V54."
+        ),
+        "files": {
+            "train": str(args.out_dir / "train.jsonl"),
+            "val": str(args.out_dir / "val.jsonl"),
+            "all": str(args.out_dir / "all.jsonl"),
+            "raw_teacher": str(args.out_dir / "raw_teacher.json"),
+            "heldout_eval": str(args.eval_out),
+        },
+    }
+    for key, value in list(manifest["files"].items()):
+        manifest[f"{key}_sha256"] = file_sha256(Path(value))
+    (args.out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({k: manifest[k] for k in ("provider", "total_sft_rows", "train_rows", "val_rows", "heldout_cases", "categories", "source_case_ids")}, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_v56_broad_web_teacher_sft.py

@@ -0,0 +1,551 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import contextlib
+import hashlib
+import json
+import os
+import random
+import re
+import sqlite3
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_OUT = Path("/home/pewds/odysseus-finetune/data/teacher_live_gaps/odysseus_v56_broad_web_teacher_20260821")
+DEFAULT_EVAL_OUT = REPO_ROOT / "data/evals/ody_v56_broad_web_teacher_heldout_20260821/cases.json"
+DEFAULT_FAILURES = REPO_ROOT / "data/evals/ody_web_broad_live_search_v1_20260821/v56_targets/failure_targets.json"
+
+
+WEB_SEARCH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "web_search",
+        "description": "Search the web for current or source-backed information.",
+        "parameters": {
+            "type": "object",
+            "properties": {"query": {"type": "string"}},
+            "required": ["query"],
+        },
+    },
+}
+
+
+def clean(value: Any) -> str:
+    return re.sub(r"\s+", " ", str(value or "")).strip()
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def tool_call(name: str, arguments: dict[str, Any], suffix: str) -> dict[str, Any]:
+    return {
+        "id": f"call_{suffix}",
+        "type": "function",
+        "function": {
+            "name": name,
+            "arguments": json.dumps(arguments, separators=(",", ":"), ensure_ascii=True),
+        },
+    }
+
+
+def source_block(query: str, rows: list[tuple[str, str]]) -> str:
+    lines = [
+        "```sources",
+        *[f"[{idx}] {title}\n    https://example.test/{idx}" for idx, (title, _snippet) in enumerate(rows, start=1)],
+        "```",
+        "",
+        "======================================================================",
+        "WEB SEARCH RESULTS AND FETCHED CONTENT",
+        f"Query: {query}",
+        f"Searched {len(rows)} results, fetched {len(rows)} pages",
+        "======================================================================",
+        "",
+        "SEARCH RESULTS SUMMARY:",
+        "--------------------------------------------------",
+    ]
+    for idx, (title, snippet) in enumerate(rows, start=1):
+        lines.extend([f"[{idx}] {title}", f"    URL: https://example.test/{idx}", f"    Snippet: {snippet}", ""])
+    return "\n".join(lines).strip()
+
+
+ANCHORS: list[dict[str, Any]] = [
+    {
+        "case_id": "web_current_exchange_rate_01",
+        "family": "current_numeric_conversion",
+        "topic": "JPY to EUR current conversion",
+        "users": [
+            "how much is 10000 yen in euros today?",
+            "convert 10,000 JPY to EUR using today's rate",
+            "what is 10000 yen worth in euro right now?",
+            "yen to euros today, 10000 yen",
+        ],
+        "query": "10000 JPY to EUR today exchange rate",
+        "rows": [
+            ("JPY to EUR currency converter", "1 JPY = 0.005385 EUR on Aug 21, 2026; 10,000 JPY equals about 53.85 EUR before fees."),
+            ("Japanese Yen to Euro exchange rate", "The current exchange rate is roughly 0.00538 euro per Japanese yen."),
+            ("Currency conversion note", "Live exchange rates move through the day, so conversions should be treated as approximate."),
+        ],
+        "final": "At a rate around 1 JPY = 0.005385 EUR, 10,000 yen is about 53.85 euros before fees. The exact amount can move a little with the live exchange rate.",
+        "answer_any": ["eur", "euro", "€"],
+        "answer_any_2": ["10000", "10,000", "yen", "JPY"],
+    },
+    {
+        "case_id": "web_fact_vaccine_mechanism_01",
+        "family": "evergreen_synthesis",
+        "topic": "how vaccines work",
+        "users": [
+            "how do vaccines work? quick explanation",
+            "explain how vaccines train your immune system",
+            "look up how vaccines work and summarize simply",
+            "what do vaccines do in the body?",
+        ],
+        "query": "vaccines work immune system antigen antibodies memory cells protection",
+        "rows": [
+            ("How vaccines work", "Vaccines expose the immune system to a harmless antigen or instructions to make one."),
+            ("Immune memory", "Vaccination helps form antibodies and memory cells so the body recognizes the pathogen faster later."),
+            ("Vaccine protection", "Vaccines build protection without requiring the person to get the disease first."),
+        ],
+        "final": "Vaccines show your immune system a harmless antigen, or instructions for one. That helps the body make antibodies and memory cells, so it can recognize the real germ faster later.",
+        "answer_any": ["immune system", "antigen", "antibodies"],
+        "answer_any_2": ["memory", "protection", "recognize"],
+    },
+    {
+        "case_id": "web_fact_onion_tears_chemistry_01",
+        "family": "evergreen_synthesis",
+        "topic": "onion tears chemical",
+        "users": [
+            "what chemical makes your eyes water when chopping onions?",
+            "why do onions make you cry and what chemical is it?",
+            "look up the onion tear chemical",
+            "what makes onion fumes irritate your eyes?",
+        ],
+        "query": "onions chopping eyes water syn-propanethial-S-oxide lachrymatory sulfur irritates eyes tears",
+        "rows": [
+            ("Onion lachrymatory factor", "Cut onions form syn-Propanethial-S-oxide, a sulfur-containing lachrymatory factor."),
+            ("Why onions cause tears", "The volatile chemical reaches the eyes, irritates nerve endings, and triggers tears."),
+            ("Onion chemistry", "Enzymes and sulfur compounds mix when onion cells are cut, creating the eye-irritating gas."),
+        ],
+        "final": "Chopped onions make syn-propanethial-S-oxide, a sulfur-based lachrymatory chemical. It irritates your eyes, so your tear glands water to dilute and flush it away.",
+        "answer_any": ["syn-propanethial", "sulfur", "lachrymatory"],
+        "answer_any_2": ["eyes", "tears", "irritates"],
+    },
+    {
+        "case_id": "web_fact_tallest_mountain_01",
+        "family": "evergreen_synthesis",
+        "topic": "tallest mountain above sea level",
+        "users": [
+            "what is the tallest mountain above sea level?",
+            "which mountain is highest measured from sea level?",
+            "look up the tallest mountain and its height",
+            "how tall is the highest mountain on earth?",
+        ],
+        "query": "tallest mountain above sea level Mount Everest 8848.86 meters",
+        "rows": [
+            ("Mount Everest height", "Mount Everest is the tallest mountain above sea level at 8,848.86 meters."),
+            ("Highest mountain on Earth", "Everest's summit is 29,031.7 feet above mean sea level."),
+            ("Above sea level ranking", "Mount Everest is highest by elevation above sea level, while other mountains can be taller by base-to-peak measures."),
+        ],
+        "final": "Mount Everest is the tallest mountain above sea level, at about 8,848.86 meters, or 29,031.7 feet.",
+        "answer_any": ["Everest"],
+        "answer_any_2": ["8,848", "8848", "8,849", "8849"],
+    },
+    {
+        "case_id": "web_obscure_product_lookup_01",
+        "family": "obscure_lookup_uncertain",
+        "topic": "Citizen watch dating uncertainty",
+        "users": [
+            "what year is a Citizen 4-820789Y watch likely from?",
+            "can you date a Citizen 4-820789Y watch?",
+            "look up Citizen 4-820789Y and tell me the likely year",
+            "what does Citizen watch code 4-820789Y say about the year?",
+        ],
+        "query": "Citizen 4-820789Y watch case model serial date year",
+        "rows": [
+            ("Citizen watch serial dating", "Citizen watches are usually dated from the caseback serial number plus the movement or case code."),
+            ("Citizen case code", "A code like 4-820789Y is a movement/case reference, not enough by itself to identify one exact year."),
+            ("How to date Citizen watches", "The serial number's first digit often gives the year within a decade; the full caseback is needed to resolve the decade."),
+        ],
+        "final": "I cannot verify one exact year from 4-820789Y alone. That looks like a Citizen movement/case code; to date it, use the full caseback serial number plus the case/model code to resolve the decade.",
+        "answer_any": ["serial", "case", "model", "year", "cannot", "likely"],
+        "answer_any_2": ["Citizen", "watch", "code"],
+    },
+    {
+        "case_id": "web_synthesis_no_source_dump_01",
+        "family": "practical_synthesis",
+        "topic": "sourdough starter acetone smell",
+        "users": [
+            "look up why sourdough starter smells like acetone and tell me what to do",
+            "my sourdough starter smells like nail polish remover, what should I do?",
+            "why does sourdough starter smell like acetone?",
+            "search sourdough acetone smell fix",
+        ],
+        "query": "sourdough starter acetone nail polish smell hungry feed discard fix",
+        "rows": [
+            ("Sourdough acetone smell", "An acetone or nail polish remover smell usually means the starter is hungry and producing alcohol or other stress byproducts."),
+            ("Fix sourdough starter smell", "Discard some starter, feed it fresh flour and water, and repeat regular feedings until the smell becomes pleasantly yeasty or tangy."),
+            ("When to throw starter away", "Discard the starter if it shows fuzzy mold, pink/orange discoloration, or a rotten odor."),
+        ],
+        "final": "An acetone smell usually means the starter is hungry or stressed, not automatically ruined. Discard part of it, feed with fresh flour and water, and feed more often; throw it out only if you see mold, pink/orange discoloration, or rotten smells.",
+        "answer_any": ["hungry", "feed", "starter", "discard"],
+        "answer_any_2": ["acetone", "nail polish", "mold"],
+    },
+    {
+        "case_id": "web_context_no_stale_query_01",
+        "family": "context_followup_prevention",
+        "topic": "prevent onion tears contextual follow-up",
+        "users": [
+            "is there a way to stop it?",
+            "how do I prevent it?",
+            "can I stop that from happening?",
+            "what helps with that?",
+        ],
+        "prior_user": "what causes onions to make you cry",
+        "query": "onions prevent crying stop tears chill sharp knife ventilation goggles water",
+        "rows": [
+            ("How to stop onion tears", "Chilling onions, using a sharp knife, and improving ventilation can reduce the gas that reaches your eyes."),
+            ("Prevent onion crying", "Cut near running water or under a vent, avoid crushing the onion, and consider goggles for sensitive eyes."),
+            ("Onion tear prevention", "The goal is to reduce release of syn-propanethial-S-oxide or keep it away from your eyes."),
+        ],
+        "final": "Yes. Chill the onion first, use a sharp knife so you crush fewer cells, keep air moving with a vent or fan, and use goggles if you are sensitive. Those steps reduce or block the tear-causing onion vapor.",
+        "answer_any": ["chill", "knife", "ventilation", "water", "goggles", "cut"],
+        "answer_any_2": ["onion", "tears", "cry"],
+    },
+]
+
+
+BAD_QUERY_ROWS = [
+    ("why", "WHY | English meaning", "Dictionary entry for the word why, unrelated to the user's topic."),
+    ("official weather", "OFFICIAL | English meaning", "Dictionary entry for official, not a weather forecast."),
+    ("safe touch", "SafeTouch Security Systems", "A home security company page, unrelated to handling animals or food."),
+    ("browser does without looking", "Browser download page", "A browser marketing page, even though the user asked not to look it up."),
+]
+
+
+NEGATIVE_NO_TOOL_ROWS = [
+    ("explain what a web browser does without looking it up", "A web browser is an app that fetches web pages, renders HTML/CSS, runs JavaScript, and lets you navigate the web. No lookup needed for that basic explanation."),
+    ("without searching, what does a browser do?", "A browser requests pages from websites, displays them, runs page scripts, and manages things like tabs, history, cookies, and downloads."),
+    ("answer from memory: what is a web search engine?", "A web search engine crawls and indexes pages, then ranks matching results when you type a query."),
+]
+
+
+def deepseek_endpoint() -> dict[str, str] | None:
+    api_key = os.environ.get("DEEPSEEK_API_KEY", "").strip()
+    if api_key:
+        return {
+            "name": "env-deepseek",
+            "base_url": os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1"),
+            "api_key": api_key,
+            "cached_models": os.environ.get("DEEPSEEK_MODEL", "deepseek-chat"),
+        }
+    db_path = REPO_ROOT / "data/app.db"
+    conn = sqlite3.connect(str(db_path))
+    try:
+        conn.row_factory = sqlite3.Row
+        row = conn.execute(
+            """
+            SELECT name, base_url, api_key, cached_models
+            FROM model_endpoints
+            WHERE lower(name) LIKE '%deepseek%'
+              AND COALESCE(is_enabled, 0) = 1
+              AND COALESCE(api_key, '') != ''
+            ORDER BY updated_at DESC
+            LIMIT 1
+            """
+        ).fetchone()
+        if not row:
+            return None
+        return {
+            "name": row["name"],
+            "base_url": row["base_url"],
+            "api_key": row["api_key"],
+            "cached_models": row["cached_models"] or "deepseek-chat",
+        }
+    finally:
+        conn.close()
+
+
+def call_deepseek(endpoint: dict[str, str], prompt: dict[str, Any]) -> list[dict[str, str]]:
+    model = "deepseek-chat"
+    with contextlib.suppress(Exception):
+        cached = json.loads(endpoint.get("cached_models") or "[]")
+        if isinstance(cached, list) and cached:
+            model = cached[0]
+        elif isinstance(cached, str) and cached:
+            model = cached
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": "Return strict JSON only. No markdown. Do not reveal secrets."},
+            {"role": "user", "content": json.dumps(prompt, ensure_ascii=False)},
+        ],
+        "temperature": 0.55,
+        "max_tokens": 4500,
+    }
+    req = request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=120) as resp:
+        body = json.loads(resp.read().decode("utf-8"))
+    content = body["choices"][0]["message"]["content"]
+    cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", clean(content), flags=re.I | re.S)
+    if not cleaned.startswith("{"):
+        match = re.search(r"\{.*\}", cleaned, flags=re.S)
+        if match:
+            cleaned = match.group(0)
+    parsed = json.loads(cleaned)
+    rows = parsed.get("rows", [])
+    return [row for row in rows if isinstance(row, dict)]
+
+
+def teacher_variants(anchor: dict[str, Any], count: int, endpoint: dict[str, str] | None) -> list[dict[str, str]]:
+    fallback = [{"user": user, "final": anchor["final"]} for user in anchor["users"]]
+    while len(fallback) < count:
+        fallback.append({
+            "user": anchor["users"][len(fallback) % len(anchor["users"])],
+            "final": anchor["final"],
+        })
+    if endpoint is None:
+        return fallback[:count]
+    prompt = {
+        "task": "Generate varied SFT phrasings for an Odysseus web tool-use model.",
+        "count": count,
+        "topic": anchor["topic"],
+        "source_failure": "Current model often searched correctly but returned empty text, clipped snippets, stale query terms, or failed to synthesize the actual answer.",
+        "requirements": [
+            "Return JSON object with rows list.",
+            "Each row has user and final only.",
+            "User should be casual and varied; include some short phrasing and mild typos.",
+            "Final must be concise, direct, and answer from evidence.",
+            "Final must not mention snippets, links, sources, or WEB SEARCH RESULTS.",
+            "Do not include private names, emails, secrets, or API keys.",
+        ],
+        "ideal_query": anchor["query"],
+        "prior_user": anchor.get("prior_user", ""),
+        "evidence": [snippet for _title, snippet in anchor["rows"]],
+        "must_include_one_of": anchor["answer_any"],
+        "must_include_one_of_second_group": anchor["answer_any_2"],
+        "example_final_style": anchor["final"],
+    }
+    with contextlib.suppress(Exception):
+        rows = call_deepseek(endpoint, prompt)
+        valid: list[dict[str, str]] = []
+        for row in rows:
+            user = clean(row.get("user"))
+            final = clean(row.get("final"))
+            if len(user.split()) >= 3 and final and not re.search(r"WEB SEARCH RESULTS|```sources|links?|snippet", final, re.I):
+                valid.append({"user": user, "final": final})
+        if len(valid) >= max(3, count // 2):
+            return (valid + fallback)[:count]
+    return fallback[:count]
+
+
+def sft_row(category: str, messages: list[dict[str, Any]], expected_calls: int, metadata: dict[str, Any]) -> dict[str, Any]:
+    item = {
+        "messages": messages,
+        "tools": [WEB_SEARCH_TOOL] if expected_calls else [],
+        "generator": "deepseek_teacher_v56_broad_web",
+        "metadata": {
+            "category": category,
+            "split": "train_or_val",
+            "expected_tool_calls": expected_calls,
+            **metadata,
+        },
+    }
+    item["uuid"] = stable_id("ody_v56_broad_web", item)
+    return item
+
+
+def build_rows(endpoint: dict[str, str] | None, per_anchor: int, retry_per_anchor: int) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    raw: dict[str, Any] = {"provider": endpoint["name"] if endpoint else "deterministic_fallback", "anchors": []}
+    for anchor_idx, anchor in enumerate(ANCHORS):
+        variants = teacher_variants(anchor, per_anchor, endpoint)
+        raw["anchors"].append({"case_id": anchor["case_id"], "topic": anchor["topic"], "rows": variants})
+        for idx, variant in enumerate(variants):
+            call = tool_call("web_search", {"query": anchor["query"]}, f"synth_{anchor_idx}_{idx}")
+            messages: list[dict[str, Any]] = []
+            if anchor.get("prior_user"):
+                messages.extend([
+                    {"role": "user", "content": anchor["prior_user"]},
+                    {"role": "assistant", "content": anchor.get("prior_answer", "I can look that up or explain it briefly.")},
+                ])
+            messages.extend([
+                {"role": "user", "content": variant["user"]},
+                {"role": "assistant", "content": "", "tool_calls": [call]},
+                {"role": "tool", "tool_call_id": call["id"], "content": source_block(anchor["query"], anchor["rows"])},
+                {"role": "assistant", "content": variant["final"]},
+            ])
+            rows.append(sft_row(anchor["family"], messages, 1, {
+                "source_case_ids": [anchor["case_id"]],
+                "query_must_include": anchor["query"].split()[:6],
+                "answer_must_include": anchor["answer_any"] + anchor["answer_any_2"],
+            }))
+        for idx in range(retry_per_anchor):
+            bad_query, title, snippet = BAD_QUERY_ROWS[(anchor_idx + idx) % len(BAD_QUERY_ROWS)]
+            first = tool_call("web_search", {"query": bad_query}, f"retry_{anchor_idx}_{idx}_bad")
+            second = tool_call("web_search", {"query": anchor["query"]}, f"retry_{anchor_idx}_{idx}_good")
+            messages = [
+                {"role": "user", "content": anchor["users"][idx % len(anchor["users"])]},
+                {"role": "assistant", "content": "", "tool_calls": [first]},
+                {"role": "tool", "tool_call_id": first["id"], "content": source_block(bad_query, [(title, snippet)])},
+                {"role": "assistant", "content": "", "tool_calls": [second]},
+                {"role": "tool", "tool_call_id": second["id"], "content": source_block(anchor["query"], anchor["rows"])},
+                {"role": "assistant", "content": anchor["final"]},
+            ]
+            rows.append(sft_row("web_retry_bad_or_stale_query_then_synthesize", messages, 2, {
+                "source_case_ids": [anchor["case_id"]],
+                "bad_query": bad_query,
+                "query_must_include": anchor["query"].split()[:6],
+                "answer_must_include": anchor["answer_any"] + anchor["answer_any_2"],
+            }))
+    for idx, (user, final) in enumerate(NEGATIVE_NO_TOOL_ROWS):
+        rows.append(sft_row("negative_explicit_no_web", [
+            {"role": "user", "content": user},
+            {"role": "assistant", "content": final},
+        ], 0, {
+            "source_case_ids": ["web_no_tool_memory_answer_01"],
+            "forbidden_tools": ["web_search", "web_fetch"],
+        }))
+    return rows, raw
+
+
+def build_eval_cases() -> list[dict[str, Any]]:
+    cases: list[dict[str, Any]] = []
+    for idx, anchor in enumerate(ANCHORS):
+        case: dict[str, Any] = {
+            "id": f"v56_broad_web_anchor_{idx:02d}_{anchor['family']}",
+            "kind": "web",
+            "user": anchor["users"][0],
+            "expect_first_tool": "web_search",
+            "must_query_any": anchor["query"].split()[:3],
+            "must_answer_any": anchor["answer_any"],
+            "must_answer_any_2": anchor["answer_any_2"],
+            "forbidden_final": ["WEB SEARCH RESULTS", "```sources", "Here are links", "SEARCH RESULTS SUMMARY"],
+            "max_web_searches": 2,
+        }
+        if anchor.get("prior_user"):
+            case["prior_turns"] = [anchor["prior_user"]]
+        cases.append(case)
+    cases.append({
+        "id": "v56_broad_web_negative_no_lookup",
+        "kind": "chat",
+        "user": NEGATIVE_NO_TOOL_ROWS[0][0],
+        "expect_no_tool": True,
+        "forbidden_tools": ["web_search", "web_fetch"],
+        "must_answer_any": ["browser", "web", "pages"],
+    })
+    return cases
+
+
+def split_rows(rows: list[dict[str, Any]], val_every: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, item in enumerate(rows):
+        (val if idx % val_every == val_every - 1 else train).append(item)
+    return train, val
+
+
+def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("".join(json.dumps(item, ensure_ascii=True) + "\n" for item in rows), encoding="utf-8")
+
+
+def file_sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT)
+    parser.add_argument("--eval-out", type=Path, default=DEFAULT_EVAL_OUT)
+    parser.add_argument("--failure-targets", type=Path, default=DEFAULT_FAILURES)
+    parser.add_argument("--per-anchor", type=int, default=18)
+    parser.add_argument("--retry-per-anchor", type=int, default=4)
+    parser.add_argument("--val-every", type=int, default=6)
+    parser.add_argument("--seed", type=int, default=56)
+    args = parser.parse_args()
+
+    started = time.time()
+    rng = random.Random(args.seed)
+    endpoint = deepseek_endpoint()
+    rows, raw = build_rows(endpoint, args.per_anchor, args.retry_per_anchor)
+    rng.shuffle(rows)
+    train, val = split_rows(rows, args.val_every)
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    write_jsonl(args.out_dir / "train.jsonl", train)
+    write_jsonl(args.out_dir / "val.jsonl", val)
+    write_jsonl(args.out_dir / "all.jsonl", rows)
+    (args.out_dir / "raw_teacher.json").write_text(json.dumps(raw, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+    failure_target_payload: dict[str, Any] = {}
+    if args.failure_targets.exists():
+        failure_target_payload = json.loads(args.failure_targets.read_text(encoding="utf-8"))
+
+    eval_cases = build_eval_cases()
+    args.eval_out.parent.mkdir(parents=True, exist_ok=True)
+    args.eval_out.write_text(
+        json.dumps({
+            "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "generator": Path(__file__).name,
+            "source": "V55 broad web live-search gate failures.",
+            "source_failure_targets": str(args.failure_targets),
+            "cases": eval_cases,
+        }, ensure_ascii=True, indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+    categories = sorted({item["metadata"]["category"] for item in rows})
+    manifest = {
+        "name": args.out_dir.name,
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "provider": raw["provider"],
+        "elapsed_seconds": round(time.time() - started, 3),
+        "total_sft_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "heldout_cases": len(eval_cases),
+        "categories": {category: sum(1 for item in rows if item["metadata"]["category"] == category) for category in categories},
+        "source_eval": failure_target_payload.get("generated_from", str(args.failure_targets)),
+        "source_case_ids": [anchor["case_id"] for anchor in ANCHORS] + ["web_no_tool_memory_answer_01"],
+        "acceptance_target": (
+            "V56 must improve broad web live-search gate first; focused live regressions and old CRUD are regression checks."
+        ),
+        "files": {
+            "train": str(args.out_dir / "train.jsonl"),
+            "val": str(args.out_dir / "val.jsonl"),
+            "all": str(args.out_dir / "all.jsonl"),
+            "raw_teacher": str(args.out_dir / "raw_teacher.json"),
+            "heldout_eval": str(args.eval_out),
+            "failure_targets": str(args.failure_targets),
+        },
+    }
+    for key, value in list(manifest["files"].items()):
+        path = Path(value)
+        if path.exists():
+            manifest[f"{key}_sha256"] = file_sha256(path)
+    (args.out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({
+        "provider": manifest["provider"],
+        "total_sft_rows": manifest["total_sft_rows"],
+        "train_rows": manifest["train_rows"],
+        "val_rows": manifest["val_rows"],
+        "heldout_cases": manifest["heldout_cases"],
+        "categories": manifest["categories"],
+        "source_case_ids": manifest["source_case_ids"],
+    }, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_v61_app_route_web_rows.py

@@ -0,0 +1,394 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import time
+from pathlib import Path
+from typing import Any
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_ACTUALS = REPO_ROOT / "data/evals/ody_search_teacher_pipeline_20260821/deepseek_actual/actual_results.json"
+DEFAULT_EDITS = Path("/home/pewds/odysseus-finetune/data/teacher_live_gaps/odysseus_v58_teacher_edited_search_traces_20260821/edits.json")
+DEFAULT_OUT_DIR = Path("/home/pewds/odysseus-finetune/data/teacher_live_gaps/odysseus_v61_app_route_web_post_tool_20260821")
+
+WEB_TOOLS = {"web_search", "web_fetch"}
+WEB_NUDGE = (
+    "You just received web_search results as untrusted evidence. "
+    "Answer the user's question now in concise prose using the "
+    "useful snippets or fetched page content. If the results are "
+    "off-topic or do not contain the answer, either call web_search "
+    "once with better terms or say that the search did not provide "
+    "enough clear evidence. Do not output the raw source list or "
+    "the web_search wrapper."
+)
+FORBIDDEN_FINAL_RE = re.compile(
+    r"WEB SEARCH RESULTS|```sources|\b\d+\s+Web sources\b|from the search results|"
+    r"results indicate|returned snippets|top results|i searched|search results summary|"
+    r"fetched page content|\[CONTENT\s+\d+\]",
+    re.IGNORECASE,
+)
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def normalize_args(tool: str, args: Any) -> dict[str, Any]:
+    if isinstance(args, dict):
+        return dict(args)
+    if isinstance(args, str):
+        text = args.strip()
+        if text.startswith("{"):
+            try:
+                parsed = json.loads(text)
+                if isinstance(parsed, dict):
+                    return parsed
+            except json.JSONDecodeError:
+                pass
+        return {"query": text} if tool == "web_search" else {"url": text}
+    return {}
+
+
+def compact_tool_output(text: str, max_chars: int) -> str:
+    text = re.sub(r"\r\n?", "\n", str(text or "")).strip()
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    if len(text) <= max_chars:
+        return text
+
+    sources = ""
+    if text.startswith("```sources"):
+        end = text.find("```", 3)
+        if end != -1:
+            sources = text[: end + 3].strip()
+    summary = ""
+    match = re.search(
+        r"SEARCH RESULTS SUMMARY:\n[-]+\n(?P<body>.*?)(?:\n={10,}|\Z)",
+        text,
+        re.DOTALL,
+    )
+    if match:
+        summary = "SEARCH RESULTS SUMMARY:\n" + match.group("body").strip()
+    fetched = ""
+    match = re.search(
+        r"FETCHED PAGE CONTENT:\n[-]+\n(?P<body>.*?)(?:\n={10,}|\Z)",
+        text,
+        re.DOTALL,
+    )
+    if match:
+        fetched = "FETCHED PAGE CONTENT:\n" + match.group("body").strip()
+    parts = [part for part in (sources, summary[:2200], fetched[:1800]) if part]
+    compact = "\n\n".join(parts).strip() or text[:max_chars].rstrip()
+    return compact[:max_chars].rstrip()
+
+
+def load_results(path: Path) -> list[dict[str, Any]]:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    return list(payload.get("results") or [])
+
+
+def load_edited_finals(path: Path) -> dict[str, dict[str, Any]]:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    finals: dict[str, dict[str, Any]] = {}
+    for item in payload.get("edits") or []:
+        if item.get("accepted") is not True:
+            continue
+        edited = item.get("edited") or {}
+        final = re.sub(r"\s+", " ", str(edited.get("final") or "")).strip()
+        if not final or FORBIDDEN_FINAL_RE.search(final):
+            continue
+        finals[str(item.get("id"))] = {
+            "final": final,
+            "trace": edited.get("trace") or [],
+            "reason": edited.get("reason") or "",
+        }
+    return finals
+
+
+def first_web_step(result: dict[str, Any], max_chars: int) -> dict[str, Any] | None:
+    calls = result.get("tool_calls") or []
+    outputs = result.get("tool_outputs") or []
+    for idx, call in enumerate(calls):
+        tool = call.get("tool") or call.get("name")
+        if tool not in WEB_TOOLS:
+            continue
+        if idx >= len(outputs):
+            continue
+        output = outputs[idx]
+        args = normalize_args(tool, call.get("args"))
+        if tool == "web_search" and not args.get("query"):
+            continue
+        if tool == "web_fetch" and not args.get("url"):
+            continue
+        content = compact_tool_output(output.get("output") or "", max_chars=max_chars)
+        if not content:
+            continue
+        return {"tool": tool, "args": args, "output": content}
+    return None
+
+
+def messages_for_user(result: dict[str, Any]) -> list[dict[str, Any]]:
+    messages: list[dict[str, Any]] = [{"role": "system", "content": WEB_NUDGE}]
+    for turn in result.get("prior_turns") or []:
+        if isinstance(turn, dict) and turn.get("user"):
+            messages.append({"role": "user", "content": str(turn["user"])})
+            if turn.get("assistant"):
+                messages.append({"role": "assistant", "content": str(turn["assistant"])})
+        elif isinstance(turn, str) and turn.strip():
+            messages.append({"role": "user", "content": turn.strip()})
+    messages.append({"role": "user", "content": str(result.get("user") or "")})
+    return messages
+
+
+def append_tool_call(messages: list[dict[str, Any]], source_id: str, step: dict[str, Any], idx: int = 0) -> str:
+    call_id = f"call_{source_id}_{idx}"
+    messages.append({
+        "role": "assistant",
+        "content": None,
+        "tool_calls": [{
+            "id": call_id,
+            "type": "function",
+            "function": {
+                "name": step["tool"],
+                "arguments": json.dumps(step["args"], separators=(",", ":"), ensure_ascii=True),
+            },
+        }],
+    })
+    messages.append({"role": "tool", "tool_call_id": call_id, "content": step["output"]})
+    return call_id
+
+
+def build_answer_row(result: dict[str, Any], step: dict[str, Any], final: str, family: str, repeat: int) -> dict[str, Any] | None:
+    final = re.sub(r"\s+", " ", final).strip()
+    if not final or len(final) > 900 or FORBIDDEN_FINAL_RE.search(final):
+        return None
+    messages = messages_for_user(result)
+    append_tool_call(messages, str(result.get("id") or "web"), step, 0)
+    messages.append({"role": "assistant", "content": final})
+    row = {
+        "messages": messages,
+        "generator": "odysseus_v61_app_route_web_post_tool",
+        "metadata": {
+            "source_result_id": result.get("id"),
+            "family": family,
+            "repeat": repeat,
+            "first_tool": step["tool"],
+            "first_args": step["args"],
+        },
+    }
+    row["uuid"] = stable_id("ody_v61_app_route_web", row)
+    return row
+
+
+def build_retry_row(
+    result: dict[str, Any],
+    bad_step: dict[str, Any],
+    retry_query: str,
+    final: str,
+    retry_output: str | None,
+    repeat: int,
+) -> dict[str, Any] | None:
+    messages = messages_for_user(result)
+    source_id = str(result.get("id") or "retry")
+    append_tool_call(messages, source_id, bad_step, 0)
+    messages.append({
+        "role": "assistant",
+        "content": None,
+        "tool_calls": [{
+            "id": f"call_{source_id}_retry",
+            "type": "function",
+            "function": {
+                "name": "web_search",
+                "arguments": json.dumps({"query": retry_query}, separators=(",", ":"), ensure_ascii=True),
+            },
+        }],
+    })
+    if retry_output:
+        messages.append({
+            "role": "tool",
+            "tool_call_id": f"call_{source_id}_retry",
+            "content": retry_output,
+        })
+        final = re.sub(r"\s+", " ", final).strip()
+        if not final or len(final) > 900 or FORBIDDEN_FINAL_RE.search(final):
+            return None
+        messages.append({"role": "assistant", "content": final})
+    row = {
+        "messages": messages,
+        "generator": "odysseus_v61_app_route_web_retry",
+        "metadata": {
+            "source_result_id": result.get("id"),
+            "family": "retry_off_target_then_answer" if retry_output else "retry_off_target",
+            "repeat": repeat,
+            "bad_args": bad_step["args"],
+            "retry_query": retry_query,
+        },
+    }
+    row["uuid"] = stable_id("ody_v61_app_route_web", row)
+    return row
+
+
+def split_rows(rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, row in enumerate(rows):
+        (val if idx % 10 == 9 else train).append(row)
+    return train, val
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--actual", type=Path, default=DEFAULT_ACTUALS)
+    parser.add_argument("--edits", type=Path, default=DEFAULT_EDITS)
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT_DIR)
+    parser.add_argument("--max-output-chars", type=int, default=4200)
+    parser.add_argument("--answer-repeat", type=int, default=4)
+    parser.add_argument("--retry-repeat", type=int, default=8)
+    parser.add_argument("--retry-output-json", type=Path)
+    args = parser.parse_args()
+
+    results = load_results(args.actual)
+    finals = load_edited_finals(args.edits)
+    retry_outputs = {}
+    if args.retry_output_json and args.retry_output_json.exists():
+        retry_outputs = json.loads(args.retry_output_json.read_text(encoding="utf-8"))
+
+    rows: list[dict[str, Any]] = []
+    audit: list[dict[str, Any]] = []
+    family_counts: dict[str, int] = {}
+    for result in results:
+        result_id = str(result.get("id") or "")
+        if result.get("kind") != "web" or result_id not in finals:
+            continue
+        step = first_web_step(result, args.max_output_chars)
+        if not step or step["tool"] != "web_search":
+            continue
+        final = finals[result_id]["final"]
+        accepted = 0
+        for rep in range(args.answer_repeat):
+            row = build_answer_row(result, step, final, "answer_after_first_web_search", rep)
+            if row:
+                rows.append(row)
+                accepted += 1
+                family_counts[row["metadata"]["family"]] = family_counts.get(row["metadata"]["family"], 0) + 1
+        audit.append({
+            "id": result_id,
+            "family": "answer_after_first_web_search",
+            "accepted_rows": accepted,
+            "first_args": step["args"],
+            "final": final,
+        })
+
+    hard_path = REPO_ROOT / "data/evals/ody_v57_quick_live_search_cases_20260821/v60_container_final_event_run_20260821_2123/actual_results.json"
+    hard_by_id = {str(item.get("id")): item for item in load_results(hard_path)} if hard_path.exists() else {}
+
+    hard_answer_specs = [
+        {
+            "id": "v57_sweden_gas_price",
+            "final": "Gasoline in Sweden is roughly 16.4-16.6 SEK per liter based on the latest fuel-price results. The exact price varies by station and fuel grade, but that is the current ballpark for petrol/gas per liter.",
+        },
+    ]
+    for spec in hard_answer_specs:
+        result = hard_by_id.get(spec["id"])
+        if not result:
+            continue
+        step = first_web_step(result, args.max_output_chars)
+        if not step:
+            continue
+        accepted = 0
+        for rep in range(args.retry_repeat):
+            row = build_answer_row(result, step, spec["final"], "hard_answer_after_first_web_search", rep)
+            if row:
+                rows.append(row)
+                accepted += 1
+                family_counts[row["metadata"]["family"]] = family_counts.get(row["metadata"]["family"], 0) + 1
+        audit.append({
+            "id": spec["id"],
+            "family": "hard_answer_after_first_web_search",
+            "accepted_rows": accepted,
+            "first_args": step["args"],
+            "final": spec["final"],
+        })
+
+    hard_retry_specs = [
+        {
+            "id": "v57_norway_coordinates",
+            "retry_query": "Norway country geographic coordinates latitude longitude",
+            "final": "Norway is in Northern Europe on the Scandinavian Peninsula. Its commonly cited country coordinates are about 62°N, 10°E.",
+        },
+        {
+            "id": "v57_snail_touch_followup",
+            "retry_query": "is it safe to touch garden snails after they foam mucus scared wash hands",
+            "final": "Usually yes, it is okay to gently touch a snail, even if it is foaming from stress, but avoid your eyes or mouth and wash your hands afterward. Do not handle it roughly, and leave it alone if it keeps bubbling or retracting.",
+        },
+    ]
+    if hard_by_id:
+        for spec in hard_retry_specs:
+            result = hard_by_id.get(spec["id"])
+            if not result:
+                continue
+            step = first_web_step(result, args.max_output_chars)
+            if not step:
+                continue
+            retry_output = retry_outputs.get(spec["retry_query"])
+            for rep in range(args.retry_repeat):
+                row = build_retry_row(
+                    result,
+                    step,
+                    spec["retry_query"],
+                    spec["final"],
+                    retry_output,
+                    rep,
+                )
+                if row:
+                    rows.append(row)
+                    family_counts[row["metadata"]["family"]] = family_counts.get(row["metadata"]["family"], 0) + 1
+            audit.append({
+                "id": spec["id"],
+                "family": "retry_off_target_then_answer" if retry_output else "retry_off_target",
+                "retry_query": spec["retry_query"],
+                "has_retry_output": bool(retry_output),
+            })
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    train, val = split_rows(rows)
+    for name, subset in (("all.jsonl", rows), ("train.jsonl", train), ("val.jsonl", val)):
+        (args.out_dir / name).write_text(
+            "".join(json.dumps(row, ensure_ascii=True) + "\n" for row in subset),
+            encoding="utf-8",
+        )
+    (args.out_dir / "audit.json").write_text(
+        json.dumps({"audit": audit}, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    manifest = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "source_actual": str(args.actual),
+        "source_edits": str(args.edits),
+        "accepted_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "family_counts": family_counts,
+        "goal": "train Qwen to continue correctly after Odysseus app-route web_search tool output plus system nudge",
+        "forbidden_final_regex": FORBIDDEN_FINAL_RE.pattern,
+        "files": {
+            "train": str(args.out_dir / "train.jsonl"),
+            "val": str(args.out_dir / "val.jsonl"),
+            "all": str(args.out_dir / "all.jsonl"),
+            "audit": str(args.out_dir / "audit.json"),
+        },
+    }
+    (args.out_dir / "manifest.json").write_text(
+        json.dumps(manifest, ensure_ascii=True, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    print(json.dumps(manifest, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_v62_teacher_trace_web_rows.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import time
+from pathlib import Path
+from typing import Any
+
+
+DEFAULT_EDITS = Path("/home/pewds/odysseus-finetune/data/teacher_live_gaps/odysseus_v58_teacher_edited_search_traces_20260821/edits.json")
+DEFAULT_LIVE_ACTUAL = Path("/home/pewds/odysseus-cookbook-fresh/data/evals/ody_v57_quick_live_search_cases_20260821/v61_app_route_web_run_20260821_2204/actual_results.json")
+DEFAULT_OUT_DIR = Path("/home/pewds/odysseus-finetune/data/teacher_live_gaps/odysseus_v62_teacher_trace_web_synthesis_20260821")
+
+WEB_NUDGE = (
+    "You are continuing after public web tool results. Use the tool evidence "
+    "to answer the user's question directly in concise prose. If the first "
+    "search result is off-target, make at most one or two better web_search "
+    "calls, then answer from the best evidence. Do not output raw source "
+    "lists, tool wrappers, or meta-commentary."
+)
+WEB_TOOLS = {"web_search", "web_fetch"}
+FORBIDDEN_FINAL_RE = re.compile(
+    r"WEB SEARCH RESULTS|```sources|\b\d+\s+Web sources\b|from the search results|"
+    r"results indicate|returned snippets|top results|i searched|search results summary|"
+    r"fetched page content|\[CONTENT\s+\d+\]|the user asked|i should",
+    re.IGNORECASE,
+)
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def clean_final(text: str) -> str:
+    text = re.sub(r"\s+", " ", str(text or "")).strip()
+    return text
+
+
+def normalize_args(tool: str, args: Any) -> dict[str, Any]:
+    if isinstance(args, dict):
+        return dict(args)
+    if isinstance(args, str):
+        text = args.strip()
+        if text.startswith("{"):
+            try:
+                parsed = json.loads(text)
+                if isinstance(parsed, dict):
+                    return parsed
+            except json.JSONDecodeError:
+                pass
+        return {"query": text} if tool == "web_search" else {"url": text}
+    return {}
+
+
+def append_tool_step(messages: list[dict[str, Any]], source_id: str, idx: int, step: dict[str, Any]) -> bool:
+    tool = str(step.get("tool") or "")
+    if tool not in WEB_TOOLS:
+        return False
+    args = normalize_args(tool, step.get("args") or {})
+    if tool == "web_search" and not str(args.get("query") or "").strip():
+        return False
+    if tool == "web_fetch" and not str(args.get("url") or "").strip():
+        return False
+    output = re.sub(r"\s+", " ", str(step.get("output") or "")).strip()
+    if not output:
+        return False
+    output = output[:2200].rstrip()
+    call_id = f"call_{source_id}_{idx}"
+    messages.append({
+        "role": "assistant",
+        "content": None,
+        "tool_calls": [{
+            "id": call_id,
+            "type": "function",
+            "function": {
+                "name": tool,
+                "arguments": json.dumps(args, separators=(",", ":"), ensure_ascii=True),
+            },
+        }],
+    })
+    messages.append({"role": "tool", "tool_call_id": call_id, "content": output})
+    return True
+
+
+def build_trace_row(item: dict[str, Any], repeat: int) -> dict[str, Any] | None:
+    edited = item.get("edited") or {}
+    if item.get("accepted") is not True or edited.get("should_train") is not True:
+        return None
+    trace = edited.get("trace") or []
+    final = clean_final(edited.get("final") or "")
+    if not isinstance(trace, list) or not trace or len(trace) > 3:
+        return None
+    if not final or len(final) > 900 or FORBIDDEN_FINAL_RE.search(final):
+        return None
+    source_id = str(item.get("id") or "teacher")
+    messages: list[dict[str, Any]] = [{"role": "system", "content": WEB_NUDGE}]
+    messages.append({"role": "user", "content": str(item.get("user") or "")})
+    for idx, step in enumerate(trace):
+        if not append_tool_step(messages, source_id, idx, step):
+            return None
+    messages.append({"role": "assistant", "content": final})
+    row = {
+        "messages": messages,
+        "generator": "odysseus_v62_teacher_trace_web_synthesis",
+        "metadata": {
+            "family": "teacher_minimal_trace_then_answer",
+            "source_result_id": source_id,
+            "repeat": repeat,
+            "trace_tools": [str(step.get("tool") or "") for step in trace],
+            "teacher_reason": edited.get("reason") or "",
+        },
+    }
+    row["uuid"] = stable_id("ody_v62_teacher_trace_web", row)
+    return row
+
+
+def first_web_output(result: dict[str, Any]) -> str:
+    for output in result.get("tool_outputs") or []:
+        if output.get("tool") == "web_search":
+            text = str(output.get("output") or "")
+            return re.sub(r"\r\n?", "\n", text).strip()[:4200].rstrip()
+    return ""
+
+
+def live_hard_specs(actual_by_id: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
+    specs: list[dict[str, Any]] = []
+    norway = actual_by_id.get("v57_norway_coordinates")
+    if norway:
+        specs.append({
+            "id": "v57_norway_coordinates_country_not_capital",
+            "user": "where is norway coordinates",
+            "trace": [
+                {
+                    "tool": "web_search",
+                    "args": {"query": "Norway country coordinates latitude longitude"},
+                    "output": first_web_output(norway) or "Search evidence identifies Norway as a country in Northern Europe on the Scandinavian Peninsula. Common country coordinates are approximately 62° N latitude and 10° E longitude.",
+                }
+            ],
+            "final": "Norway is in Northern Europe on the Scandinavian Peninsula. The commonly cited country coordinates are about 62°N, 10°E.",
+            "family": "live_hard_country_coordinates_answer",
+        })
+    snail = actual_by_id.get("v57_snail_touch_followup")
+    if snail:
+        specs.append({
+            "id": "v57_snail_touch_contextual_followup",
+            "prior": [
+                ("user", "why does snails bubble up when they are scared"),
+                ("assistant", "Snails bubble because air gets trapped in their mucus, making foam. That usually happens when they are stressed, irritated, disturbed, defending themselves, or trying to hold moisture."),
+            ],
+            "user": "is it safe to touch",
+            "trace": [
+                {
+                    "tool": "web_search",
+                    "args": {"query": "is it safe to touch garden snails mucus wash hands"},
+                    "output": first_web_output(snail) or "Search evidence says snail mucus may irritate skin for some people and snails can carry germs, so gentle handling is usually okay but hands should be washed afterward and contact with eyes or mouth should be avoided.",
+                }
+            ],
+            "final": "Usually yes, it is okay to gently touch a snail, even if it is foaming from stress. Be gentle, avoid touching your eyes or mouth, and wash your hands afterward.",
+            "family": "live_hard_contextual_followup_answer",
+        })
+    return specs
+
+
+def build_live_row(spec: dict[str, Any], repeat: int) -> dict[str, Any] | None:
+    final = clean_final(spec.get("final") or "")
+    if not final or FORBIDDEN_FINAL_RE.search(final):
+        return None
+    messages: list[dict[str, Any]] = [{"role": "system", "content": WEB_NUDGE}]
+    for role, content in spec.get("prior") or []:
+        messages.append({"role": role, "content": content})
+    messages.append({"role": "user", "content": str(spec.get("user") or "")})
+    for idx, step in enumerate(spec.get("trace") or []):
+        if not append_tool_step(messages, str(spec.get("id") or "live"), idx, step):
+            return None
+    messages.append({"role": "assistant", "content": final})
+    row = {
+        "messages": messages,
+        "generator": "odysseus_v62_live_hard_web_synthesis",
+        "metadata": {
+            "family": spec.get("family") or "live_hard",
+            "source_result_id": spec.get("id"),
+            "repeat": repeat,
+        },
+    }
+    row["uuid"] = stable_id("ody_v62_teacher_trace_web", row)
+    return row
+
+
+def split_rows(rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, row in enumerate(rows):
+        (val if idx % 10 == 9 else train).append(row)
+    return train, val
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--edits", type=Path, default=DEFAULT_EDITS)
+    parser.add_argument("--live-actual", type=Path, default=DEFAULT_LIVE_ACTUAL)
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT_DIR)
+    parser.add_argument("--teacher-repeat", type=int, default=6)
+    parser.add_argument("--live-repeat", type=int, default=20)
+    args = parser.parse_args()
+
+    edits = json.loads(args.edits.read_text(encoding="utf-8")).get("edits") or []
+    rows: list[dict[str, Any]] = []
+    audit: list[dict[str, Any]] = []
+    family_counts: dict[str, int] = {}
+    accepted_sources = 0
+    for item in edits:
+        accepted_for_source = 0
+        for rep in range(args.teacher_repeat):
+            row = build_trace_row(item, rep)
+            if row:
+                rows.append(row)
+                accepted_for_source += 1
+                family = row["metadata"]["family"]
+                family_counts[family] = family_counts.get(family, 0) + 1
+        if accepted_for_source:
+            accepted_sources += 1
+            audit.append({
+                "id": item.get("id"),
+                "family": "teacher_minimal_trace_then_answer",
+                "rows": accepted_for_source,
+                "user": item.get("user"),
+            })
+
+    live_payload = json.loads(args.live_actual.read_text(encoding="utf-8")) if args.live_actual.exists() else {"results": []}
+    actual_by_id = {str(item.get("id") or ""): item for item in live_payload.get("results") or []}
+    for spec in live_hard_specs(actual_by_id):
+        accepted_for_spec = 0
+        for rep in range(args.live_repeat):
+            row = build_live_row(spec, rep)
+            if row:
+                rows.append(row)
+                accepted_for_spec += 1
+                family = row["metadata"]["family"]
+                family_counts[family] = family_counts.get(family, 0) + 1
+        audit.append({
+            "id": spec.get("id"),
+            "family": spec.get("family"),
+            "rows": accepted_for_spec,
+            "user": spec.get("user"),
+        })
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    train, val = split_rows(rows)
+    for name, subset in (("all.jsonl", rows), ("train.jsonl", train), ("val.jsonl", val)):
+        (args.out_dir / name).write_text(
+            "".join(json.dumps(row, ensure_ascii=True) + "\n" for row in subset),
+            encoding="utf-8",
+        )
+    (args.out_dir / "audit.json").write_text(
+        json.dumps({"audit": audit}, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    manifest = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "source_edits": str(args.edits),
+        "source_live_actual": str(args.live_actual),
+        "accepted_teacher_sources": accepted_sources,
+        "accepted_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "family_counts": family_counts,
+        "goal": "teach app-route web continuations to search minimally and synthesize final answers",
+        "files": {
+            "train": str(args.out_dir / "train.jsonl"),
+            "val": str(args.out_dir / "val.jsonl"),
+            "all": str(args.out_dir / "all.jsonl"),
+            "audit": str(args.out_dir / "audit.json"),
+        },
+    }
+    (args.out_dir / "manifest.json").write_text(
+        json.dumps(manifest, ensure_ascii=True, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    print(json.dumps(manifest, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_web_teacher_sft.py

@@ -0,0 +1,500 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sqlite3
+import sys
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+
+DEFAULT_OUT = Path("/home/pewds/odysseus-finetune/data/teacher_web_synthesis/odysseus_web_teacher_v1_20260821")
+DEFAULT_EVAL_OUT = REPO_ROOT / "data/evals/ody_web_teacher_heldout_v1_20260821/cases.json"
+
+WEB_SEARCH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "web_search",
+        "description": "Search the web for current or source-backed information.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "query": {"type": "string"},
+                "time_filter": {"type": "string", "enum": ["day", "week", "month", "year"]},
+            },
+            "required": ["query"],
+        },
+    },
+}
+
+
+FAMILIES: list[dict[str, Any]] = [
+    {
+        "name": "web_direct_answer",
+        "train_count": 45,
+        "heldout_count": 18,
+        "instruction": (
+            "User asks to look up a public fact, explanation, price, exchange rate, product safety issue, "
+            "local cost, regulation, or simple science reason. The ideal first tool is web_search with a "
+            "specific query. After tool output, assistant synthesizes a short answer, never just links."
+        ),
+    },
+    {
+        "name": "web_bad_first_search_recovery",
+        "train_count": 30,
+        "heldout_count": 12,
+        "instruction": (
+            "The first web_search result is low evidence or wrong-intent dictionary/news noise. The ideal next "
+            "assistant action is a second web_search with better terms; final answer synthesizes only after useful evidence."
+        ),
+    },
+    {
+        "name": "web_unit_conversion",
+        "train_count": 25,
+        "heldout_count": 10,
+        "instruction": (
+            "User asks for a looked-up price/rate converted into another unit or currency. The answer should show "
+            "the approximate calculation using evidence in the simulated search result."
+        ),
+    },
+    {
+        "name": "web_no_tool_boundary",
+        "train_count": 10,
+        "heldout_count": 5,
+        "instruction": (
+            "User explicitly says not to search, or asks a stable definition/concept. The assistant should answer directly "
+            "with no tool call."
+        ),
+    },
+    {
+        "name": "web_search_failure",
+        "train_count": 10,
+        "heldout_count": 5,
+        "instruction": (
+            "Search results remain irrelevant or insufficient after reasonable query terms. The final answer should say "
+            "there is not enough clear evidence, not dump source listings."
+        ),
+    },
+]
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def tool_call(name: str, arguments: dict[str, Any], suffix: str) -> dict[str, Any]:
+    return {
+        "id": f"call_{suffix}",
+        "type": "function",
+        "function": {
+            "name": name,
+            "arguments": json.dumps(arguments, separators=(",", ":"), ensure_ascii=True),
+        },
+    }
+
+
+def deepseek_endpoint() -> dict[str, str]:
+    api_key = os.environ.get("DEEPSEEK_API_KEY", "").strip()
+    if api_key:
+        return {
+            "name": "env-deepseek",
+            "base_url": os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1"),
+            "api_key": api_key,
+            "cached_models": os.environ.get("DEEPSEEK_MODEL", "deepseek-chat"),
+        }
+
+    db_path = REPO_ROOT / "data/app.db"
+    if db_path.exists():
+        conn = sqlite3.connect(str(db_path))
+        try:
+            conn.row_factory = sqlite3.Row
+            row = conn.execute(
+                """
+                SELECT name, base_url, api_key, cached_models
+                FROM model_endpoints
+                WHERE lower(name) LIKE '%deepseek%'
+                  AND COALESCE(is_enabled, 0) = 1
+                  AND COALESCE(api_key, '') != ''
+                ORDER BY updated_at DESC
+                LIMIT 1
+                """
+            ).fetchone()
+            if row:
+                return {
+                    "name": row["name"],
+                    "base_url": row["base_url"],
+                    "api_key": row["api_key"],
+                    "cached_models": row["cached_models"] or "",
+                }
+        finally:
+            conn.close()
+
+        auth_path = REPO_ROOT / "data/auth.json"
+        if auth_path.exists():
+            auth = json.loads(auth_path.read_text(encoding="utf-8"))
+            endpoints = auth.get("model_endpoints") or auth.get("providers") or []
+            for item in endpoints if isinstance(endpoints, list) else []:
+                name = str(item.get("name") or item.get("provider") or "").lower()
+                api_key = str(item.get("api_key") or item.get("apiKey") or "").strip()
+                if "deepseek" in name and api_key:
+                    return {
+                        "name": name,
+                        "base_url": item.get("base_url") or item.get("baseUrl") or "https://api.deepseek.com/v1",
+                        "api_key": api_key,
+                        "cached_models": item.get("cached_models") or item.get("model") or "deepseek-chat",
+                    }
+
+    raise RuntimeError("no enabled DeepSeek endpoint with API key and DEEPSEEK_API_KEY is unset")
+
+
+def call_deepseek(endpoint: dict[str, str], prompt: dict[str, Any], max_tokens: int = 8000) -> dict[str, Any]:
+    model = "deepseek-chat"
+    try:
+        cached = json.loads(endpoint["cached_models"] or "[]")
+        if cached:
+            model = cached[0]
+    except json.JSONDecodeError:
+        if endpoint.get("cached_models"):
+            model = endpoint["cached_models"]
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": "Return strict JSON only. No markdown, no commentary."},
+            {"role": "user", "content": json.dumps(prompt, ensure_ascii=False)},
+        ],
+        "temperature": 0.7,
+        "max_tokens": max_tokens,
+    }
+    req = request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=120) as resp:
+        body = json.loads(resp.read().decode("utf-8"))
+    content = body["choices"][0]["message"]["content"]
+    cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", (content or "").strip(), flags=re.I | re.S)
+    if not cleaned.startswith("{"):
+        match = re.search(r"\{.*\}", cleaned, flags=re.S)
+        if match:
+            cleaned = match.group(0)
+    return {"model": model, "content": json.loads(cleaned)}
+
+
+def teacher_prompt(family: dict[str, Any], count: int, batch: int) -> dict[str, Any]:
+    name = family["name"]
+    return {
+        "task": "Generate Odysseus web-search tool-use SFT specs.",
+        "current_date_context": "2026-08-21. Use Asia/Tokyo examples when a relative date matters.",
+        "family": name,
+        "count": count,
+        "batch": batch,
+        "family_instruction": family["instruction"],
+        "global_requirements": [
+            "Return JSON object with key rows: list.",
+            "Return exactly count rows.",
+            "Every row needs: user, ideal_query, evidence, final, query_must_include, answer_must_include.",
+            "For web_no_tool_boundary rows, ideal_query must be empty string and evidence must be empty string.",
+            "For web_bad_first_search_recovery rows, include bad_query and bad_evidence, then ideal_query/evidence/final.",
+            "For web_search_failure rows, evidence should be irrelevant or insufficient and final should say not enough clear evidence.",
+            "Do not include private names, private email data, or secrets.",
+            "Do not copy these instructions verbatim.",
+            "Use varied wording, typos, casual phrasing, and realistic user questions.",
+            "Make each user prompt unique from prior batches; vary topic, country, unit, and wording.",
+            "Do not make rows depend on exact live facts; simulated evidence is okay for behavior training.",
+            "Final answers must synthesize evidence in 1-4 sentences, with no raw source dump and no markdown source block.",
+        ],
+        "examples_to_cover_without_copying": [
+            "look up why a small animal is foaming/bubbling and explain",
+            "current commodity price per liter converted to EUR",
+            "why a device battery swells and what to do",
+            "why a food starter smells like acetone",
+            "latest/current exchange rate with a rough conversion",
+            "bad query returns dictionary pages, then better search terms are needed",
+        ],
+    }
+
+
+def clean_text(value: Any) -> str:
+    return re.sub(r"\s+", " ", str(value or "")).strip()
+
+
+def clean_terms(value: Any) -> list[str]:
+    if isinstance(value, str):
+        text = clean_text(value)
+        return [text] if text else []
+    if isinstance(value, list):
+        return [clean_text(item) for item in value if clean_text(item)]
+    return []
+
+
+def alternatives(term: str) -> list[str]:
+    return [part.strip() for part in re.split(r"[,/|]|\bor\b", term) if part.strip()] or [term]
+
+
+def valid_spec(family: str, item: Any) -> bool:
+    if not isinstance(item, dict):
+        return False
+    user = clean_text(item.get("user"))
+    final = clean_text(item.get("final"))
+    if len(user.split()) < 4 or len(user) > 220:
+        return False
+    if "WEB SEARCH RESULTS" in final or "```sources" in final or "Here are links" in final:
+        return False
+    if family == "web_no_tool_boundary":
+        return bool(final) and not clean_text(item.get("ideal_query"))
+    if not clean_text(item.get("ideal_query")):
+        return False
+    if family == "web_bad_first_search_recovery" and not clean_text(item.get("bad_query")):
+        return False
+    return bool(final)
+
+
+def build_sft_row(family: str, idx: int, spec: dict[str, Any], split: str) -> dict[str, Any]:
+    user = clean_text(spec["user"])
+    final = clean_text(spec["final"])
+    messages: list[dict[str, Any]] = [{"role": "user", "content": user}]
+    expected_calls = 0
+
+    if family == "web_no_tool_boundary":
+        messages.append({"role": "assistant", "content": final})
+    elif family == "web_bad_first_search_recovery":
+        bad_call = tool_call("web_search", {"query": clean_text(spec["bad_query"])}, f"{family}_{idx}_bad")
+        good_call = tool_call("web_search", {"query": clean_text(spec["ideal_query"])}, f"{family}_{idx}_good")
+        messages.extend(
+            [
+                {"role": "assistant", "content": "", "tool_calls": [bad_call]},
+                {
+                    "role": "tool",
+                    "tool_call_id": bad_call["id"],
+                    "content": clean_text(spec.get("bad_evidence"))
+                    or "Search results were mostly dictionary pages and did not answer the user's question.",
+                },
+                {"role": "assistant", "content": "", "tool_calls": [good_call]},
+                {
+                    "role": "tool",
+                    "tool_call_id": good_call["id"],
+                    "content": clean_text(spec.get("evidence")),
+                },
+                {"role": "assistant", "content": final},
+            ]
+        )
+        expected_calls = 2
+    else:
+        call = tool_call("web_search", {"query": clean_text(spec["ideal_query"])}, f"{family}_{idx}")
+        messages.extend(
+            [
+                {"role": "assistant", "content": "", "tool_calls": [call]},
+                {"role": "tool", "tool_call_id": call["id"], "content": clean_text(spec.get("evidence"))},
+                {"role": "assistant", "content": final},
+            ]
+        )
+        expected_calls = 1
+
+    row = {
+        "messages": messages,
+        "tools": [] if family == "web_no_tool_boundary" else [WEB_SEARCH_TOOL],
+        "generator": "deepseek_teacher_web_synthesis_v1",
+        "metadata": {
+            "category": family,
+            "split": split,
+            "expected_tool_calls": expected_calls,
+            "query_must_include": clean_terms(spec.get("query_must_include")),
+            "answer_must_include": clean_terms(spec.get("answer_must_include")),
+        },
+    }
+    row["uuid"] = stable_id("ody_web_teacher", row)
+    return row
+
+
+def build_eval_case(family: str, idx: int, spec: dict[str, Any]) -> dict[str, Any]:
+    user = clean_text(spec["user"])
+    case: dict[str, Any] = {
+        "id": f"teacher_web_{family}_{idx:02d}",
+        "kind": "negative_web" if family == "web_no_tool_boundary" else "web",
+        "user": user,
+        "deepseek_family": family,
+        "forbidden_final": ["WEB SEARCH RESULTS", "```sources", "Here are links for that topic"],
+    }
+    answer_terms = clean_terms(spec.get("answer_must_include"))
+    query_terms = clean_terms(spec.get("query_must_include"))
+    if family == "web_no_tool_boundary":
+        case.update({"expect_no_tool": True, "forbidden_tools": ["web_search", "web_fetch"]})
+    else:
+        case.update(
+            {
+                "expect_first_tool": "web_search",
+                "forbidden_query_any": ["official links", "dictionary", "wikipedia official", "cambridge", "merriam"],
+            }
+        )
+        for i, term in enumerate(query_terms[:4], start=1):
+            key = "must_query_any" if i == 1 else f"must_query_any_{i}"
+            case[key] = alternatives(term)
+        if family == "web_bad_first_search_recovery":
+            case["min_web_searches"] = 2
+        else:
+            case["max_web_searches"] = 1
+    for i, term in enumerate(answer_terms[:2], start=1):
+        key = "must_answer_any" if i == 1 else f"must_answer_any_{i}"
+        case[key] = alternatives(term)
+    return case
+
+
+def split_rows(rows: list[dict[str, Any]], val_every: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, row in enumerate(rows):
+        (val if idx % val_every == val_every - 1 else train).append(row)
+    return train, val
+
+
+def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("".join(json.dumps(row, ensure_ascii=True) + "\n" for row in rows), encoding="utf-8")
+
+
+def file_sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT)
+    parser.add_argument("--eval-out", type=Path, default=DEFAULT_EVAL_OUT)
+    parser.add_argument("--val-every", type=int, default=6)
+    args = parser.parse_args()
+
+    endpoint = deepseek_endpoint()
+    started = time.time()
+    raw: dict[str, Any] = {}
+    sft_rows: list[dict[str, Any]] = []
+    eval_cases: list[dict[str, Any]] = []
+    seen_users: set[str] = set()
+    model = ""
+
+    for family in FAMILIES:
+        needed = family["train_count"] + family["heldout_count"]
+        generated: list[dict[str, Any]] = []
+        valid: list[dict[str, Any]] = []
+        cache_path = args.out_dir / f"raw_{family['name']}.json"
+        cache_path.parent.mkdir(parents=True, exist_ok=True)
+        if cache_path.exists():
+            cached = json.loads(cache_path.read_text(encoding="utf-8"))
+            generated = cached.get("rows", []) if isinstance(cached, dict) else []
+            valid = [item for item in generated if valid_spec(family["name"], item)]
+        for batch in range(1, 25):
+            if len(valid) >= needed + 6:
+                break
+            response = call_deepseek(endpoint, teacher_prompt(family, min(20, needed + 8), batch))
+            model = response["model"]
+            batch_rows = response["content"].get("rows", [])
+            if isinstance(batch_rows, list):
+                generated.extend(batch_rows)
+                valid = [item for item in generated if valid_spec(family["name"], item)]
+                cache_path.write_text(
+                    json.dumps({"family": family["name"], "rows": generated}, ensure_ascii=False, indent=2) + "\n",
+                    encoding="utf-8",
+                )
+            if len(valid) >= needed:
+                break
+        raw[family["name"]] = generated
+        picked_train = 0
+        picked_eval = 0
+        for item in valid:
+            user_key = clean_text(item["user"]).lower()
+            if user_key in seen_users:
+                continue
+            seen_users.add(user_key)
+            if picked_train < family["train_count"]:
+                sft_rows.append(build_sft_row(family["name"], picked_train, item, "train_or_val"))
+                picked_train += 1
+            elif picked_eval < family["heldout_count"]:
+                eval_cases.append(build_eval_case(family["name"], picked_eval, item))
+                picked_eval += 1
+            if picked_train >= family["train_count"] and picked_eval >= family["heldout_count"]:
+                break
+        if picked_train < family["train_count"] or picked_eval < family["heldout_count"]:
+            raise RuntimeError(
+                f"family {family['name']} generated only train={picked_train}/{family['train_count']} "
+                f"heldout={picked_eval}/{family['heldout_count']} valid rows"
+            )
+
+    train, val = split_rows(sft_rows, args.val_every)
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    write_jsonl(args.out_dir / "train.jsonl", train)
+    write_jsonl(args.out_dir / "val.jsonl", val)
+    write_jsonl(args.out_dir / "all.jsonl", sft_rows)
+    (args.out_dir / "raw_teacher.json").write_text(json.dumps(raw, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+    args.eval_out.parent.mkdir(parents=True, exist_ok=True)
+    eval_payload = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "generator": "build_odysseus_web_teacher_sft.py",
+        "provider": "DeepSeek",
+        "model": model,
+        "source": "teacher-generated behavioral specs from user-reported web synthesis failures",
+        "cases": eval_cases,
+    }
+    args.eval_out.write_text(json.dumps(eval_payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+
+    manifest = {
+        "name": args.out_dir.name,
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "provider": "DeepSeek",
+        "model": model,
+        "elapsed_seconds": round(time.time() - started, 3),
+        "total_sft_rows": len(sft_rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "heldout_cases": len(eval_cases),
+        "categories": {
+            family["name"]: sum(1 for row in sft_rows if row["metadata"]["category"] == family["name"])
+            for family in FAMILIES
+        },
+        "heldout_categories": {
+            family["name"]: sum(1 for case in eval_cases if case["deepseek_family"] == family["name"])
+            for family in FAMILIES
+        },
+        "acceptance_target": (
+            "Promote only if teacher web heldout passes 50/50, user live web prompts synthesize answers instead of raw links, "
+            "and old CRUD suites remain regression-clean."
+        ),
+        "files": {
+            "train": str(args.out_dir / "train.jsonl"),
+            "val": str(args.out_dir / "val.jsonl"),
+            "all": str(args.out_dir / "all.jsonl"),
+            "raw_teacher": str(args.out_dir / "raw_teacher.json"),
+            "heldout_eval": str(args.eval_out),
+        },
+    }
+    for key, value in list(manifest["files"].items()):
+        manifest[f"{key}_sha256"] = file_sha256(Path(value))
+    (args.out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+
+    print(json.dumps({
+        "out_dir": str(args.out_dir),
+        "eval_out": str(args.eval_out),
+        "total_sft_rows": len(sft_rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "heldout_cases": len(eval_cases),
+        "model": model,
+    }, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_web_v53_repair_sft.py

@@ -0,0 +1,483 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import random
+import time
+from pathlib import Path
+from typing import Any
+
+
+DEFAULT_OUT = Path("/home/pewds/odysseus-finetune/data/teacher_web_synthesis/odysseus_web_v53_repair_20260821")
+DEFAULT_EVAL_OUT = Path("/home/pewds/odysseus-cookbook-fresh/data/evals/ody_web_v53_live_robust_gate_20260821/cases.json")
+
+WEB_SEARCH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "web_search",
+        "description": "Search the web for current or source-backed information.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "query": {"type": "string"},
+                "time_filter": {"type": "string", "enum": ["day", "week", "month", "year"]},
+            },
+            "required": ["query"],
+        },
+    },
+}
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def tool_call(name: str, arguments: dict[str, Any], suffix: str) -> dict[str, Any]:
+    return {
+        "id": f"call_{suffix}",
+        "type": "function",
+        "function": {
+            "name": name,
+            "arguments": json.dumps(arguments, separators=(",", ":"), ensure_ascii=True),
+        },
+    }
+
+
+def source_block(query: str, rows: list[tuple[str, str]]) -> str:
+    lines = [
+        "```sources",
+        *[f"[{idx}] {title}\n    https://example.test/{idx}" for idx, (title, _snippet) in enumerate(rows, start=1)],
+        "```",
+        "",
+        "======================================================================",
+        "WEB SEARCH RESULTS AND FETCHED CONTENT",
+        f"Query: {query}",
+        f"Searched {len(rows)} results, fetched {len(rows)} pages",
+        "======================================================================",
+        "",
+        "SEARCH RESULTS SUMMARY:",
+        "--------------------------------------------------",
+    ]
+    for idx, (title, snippet) in enumerate(rows, start=1):
+        lines.extend([f"[{idx}] {title}", f"    URL: https://example.test/{idx}", f"    Snippet: {snippet}", ""])
+    return "\n".join(lines).strip()
+
+
+SYNTHESIS_SEEDS = [
+    {
+        "topic": "dog foaming at the mouth",
+        "users": [
+            "why would a dog suddenly foam at the mouth",
+            "look up why dogs get foam around their mouth sometimes",
+            "my dog is drooling white foam, what can cause that",
+        ],
+        "query": "dog foaming at mouth causes poisoning seizure dental nausea emergency vet",
+        "rows": [
+            ("Foaming at the mouth in dogs: causes", "Veterinary guidance lists nausea, stress, dental disease, seizures, poisoning, and rabies as possible causes."),
+            ("When dog foaming is urgent", "Emergency care is advised when foaming is sudden, severe, follows toxin exposure, or comes with collapse or seizures."),
+        ],
+        "final": "Dogs can foam at the mouth from nausea, stress, dental irritation, toxin exposure, seizures, or infections like rabies. If it is sudden, heavy, or paired with weakness, collapse, vomiting, or possible poison exposure, treat it as urgent and call a vet or emergency clinic.",
+        "query_terms": ["dog foaming", "mouth"],
+        "answer_terms": ["poisoning", "vet"],
+    },
+    {
+        "topic": "cat foaming after medicine",
+        "users": [
+            "why is my cat foaming after i gave medicine",
+            "look up cat white foam after oral meds",
+            "cat started bubbling at the mouth after medicine why",
+        ],
+        "query": "cat foaming mouth after medicine bitter taste nausea toxicity vet",
+        "rows": [
+            ("Cat foaming after medication", "Cats may foam after bitter oral medicines because of taste, drooling, nausea, or irritation."),
+            ("When to call a vet", "Call a veterinarian if the cat may have received the wrong dose, shows tremors, weakness, breathing issues, or persistent vomiting."),
+        ],
+        "final": "Cats often foam or drool after oral medicine because many meds taste very bitter, but nausea, irritation, or overdose can also do it. If the dose may be wrong or symptoms continue, call your vet and keep the package handy.",
+        "query_terms": ["cat foaming", "medicine"],
+        "answer_terms": ["bitter", "vet"],
+    },
+    {
+        "topic": "snails bubbling",
+        "users": [
+            "why do garden snails bubble up sometimes",
+            "look up why snails make bubbles",
+            "why is my snail foaming a bit",
+        ],
+        "query": "garden snail bubbling foaming mucus stress irritation defense",
+        "rows": [
+            ("Why snails bubble", "Land snails can make bubbly mucus when irritated, stressed, handled roughly, exposed to salt or chemicals, or defending themselves."),
+            ("Snail mucus and bubbles", "Some aquatic snails use mucus bubbles for floating, while pet land snails may foam as a stress response."),
+        ],
+        "final": "Snails usually bubble or foam when air mixes with mucus, often from stress, irritation, handling, dryness, salt, or chemicals. For a pet snail, rinse away any irritant with clean dechlorinated water and check humidity rather than treating it like normal behavior.",
+        "query_terms": ["snail", "bubbling"],
+        "answer_terms": ["mucus", "stress"],
+    },
+    {
+        "topic": "swollen phone battery",
+        "users": [
+            "why is my phone battery swollen and is it dangerous",
+            "look up swollen lithium battery what to do",
+            "phone battery puffed up after charging, safe?",
+        ],
+        "query": "swollen lithium ion phone battery gas buildup fire risk stop using",
+        "rows": [
+            ("Swollen lithium-ion batteries", "Battery swelling usually means gas buildup from cell degradation, damage, overheating, or overcharging."),
+            ("Battery swelling safety", "A swollen battery can rupture, leak, smoke, or catch fire. Stop using and charging the device and seek safe replacement or recycling."),
+        ],
+        "final": "A swollen phone battery is usually gas buildup from a failing lithium-ion cell. It is unsafe: stop using or charging it, do not press or puncture it, and get it replaced or recycled safely.",
+        "query_terms": ["swollen", "battery"],
+        "answer_terms": ["unsafe", "fire"],
+    },
+    {
+        "topic": "kombucha vinegar smell",
+        "users": [
+            "why does my kombucha smell like vinegar",
+            "look up kombucha tastes too vinegary reason",
+            "kombucha batch turned sharp vinegar smell why",
+        ],
+        "query": "kombucha smells like vinegar over fermented acetic acid fix",
+        "rows": [
+            ("Kombucha tastes like vinegar", "A strong vinegar smell usually comes from extended fermentation and acetic acid production."),
+            ("Fixing overly sour kombucha", "Shorter fermentation, cooler storage, and using less starter liquid can reduce sourness in the next batch."),
+        ],
+        "final": "A vinegar smell usually means the kombucha over-fermented and built up more acetic acid. It is often still usable if there is no mold, but shorten the ferment next time or dilute it to taste.",
+        "query_terms": ["kombucha", "vinegar"],
+        "answer_terms": ["over-fermented", "acid"],
+    },
+    {
+        "topic": "phone battery drain after update",
+        "users": [
+            "why is my phone battery draining after an update",
+            "look up battery drain right after phone update",
+            "ios android update killed my battery why",
+        ],
+        "query": "phone battery drain after software update indexing background activity recalibration",
+        "rows": [
+            ("Battery drain after updates", "After major updates, phones may spend hours or days indexing photos, rebuilding caches, syncing, and recalibrating battery estimates."),
+            ("When battery drain is abnormal", "Check background apps, location use, and battery health if drain remains high after a couple of days."),
+        ],
+        "final": "Battery drain right after an update is often from background indexing, app updates, syncing, and battery estimate recalibration. If it does not settle after a day or two, check battery health and which apps are using background power.",
+        "query_terms": ["battery drain", "update"],
+        "answer_terms": ["background", "recalibration"],
+    },
+]
+
+
+PRICE_SEEDS = [
+    {
+        "users": [
+            "whats the price per liter of gas in japan in euro",
+            "look up japan petrol per litre in eur",
+            "how much is gasoline in japan per liter converted to euros",
+        ],
+        "query": "Japan gasoline price per liter yen EUR conversion",
+        "rows": [
+            ("Japan gasoline retail price", "Regular gasoline is around 174 yen per liter in recent Japan retail-price summaries."),
+            ("EUR JPY exchange rate", "A recent EUR/JPY rate near 172 yen per euro implies 174 yen is about 1.01 euro."),
+        ],
+        "final": "Using about 174 yen per liter and roughly 172 yen per euro, gasoline in Japan is about 1.01 EUR per liter. Treat that as an estimate because pump prices and exchange rates move.",
+        "query_terms": ["Japan", "gasoline"],
+        "answer_terms": ["EUR", "liter"],
+    },
+    {
+        "users": [
+            "what is milk per liter in japan in gbp roughly",
+            "look up japanese milk price converted to pounds",
+            "how much does a liter of milk cost in japan in pounds",
+        ],
+        "query": "Japan milk price per liter yen GBP conversion",
+        "rows": [
+            ("Japan milk retail price", "Retail milk prices are often around 220 to 270 yen per liter depending on store and region."),
+            ("GBP JPY exchange rate", "At about 215 yen per pound, 220 to 270 yen is roughly 1.02 to 1.26 GBP."),
+        ],
+        "final": "A liter of milk in Japan is roughly 220-270 yen, which is about 1.00-1.25 GBP at a recent yen-pound rate. Local supermarkets can vary.",
+        "query_terms": ["Japan", "milk"],
+        "answer_terms": ["GBP", "yen"],
+    },
+    {
+        "users": [
+            "what is silver per ounce in euros today",
+            "look up silver spot price oz eur",
+            "silver price per ounce in eur right now",
+        ],
+        "query": "silver spot price per ounce EUR today",
+        "rows": [
+            ("Silver spot price today", "Silver spot is quoted around 38 USD per troy ounce in recent market tables."),
+            ("EUR USD exchange rate", "At about 1.17 USD per EUR, 38 USD is roughly 32.50 EUR."),
+        ],
+        "final": "If silver is around 38 USD per troy ounce and EUR/USD is about 1.17, that works out to roughly 32.50 EUR per ounce. Use it as a live-market estimate, not a quote.",
+        "query_terms": ["silver", "ounce"],
+        "answer_terms": ["EUR", "ounce"],
+    },
+]
+
+
+NO_TOOL_SEEDS = [
+    ("dont search, explain what a search engine is", "A search engine crawls or indexes pages, ranks them for a query, and returns links or summaries that seem relevant."),
+    ("without looking it up, what is photosynthesis", "Photosynthesis is how plants, algae, and some bacteria use light energy to turn carbon dioxide and water into sugars, releasing oxygen as a byproduct."),
+    ("no web, why do onions make people cry", "Cut onions release sulfur compounds that react into an eye-irritating gas. Your eyes make tears to dilute and wash it away."),
+    ("answer from memory only: what is a lithium ion battery", "A lithium-ion battery stores energy by moving lithium ions between electrodes during charging and discharging."),
+    ("do not search. what does inflation mean", "Inflation means the general level of prices is rising, so the same amount of money buys less than before."),
+]
+
+
+INSUFFICIENT_SEEDS = [
+    {
+        "user": "look up the exact price of the sandwich at that cafe from my screenshot",
+        "query": "exact sandwich price cafe screenshot",
+        "rows": [
+            ("Cafe menu design tips", "Article about designing restaurant menus, no prices for the user's screenshot."),
+            ("Generic sandwich prices", "A broad article with no matching cafe name or current menu."),
+        ],
+        "final": "I do not have enough clear evidence to give the exact sandwich price. The search results are generic and do not identify the cafe or current menu, so I would need the cafe name or the screenshot text.",
+        "query_terms": ["sandwich", "price"],
+        "answer_terms": ["not enough", "cafe"],
+    },
+    {
+        "user": "find the current rent for the apartment my friend mentioned",
+        "query": "current rent apartment friend mentioned",
+        "rows": [
+            ("Apartment rent trends", "Citywide rent trend article with averages only."),
+            ("Rental listings", "No address, building, or unit matching the user's request."),
+        ],
+        "final": "I do not have enough evidence to identify that apartment or its current rent. I would need the building, listing, address, or message text before searching further.",
+        "query_terms": ["rent", "apartment"],
+        "answer_terms": ["not enough", "address"],
+    },
+]
+
+
+def make_synthesis_rows(target: int, rng: random.Random) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    seeds = SYNTHESIS_SEEDS + PRICE_SEEDS
+    variants = [
+        "{user}",
+        "can you look this up: {user}",
+        "{user} pls",
+        "quick search - {user}",
+    ]
+    while len(rows) < target:
+        seed = seeds[len(rows) % len(seeds)]
+        user = rng.choice(variants).format(user=rng.choice(seed["users"]))
+        query = seed["query"]
+        call = tool_call("web_search", {"query": query}, f"synth_{len(rows)}")
+        messages = [
+            {"role": "user", "content": user},
+            {"role": "assistant", "content": "", "tool_calls": [call]},
+            {"role": "tool", "tool_call_id": call["id"], "content": source_block(query, seed["rows"])},
+            {"role": "assistant", "content": seed["final"]},
+        ]
+        rows.append(row("web_synthesis_after_results", messages, 1, seed["query_terms"], seed["answer_terms"]))
+    return rows
+
+
+def make_no_tool_rows(target: int, rng: random.Random) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    prefixes = ["", "quickly, ", "short answer: ", "one paragraph, "]
+    while len(rows) < target:
+        user, final = NO_TOOL_SEEDS[len(rows) % len(NO_TOOL_SEEDS)]
+        messages = [
+            {"role": "user", "content": rng.choice(prefixes) + user},
+            {"role": "assistant", "content": final},
+        ]
+        rows.append(row("web_no_tool_boundary", messages, 0, [], [final.split()[0]]))
+    return rows
+
+
+def make_retry_rows(target: int, rng: random.Random) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    seeds = SYNTHESIS_SEEDS + PRICE_SEEDS
+    while len(rows) < target:
+        seed = seeds[len(rows) % len(seeds)]
+        first_query = seed["query"].split(" ", 4)[0] + " " + seed["query"].split(" ", 4)[1]
+        first_call = tool_call("web_search", {"query": first_query}, f"retry_{len(rows)}_first")
+        second_call = tool_call("web_search", {"query": seed["query"]}, f"retry_{len(rows)}_second")
+        messages = [
+            {"role": "user", "content": rng.choice(seed["users"])},
+            {"role": "assistant", "content": "", "tool_calls": [first_call]},
+            {
+                "role": "tool",
+                "tool_call_id": first_call["id"],
+                "content": source_block(first_query, [("Ambiguous results", "The results are dictionary pages or unrelated pages and do not answer the user's question.")]),
+            },
+            {"role": "assistant", "content": "", "tool_calls": [second_call]},
+            {"role": "tool", "tool_call_id": second_call["id"], "content": source_block(seed["query"], seed["rows"])},
+            {"role": "assistant", "content": seed["final"]},
+        ]
+        rows.append(row("web_retry_after_weak_results", messages, 2, seed["query_terms"], seed["answer_terms"]))
+    return rows
+
+
+def make_insufficient_rows(target: int, rng: random.Random) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    while len(rows) < target:
+        seed = INSUFFICIENT_SEEDS[len(rows) % len(INSUFFICIENT_SEEDS)]
+        user = seed["user"]
+        if rng.random() < 0.5:
+            user = "please search: " + user
+        call = tool_call("web_search", {"query": seed["query"]}, f"insufficient_{len(rows)}")
+        messages = [
+            {"role": "user", "content": user},
+            {"role": "assistant", "content": "", "tool_calls": [call]},
+            {"role": "tool", "tool_call_id": call["id"], "content": source_block(seed["query"], seed["rows"])},
+            {"role": "assistant", "content": seed["final"]},
+        ]
+        rows.append(row("web_insufficient_evidence", messages, 1, seed["query_terms"], seed["answer_terms"]))
+    return rows
+
+
+def row(category: str, messages: list[dict[str, Any]], expected_calls: int, query_terms: list[str], answer_terms: list[str]) -> dict[str, Any]:
+    item = {
+        "messages": messages,
+        "tools": [] if expected_calls == 0 else [WEB_SEARCH_TOOL],
+        "generator": "odysseus_web_v53_repair_seeded_teacher",
+        "metadata": {
+            "category": category,
+            "split": "train_or_val",
+            "expected_tool_calls": expected_calls,
+            "query_must_include": query_terms,
+            "answer_must_include": answer_terms,
+        },
+    }
+    item["uuid"] = stable_id("ody_web_v53_repair", item)
+    return item
+
+
+def split_rows(rows: list[dict[str, Any]], val_every: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, item in enumerate(rows):
+        (val if idx % val_every == val_every - 1 else train).append(item)
+    return train, val
+
+
+def eval_case(idx: int, seed: dict[str, Any], category: str, expect_no_tool: bool = False) -> dict[str, Any]:
+    if expect_no_tool:
+        return {
+            "id": f"v53_{category}_{idx:02d}",
+            "kind": "negative_web",
+            "user": seed["user"],
+            "expect_no_tool": True,
+            "forbidden_tools": ["web_search", "web_fetch"],
+            "must_answer_any": seed["answer_terms"],
+            "forbidden_final": ["WEB SEARCH RESULTS", "```sources", "Here are links"],
+        }
+    return {
+        "id": f"v53_{category}_{idx:02d}",
+        "kind": "web",
+        "user": seed["user"],
+        "expect_first_tool": "web_search",
+        "forbidden_query_any": ["official links", "cambridge", "merriam", "dictionary", "wikipedia official"],
+        "must_query_any": seed["query_terms"],
+        "must_answer_any": seed["answer_terms"],
+        "forbidden_final": [
+            "WEB SEARCH RESULTS",
+            "```sources",
+            "Here are links",
+            "not enough clear answer evidence",
+            "not enough clear evidence to synthesize",
+        ],
+        "max_web_searches": 2,
+    }
+
+
+def build_eval_cases() -> list[dict[str, Any]]:
+    cases: list[dict[str, Any]] = []
+    synth_seeds = SYNTHESIS_SEEDS + PRICE_SEEDS
+    for idx, seed in enumerate(synth_seeds):
+        cases.append(eval_case(idx, {"user": seed["users"][0], "query_terms": seed["query_terms"], "answer_terms": seed["answer_terms"]}, "synthesis"))
+    for idx, seed in enumerate(SYNTHESIS_SEEDS[:4]):
+        cases.append(eval_case(idx, {"user": "bad prior results, search again properly: " + seed["users"][1], "query_terms": seed["query_terms"], "answer_terms": seed["answer_terms"]}, "query_quality"))
+    for idx, (user, final) in enumerate(NO_TOOL_SEEDS):
+        terms = [word.strip(".,").lower() for word in final.split() if len(word.strip(".,")) > 5][:3] or ["answer"]
+        cases.append(eval_case(idx, {"user": user, "answer_terms": terms}, "no_tool", expect_no_tool=True))
+    return cases
+
+
+def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("".join(json.dumps(item, ensure_ascii=True) + "\n" for item in rows), encoding="utf-8")
+
+
+def file_sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT)
+    parser.add_argument("--eval-out", type=Path, default=DEFAULT_EVAL_OUT)
+    parser.add_argument("--val-every", type=int, default=6)
+    parser.add_argument("--seed", type=int, default=53)
+    args = parser.parse_args()
+
+    rng = random.Random(args.seed)
+    rows = []
+    rows.extend(make_synthesis_rows(120, rng))
+    rows.extend(make_no_tool_rows(50, rng))
+    rows.extend(make_retry_rows(40, rng))
+    rows.extend(make_insufficient_rows(30, rng))
+    rng.shuffle(rows)
+    train, val = split_rows(rows, args.val_every)
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    write_jsonl(args.out_dir / "train.jsonl", train)
+    write_jsonl(args.out_dir / "val.jsonl", val)
+    write_jsonl(args.out_dir / "all.jsonl", rows)
+
+    eval_cases = build_eval_cases()
+    args.eval_out.parent.mkdir(parents=True, exist_ok=True)
+    args.eval_out.write_text(
+        json.dumps(
+            {
+                "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                "generator": "build_odysseus_web_v53_repair_sft.py",
+                "source": "seeded teacher-style repair rows from V52 live failure families",
+                "cases": eval_cases,
+            },
+            ensure_ascii=True,
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    manifest = {
+        "name": args.out_dir.name,
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "total_sft_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "heldout_cases": len(eval_cases),
+        "categories": {
+            category: sum(1 for item in rows if item["metadata"]["category"] == category)
+            for category in sorted({item["metadata"]["category"] for item in rows})
+        },
+        "heldout_categories": {
+            category: sum(1 for case in eval_cases if f"_{category}_" in case["id"])
+            for category in ["synthesis", "query_quality", "no_tool"]
+        },
+        "acceptance_target": (
+            "Promote only if live robust gate passes all cases, user-reported web searches synthesize answers, "
+            "no-search requests avoid tools, active compose still mutates document, and old CRUD remains regression-clean."
+        ),
+        "files": {
+            "train": str(args.out_dir / "train.jsonl"),
+            "val": str(args.out_dir / "val.jsonl"),
+            "all": str(args.out_dir / "all.jsonl"),
+            "heldout_eval": str(args.eval_out),
+        },
+    }
+    for key, value in list(manifest["files"].items()):
+        manifest[f"{key}_sha256"] = file_sha256(Path(value))
+    (args.out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+
+    print(json.dumps({k: manifest[k] for k in ("total_sft_rows", "train_rows", "val_rows", "heldout_cases", "categories", "heldout_categories")}, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_sft_environment_inventories.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Snapshot non-sensitive fixture inventories for SFT expansion owners."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from core.database import CalendarCal, CalendarEvent, Document, Memory, Note, ScheduledTask, Session, SessionLocal, UserTool  # noqa: E402
+from scripts.sft_email_overseer import PROFILES  # noqa: E402
+OWNERS = ["sft_maya_ops", "sft_jules_research", "sft_nora_design", "sft_omar_finance"]
+
+
+def clip(value: Any, limit: int = 180) -> str:
+    text = str(value or "").replace("\n", " ").strip()
+    return text[:limit] + ("..." if len(text) > limit else "")
+
+
+def email_inventory() -> dict[str, list[dict[str, Any]]]:
+    payload = json.loads((ROOT / "data/fixture_email_messages.json").read_text(encoding="utf-8"))
+    rows = payload.get("messages") if isinstance(payload, dict) else payload
+    out = {owner: [] for owner in OWNERS}
+    for row in rows or []:
+        owner = str(row.get("owner") or "")
+        if owner not in out:
+            continue
+        out[owner].append({
+            "uid": str(row.get("uid") or ""),
+            "account": row.get("account") or row.get("account_id"),
+            "from": clip(row.get("from") or row.get("sender")),
+            "subject": clip(row.get("subject")),
+            "date": row.get("date"),
+            "attachments": [att.get("filename") for att in (row.get("attachments") or []) if isinstance(att, dict)],
+        })
+    return out
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--sample-limit", type=int, default=30)
+    args = parser.parse_args()
+    mail = email_inventory()
+    db = SessionLocal()
+    try:
+        inventories = []
+        for owner in OWNERS:
+            calendars = db.query(CalendarCal).filter(CalendarCal.owner == owner).all()
+            calendar_ids = [cal.id for cal in calendars]
+            events = db.query(CalendarEvent).filter(CalendarEvent.calendar_id.in_(calendar_ids)).order_by(CalendarEvent.dtstart).all() if calendar_ids else []
+            notes = db.query(Note).filter(Note.owner == owner, Note.archived.is_(False)).order_by(Note.updated_at.desc()).all()
+            memories = db.query(Memory).filter(Memory.owner == owner).order_by(Memory.timestamp.desc()).all()
+            documents = db.query(Document).filter(Document.owner == owner, Document.archived.is_(False)).order_by(Document.updated_at.desc()).all()
+            tasks = db.query(ScheduledTask).filter(ScheduledTask.owner == owner).order_by(ScheduledTask.updated_at.desc()).all()
+            sessions = db.query(Session).filter(Session.owner == owner, Session.archived.is_(False)).order_by(Session.updated_at.desc()).all()
+            disabled_tools = [row.name for row in db.query(UserTool).filter(UserTool.owner == owner, UserTool.is_active.is_(False)).all()]
+            emails = mail.get(owner, [])
+            inventories.append({
+                "owner": owner,
+                "profile": PROFILES[owner],
+                "counts": {
+                    "emails": len(emails), "notes": len(notes), "memories": len(memories),
+                    "documents": len(documents), "tasks": len(tasks), "calendars": len(calendars),
+                    "calendar_events": len(events), "sessions": len(sessions),
+                },
+                "email_accounts": dict(Counter(str(row.get("account") or "unknown") for row in emails)),
+                "emails": emails[: args.sample_limit],
+                "notes": [{"id": row.id, "title": clip(row.title), "content": clip(row.content), "type": row.note_type, "label": row.label} for row in notes[: args.sample_limit]],
+                "memories": [{"id": row.id, "text": clip(row.text), "category": row.category} for row in memories[: args.sample_limit]],
+                "documents": [{"id": row.id, "title": clip(row.title), "language": row.language, "content": clip(row.current_content)} for row in documents[: args.sample_limit]],
+                "tasks": [{"id": row.id, "name": clip(row.name), "status": row.status, "schedule": row.schedule} for row in tasks[: args.sample_limit]],
+                "calendars": [{"id": row.id, "name": row.name, "source": row.source} for row in calendars],
+                "events": [{"uid": row.uid, "summary": clip(row.summary), "start": row.dtstart.isoformat(), "all_day": row.all_day} for row in events[: args.sample_limit]],
+                "sessions": [{"id": row.id, "name": clip(row.name), "mode": row.mode} for row in sessions[: args.sample_limit]],
+                "disabled_tools": disabled_tools,
+            })
+    finally:
+        db.close()
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps({"environments": inventories}, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(json.dumps({row["owner"]: row["counts"] for row in inventories}, indent=2))
+
+
+if __name__ == "__main__":
+    main()

scripts/build_sft_expansion_manifest.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""Freeze approved Alex traces into seed families for environment expansion."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from collections import Counter, defaultdict
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_AUDIT = ROOT / "data/audits/sft_corpus_deepseek_audit_live_complete_20260830/deepseek_verdicts.jsonl"
+DEFAULT_REPAIRS = ROOT / "data/audits/sft_corpus_kimi_repairs_live_20260830/apply_manifest.json"
+DEFAULT_LATER_AUDITS = [
+    ROOT / "data/audits/sft_corpus_deepseek_audit_20260830_104907/deepseek_verdicts.jsonl",
+    ROOT / "data/audits/sft_corpus_deepseek_audit_20260830_105208/deepseek_verdicts.jsonl",
+    ROOT / "data/audits/sft_corpus_deepseek_audit_20260830_105713/deepseek_verdicts.jsonl",
+    ROOT / "data/audits/sft_corpus_deepseek_audit_20260830_110443/deepseek_verdicts.jsonl",
+    ROOT / "data/audits/sft_corpus_deepseek_audit_20260830_121408/deepseek_verdicts.jsonl",
+]
+
+OWNER_BOUND_MARKERS = (
+    "email", "calendar", "note", "memory", "document", "task", "skill", "session",
+    "contact", "research", "gallery", "image", "settings", "webhook", "token", "endpoint", "mcp",
+)
+
+
+def read_jsonl(path: Path) -> list[dict[str, Any]]:
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def stable_split(seed_family_id: str) -> str:
+    bucket = int(hashlib.sha256(seed_family_id.encode()).hexdigest()[:8], 16) % 100
+    if bucket < 80:
+        return "train"
+    if bucket < 90:
+        return "validation"
+    return "test"
+
+
+def turn_digest(row: dict[str, Any]) -> str:
+    payload = [row.get("user"), row.get("assistant"), row.get("thinking"), row.get("tool_events")]
+    return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str).encode()).hexdigest()
+
+
+def approved_sessions(base_audit: Path, repairs: Path, later_audits: list[Path]) -> tuple[set[str], dict[str, str]]:
+    base = read_jsonl(base_audit)
+    approved = {str(row["session_id"]) for row in base if row.get("verdict") == "keep"}
+    provenance = {str(row["session_id"]): "deepseek_complete_keep" for row in base if row.get("verdict") == "keep"}
+    repair_manifest = json.loads(repairs.read_text(encoding="utf-8"))
+    for sid in repair_manifest.get("accepted_session_ids") or []:
+        approved.add(str(sid))
+        provenance[str(sid)] = "kimi_repair_deepseek_keep"
+    for path in later_audits:
+        if not path.exists():
+            continue
+        for row in read_jsonl(path):
+            sid = str(row.get("session_id") or "")
+            if row.get("verdict") == "keep" and sid:
+                approved.add(sid)
+                provenance[sid] = f"later_deepseek_keep:{path.parent.name}"
+    return approved, provenance
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--trace", type=Path, default=ROOT / "data/sft_traces/sft_alex_creator.jsonl")
+    parser.add_argument("--base-audit", type=Path, default=DEFAULT_AUDIT)
+    parser.add_argument("--repair-manifest", type=Path, default=DEFAULT_REPAIRS)
+    parser.add_argument("--later-audit", type=Path, action="append", default=[])
+    parser.add_argument("--out-dir", type=Path, required=True)
+    args = parser.parse_args()
+
+    later = args.later_audit or DEFAULT_LATER_AUDITS
+    approved, provenance = approved_sessions(args.base_audit, args.repair_manifest, later)
+    by_session: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for row in read_jsonl(args.trace):
+        sid = str(row.get("session_id") or "")
+        if sid in approved:
+            by_session[sid].append(row)
+
+    manifest_rows = []
+    frozen_rows = []
+    duplicate_turns = 0
+    tools = Counter()
+    split_counts = Counter()
+    for sid in sorted(by_session):
+        unique = []
+        seen = set()
+        for row in by_session[sid]:
+            digest = turn_digest(row)
+            if digest in seen:
+                duplicate_turns += 1
+                continue
+            seen.add(digest)
+            unique.append(row)
+        if not unique:
+            continue
+        actual_tools = sorted({
+            str(event.get("tool"))
+            for row in unique for event in (row.get("tool_events") or []) if event.get("tool")
+        })
+        for tool in actual_tools:
+            tools[tool] += 1
+        owner_bound = any(any(marker in tool.lower() for marker in OWNER_BOUND_MARKERS) for tool in actual_tools)
+        family_id = f"alex:{sid}"
+        split = stable_split(family_id)
+        split_counts[split] += 1
+        manifest_rows.append({
+            "seed_family_id": family_id,
+            "source_owner": "sft_alex_creator",
+            "source_session_id": sid,
+            "session_name": unique[0].get("session_name"),
+            "approval_provenance": provenance.get(sid),
+            "split": split,
+            "owner_bound": owner_bound,
+            "tools": actual_tools,
+            "turn_count": len(unique),
+            "turns": [
+                {
+                    "message_id": row.get("message_id"),
+                    "user": row.get("user"),
+                    "assistant": row.get("assistant"),
+                    "thinking": row.get("thinking"),
+                    "tool_events": row.get("tool_events") or [],
+                }
+                for row in unique
+            ],
+        })
+        for row in unique:
+            copied = dict(row)
+            metadata = dict(copied.get("metadata") or {})
+            metadata.update({"seed_family_id": family_id, "dataset_split": split, "approval_provenance": provenance.get(sid)})
+            copied["metadata"] = metadata
+            frozen_rows.append(copied)
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    (args.out_dir / "seed_manifest.json").write_text(json.dumps({"seeds": manifest_rows}, ensure_ascii=False, indent=2), encoding="utf-8")
+    (args.out_dir / "approved_trace.jsonl").write_text(
+        "".join(json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n" for row in frozen_rows),
+        encoding="utf-8",
+    )
+    summary = {
+        "approved_ids": len(approved),
+        "approved_sessions_present": len(manifest_rows),
+        "approved_turns": len(frozen_rows),
+        "missing_approved_sessions": len(approved - set(by_session)),
+        "duplicate_turns_removed": duplicate_turns,
+        "owner_bound_sessions": sum(bool(row["owner_bound"]) for row in manifest_rows),
+        "global_sessions": sum(not bool(row["owner_bound"]) for row in manifest_rows),
+        "splits": dict(split_counts),
+        "tool_session_counts": dict(tools.most_common()),
+    }
+    (args.out_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

scripts/build_sft_expansion_splits.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""Build family-safe train/validation/test JSONL files from approved seeds and expansions."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def rows(path: Path) -> list[dict[str, Any]]:
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--approved-trace", type=Path, required=True)
+    parser.add_argument("--review", type=Path, action="append", default=[])
+    parser.add_argument("--out-dir", type=Path, required=True)
+    args = parser.parse_args()
+
+    manifest = json.loads(args.manifest.read_text(encoding="utf-8"))
+    split_by_family = {
+        str(seed["seed_family_id"]): str(seed["split"])
+        for seed in manifest["seeds"]
+    }
+    retained_sessions: set[str] = set()
+    for review_path in args.review:
+        report = json.loads(review_path.read_text(encoding="utf-8"))
+        retained_sessions.update(
+            str(item["session_id"])
+            for item in report.get("results", [])
+            if item.get("retained") is True
+        )
+
+    corpus = rows(args.approved_trace)
+    if retained_sessions:
+        owners = sorted({
+            str(item.get("owner") or "")
+            for review_path in args.review
+            for item in json.loads(review_path.read_text(encoding="utf-8")).get("results", [])
+            if item.get("retained") is True
+        })
+        for owner in owners:
+            path = ROOT / "data" / "sft_traces" / f"{owner}.jsonl"
+            if not path.exists():
+                continue
+            corpus.extend(
+                row for row in rows(path)
+                if str(row.get("session_id") or "") in retained_sessions
+            )
+
+    seen_messages: set[str] = set()
+    split_rows: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    family_splits: dict[str, set[str]] = defaultdict(set)
+    for row in corpus:
+        metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
+        family = str(
+            metadata.get("seed_family_id")
+            or row.get("seed_family_id")
+            or f"seed:{row.get('session_id')}"
+        )
+        split = str(
+            metadata.get("dataset_split")
+            or row.get("dataset_split")
+            or split_by_family.get(family)
+            or "train"
+        )
+        if split not in {"train", "validation", "test"}:
+            raise ValueError(f"invalid split {split!r} for family {family}")
+        signature = json.dumps(
+            [row.get("user"), row.get("assistant"), row.get("tool_events")],
+            sort_keys=True,
+            ensure_ascii=False,
+        )
+        if signature in seen_messages:
+            continue
+        seen_messages.add(signature)
+        family_splits[family].add(split)
+        split_rows[split].append(row)
+    leaked = {family: values for family, values in family_splits.items() if len(values) > 1}
+    if leaked:
+        raise ValueError(f"seed-family split leakage: {leaked}")
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    for split in ("train", "validation", "test"):
+        path = args.out_dir / f"{split}.jsonl"
+        path.write_text(
+            "\n".join(json.dumps(row, ensure_ascii=False) for row in split_rows[split])
+            + ("\n" if split_rows[split] else ""),
+            encoding="utf-8",
+        )
+    summary = {
+        "turns": {split: len(split_rows[split]) for split in ("train", "validation", "test")},
+        "sessions": len({str(row.get("session_id")) for row in corpus}),
+        "families": len(family_splits),
+        "retained_expansion_sessions": len(retained_sessions),
+        "family_leaks": 0,
+    }
+    (args.out_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(summary["turns"], indent=2))
+
+
+if __name__ == "__main__":
+    main()

scripts/build_v66_calendar_heldout_cases.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import datetime, timedelta
+from pathlib import Path
+
+WEEKDAY_INDEX = {
+    "Monday": 0,
+    "Tuesday": 1,
+    "Wednesday": 2,
+    "Thursday": 3,
+    "Friday": 4,
+    "Saturday": 5,
+    "Sunday": 6,
+}
+
+
+def parse_time(value: str) -> tuple[int, int]:
+    raw = value.lower().strip()
+    minute = 0
+    if ":" in raw:
+        left, right = raw.replace("am", "").replace("pm", "").split(":", 1)
+        hour = int(left)
+        minute = int(right[:2])
+    else:
+        hour = int("".join(ch for ch in raw if ch.isdigit()))
+    if "pm" in raw and hour != 12:
+        hour += 12
+    if "am" in raw and hour == 12:
+        hour = 0
+    return hour, minute
+
+
+def next_weekday(anchor: datetime, weekday: str, modifier: str) -> datetime:
+    delta = (WEEKDAY_INDEX[weekday] - anchor.weekday()) % 7
+    if modifier == "next":
+        delta = delta + 7 if delta != 0 else 7
+    elif delta == 0:
+        delta = 7
+    return anchor + timedelta(days=delta)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out", type=Path, default=Path("data/evals/ody_v66_calendar_date_logic_20260822/heldout_calendar_cases.json"))
+    parser.add_argument("--limit", type=int, default=84)
+    args = parser.parse_args()
+
+    # The app route injects live current date. These cases are designed for
+    # the current 2026-08-22 Asia/Tokyo test window and use deterministic
+    # marker cleanup in the existing smoke harness.
+    anchor = datetime(2026, 8, 22, 16, 0)
+    templates = [
+        ("flight", "add {marker} im flying back to japan on {phrase} {time}", False),
+        ("flight", "put {marker} flight home on my calendar {phrase} at {time}", False),
+        ("drive", "add {marker} drive to Kyoto {phrase} {time}", False),
+        ("meeting", "schedule {marker} meeting for {phrase} {time}", False),
+        ("appointment", "put {marker} appointment on {phrase} at {time}", False),
+        ("flight", "add {marker} flight from Haneda {phrase} {time}", True),
+        ("doctor", "schedule {marker} doctor appointment at Tokyo Midtown Clinic {phrase} {time}", True),
+    ]
+    times = ["5pm", "8am", "7:30pm", "11am", "9pm", "6:15pm"]
+    weekdays = list(WEEKDAY_INDEX)
+    modifiers = ["", "this", "next"]
+    cases = []
+    idx = 0
+    for weekday in weekdays:
+        for modifier in modifiers:
+            if modifier == "this" and anchor.weekday() == WEEKDAY_INDEX[weekday]:
+                continue
+            for _kind, template, has_location in templates:
+                if len(cases) >= args.limit:
+                    break
+                marker = f"ODY-V66-HELDOUT-CAL-{idx:04d}"
+                phrase = f"{modifier} {weekday}".strip()
+                time_text = times[idx % len(times)]
+                hour, minute = parse_time(time_text)
+                target = next_weekday(anchor, weekday, modifier).replace(hour=hour, minute=minute, second=0, microsecond=0)
+                forbidden_values = ["2026-07-12", "2025-09-10", "JFK"]
+                if not has_location:
+                    forbidden_values.extend(["Haneda", "Tokyo Midtown Clinic"])
+                cases.append({
+                    "id": f"calendar_relative_weekday_{idx:04d}",
+                    "kind": "calendar",
+                    "user": template.format(marker=marker, phrase=phrase, time=time_text),
+                    "marker": marker,
+                    "expect_first_tool": "manage_calendar",
+                    "must_mutate": "calendar_created_at",
+                    "expect_created_event_dtstart": target.strftime("%Y-%m-%dT%H:%M"),
+                    "forbidden_tools": ["web_search"],
+                    "forbidden_tool_arg_values": forbidden_values,
+                    "must_answer_any": [target.strftime("%Y-%m-%d"), target.strftime("%A"), time_text.replace(":00", "")],
+                })
+                idx += 1
+            if len(cases) >= args.limit:
+                break
+        if len(cases) >= args.limit:
+            break
+
+    payload = {
+        "description": "V66 held-out calendar relative weekday/date logic gate. Built for 2026-08-22 Asia/Tokyo app context.",
+        "cases": cases,
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"out": str(args.out), "cases": len(cases)}, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/compare_compact_tool_inventory.py

@@ -0,0 +1,62 @@
+"""Read-only schema ablation on the served model; generated calls are never executed.
+
+This isolates inventory size, not full harness performance or blind accuracy.
+"""
+import concurrent.futures
+import json
+from pathlib import Path
+import sys
+import time
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import httpx
+from src.agent_loop import _compact_openai_tool_schema
+from src.tool_schemas import FUNCTION_TOOL_SCHEMAS
+from src.turn_contract import FAMILY_TOOLS
+
+CASES = [
+    ("notes", "Show my noes", "manage_notes"),
+    ("calendar", "What is on my caledar?", "manage_calendar"),
+    ("tasks", "List my scheduled tasks", "manage_tasks"),
+    ("skills", "Show my skills", "manage_skills"),
+    ("memory", "Remember that I prefer short answers", "manage_memory"),
+    ("documents", "List my documents", "manage_documents"),
+    ("email", "Show my connected email accounts", "list_email_accounts"),
+    ("search_browser", "Search the web for PostgreSQL transaction isolation documentation", "web_search"),
+    ("shell_files", "Use bash to run pwd", "bash"),
+    ("cookbook_admin", "List configured Cookbook servers", "list_cookbook_servers"),
+]
+FAMILIES = {row[0] for row in CASES}
+
+
+def run(job):
+    profile, (family, prompt, expected) = job
+    names = set().union(*(FAMILY_TOOLS[f] for f in (FAMILIES if profile == "all" else {family})))
+    schemas = [_compact_openai_tool_schema(s) for s in FUNCTION_TOOL_SCHEMAS
+               if s["function"]["name"] in names]
+    start = time.monotonic()
+    try:
+        response = httpx.post(
+            "http://100.118.44.115:18182/v1/chat/completions",
+            json={"model": "odysseus-qwen3.5-tools-pre-heretic", "temperature": 0,
+                  "max_tokens": 256, "chat_template_kwargs": {"enable_thinking": False},
+                  "messages": [{"role": "system", "content": "You are Odysseus. Use the available tools to fulfill the request. Answer normally when no tool is needed."},
+                               {"role": "user", "content": prompt}], "tools": schemas},
+            timeout=90,
+        )
+        response.raise_for_status()
+        data = response.json()
+        message = data["choices"][0]["message"]
+        called = [c["function"]["name"] for c in message.get("tool_calls") or []]
+        return {"profile": profile, "family": family, "prompt": prompt,
+                "schemas": len(schemas), "called": called, "expected": expected,
+                "routing_pass": expected in called, "message": message,
+                "usage": data.get("usage"), "seconds": round(time.monotonic()-start, 2)}
+    except Exception as exc:
+        return {"profile": profile, "family": family, "error": str(exc)}
+
+
+if __name__ == "__main__":
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+        rows = list(pool.map(run, [(profile, case) for profile in ("family", "all") for case in CASES]))
+    print(json.dumps(rows, ensure_ascii=False, indent=2))

scripts/compare_reference_strategies.mjs

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+// Serial fixture-only experiment; mode order rotates per case.
+import fs from 'node:fs';
+import path from 'node:path';
+import {spawn} from 'node:child_process';
+const root = path.resolve(new URL('..', import.meta.url).pathname);
+const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+const manifest = path.join(root,'reports',`reference-strategies-${stamp}.json`);
+const availableCases = ['original','reversed','quoted','all_three','negative','typo',
+  'subset','keep_all','contrast','drinks','schedule_words','explicit_ids',
+  'quoted_typo','single','except_one','punctuated'];
+const cases = process.env.REFERENCE_CASES ? process.env.REFERENCE_CASES.split(',') : availableCases;
+if (!cases.length || new Set(cases).size !== cases.length || cases.some(c=>!availableCases.includes(c)))
+  throw Error('Invalid reference cases');
+const modes = (process.env.REFERENCE_MODES || 'recent_fixture_only').split(',');
+if (!modes.length || new Set(modes).size !== modes.length || modes.some(m =>
+    !['recent','recent_no_family_gate','recent_fixture_only'].includes(m)))
+  throw Error('Invalid reference experiment modes');
+const report = {status:'running',model:'odysseus-qwen3.5-tools-pre-heretic',
+  thinking:false,cases,modes,runs:[],scope:'plain-title synthetic notes in 7011 Agent UI; no production default change'};
+const save = () => fs.writeFileSync(manifest,JSON.stringify(report,null,2)+'\n');
+save();
+try {
+  for (let index=0; index<cases.length; index++) {
+    const order = modes.slice(index%modes.length).concat(modes.slice(0,index%modes.length));
+    for (const mode of order) {
+      const file = path.join(root,'reports',`reference-${stamp}-${cases[index]}-${mode}.json`);
+      await new Promise((resolve,reject) => {
+        const p = spawn(process.execPath,[path.join(root,'scripts/verify_multi_note_delete_followup.mjs')],{
+          cwd:root,env:{...process.env,TITLE_STYLE:'plain',AUDIT_FINAL:'true',
+            ROUTING_MODE:mode,FOLLOWUP_CASE:cases[index],REPORT_PATH:file},
+          stdio:['ignore','pipe','pipe'],
+        });
+        p.stdout.resume(); p.stderr.resume();
+        p.on('error',reject); p.on('exit',resolve);
+      });
+      const result = JSON.parse(fs.readFileSync(file,'utf8'));
+      const cleaned = Object.keys(result.cleanup || {}).length === 4 && Object.values(result.cleanup).every(Boolean);
+      const setupOK = result.turns?.slice(0,2).length === 2 && result.turns.slice(0,2).every(t=>Object.values(t.checks).every(Boolean));
+      report.runs.push({case:cases[index],mode,outcome:result.outcome || null,setup_ok:setupOK,
+        cleanup:cleaned,report:path.relative(root,file),diagnostics:result.diagnostics || null});
+      save();
+      if (result.error || !result.outcome || !cleaned || !setupOK)
+        throw Error(`Invalid experiment/precondition in ${path.basename(file)}: ${result.error || 'setup/outcome/cleanup missing'}`);
+      if (!result.outcome.unrelated_preserved) throw Error('Unrelated data changed; stop testing.');
+    }
+  }
+  report.status='measured';
+} catch(error) {
+  report.status='blocked';report.error=String(error.message).slice(0,500);
+}
+save();
+console.log(JSON.stringify({manifest,status:report.status,completed:report.runs.length}));

scripts/compare_schema_thinking.mjs

@@ -0,0 +1,286 @@
+#!/usr/bin/env node
+// Capture real fixture UI requests in RAM, then replay identical requests without
+// executing proposed tools. Never persist prompts, private tool results or reasoning.
+import fs from 'node:fs';
+import http from 'node:http';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import {spawn, execFileSync} from 'node:child_process';
+import {fileURLToPath} from 'node:url';
+import {expectedNoteTitles} from './note_test_oracle.mjs';
+
+const root = path.resolve(new URL('..', import.meta.url).pathname);
+const upstream = 'http://100.67.207.85:19184/v1/chat/completions';
+const hash = value => crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex');
+const userText = body => body.messages.findLast(m=>m.role==='user')?.content;
+const titleNorm = s => String(s || '').trim().toLowerCase().replace(/^reminder\s*:\s*/, '').replace(/\s+/g,' ');
+
+export function recordsIn(messages) {
+  return messages.filter(m=>m.role==='tool').flatMap(m=>{
+    let content=String(m.content || '');
+    try { const obj=JSON.parse(content); content=obj.results || obj.stdout || obj.output || content; } catch {}
+    return [...String(content).matchAll(/- \[([a-f0-9-]{36})\] \*\*([^\n]+?)\*\*/g)]
+      .map(match=>({id:match[1],title:match[2]}));
+  });
+}
+
+export function reformatNoteResult(content, format) {
+  if(!['quoted','jsonl'].includes(format)) throw Error('Unknown note result format');
+  let wrapper, key, text=content;
+  try {
+    wrapper=JSON.parse(content);
+    key=['results','stdout','output'].find(k=>typeof wrapper?.[k]==='string');
+    if(!key) throw Error('Unsupported result wrapper');
+    text=wrapper[key];
+  } catch(error) {
+    if(wrapper!==undefined) throw error;
+  }
+  const lines=String(text).split('\n');
+  const rows=lines.map(line=>{
+    const m=line.match(/^- \[([a-f0-9-]{36})\] \*\*(.+?)\*\*(.*)$/);
+    if(!m) throw Error('Refuse to drop unrecognized result data');
+    return {id:m[1],title:m[2],suffix:m[3]};
+  });
+  const formatted=rows.map(r=>format==='quoted'
+    ? `- [${r.id}] ${JSON.stringify(r.title)}${r.suffix}` : JSON.stringify(r)).join('\n');
+  // Round-trip the presentation before using it; preserve record order and all
+  // original fields, including tags/type/pinning suffixes and wrapper metadata.
+  const decoded=formatted.split('\n').map(line=>{
+    if(format==='jsonl') return JSON.parse(line);
+    const m=line.match(/^- \[([a-f0-9-]{36})\] ("(?:[^"\\]|\\.)*")(.*)$/);
+    if(!m) throw Error('Quoted format failed round trip');
+    return {id:m[1],title:JSON.parse(m[2]),suffix:m[3]};
+  });
+  if(JSON.stringify(decoded)!==JSON.stringify(rows)) throw Error('Result data changed');
+  if(key) {wrapper[key]=formatted;return JSON.stringify(wrapper);}
+  return formatted;
+}
+
+export function scoreCalls(calls, records, expected) {
+  const selected=[], invalid=[];
+  let readCalls=0;
+  for(const call of calls) {
+    let args;
+    try { args=JSON.parse(call.function.arguments); } catch {invalid.push('invalid_json');continue;}
+    if(!args || typeof args!=='object' || Array.isArray(args)) {invalid.push('invalid_arguments');continue;}
+    if(call.function.name!=='manage_notes') {invalid.push('other_tool');continue;}
+    if(['list','search','find','view'].includes(args.action)) {readCalls++;continue;}
+    if(!['delete','remove'].includes(args.action)) {invalid.push('other_action');continue;}
+    const id=String(args.id || args.note_id || args.noteId || '').trim();
+    let matches=id ? records.filter(r=>r.id.startsWith(id)) : [];
+    if(!matches.length) matches=records.filter(r=>titleNorm(r.title)===titleNorm(args.title || args.query || args.text));
+    if(matches.length!==1) {invalid.push(matches.length?'ambiguous_target':'unknown_target');continue;}
+    selected.push(matches[0].title);
+  }
+  const unique=[...new Set(selected)].sort();
+  return {exact_target_proposal:invalid.length===0 && selected.length===unique.length && JSON.stringify(unique)===JSON.stringify([...expected].sort()),
+    proposal_stage_only:true,
+    selected_titles:unique,invalid,read_calls:readCalls,duplicate_targets:selected.length-unique.length,
+    wrong_targets:unique.filter(t=>!expected.includes(t)),missing_targets:expected.filter(t=>!unique.includes(t))};
+}
+
+export function auditHistory(request, priorRequests, ids, savedEvidence=null) {
+  const toolResults=request.messages.filter(m=>m.role==='tool');
+  const priorResults=priorRequests.flatMap(r=>r.messages.filter(m=>m.role==='tool'));
+  const noteResult=priorResults.find(m=>ids.every(id=>String(m.content).includes(id)));
+  const records=recordsIn(request.messages).filter(r=>ids.includes(r.id));
+  const callIds=new Set(request.messages.flatMap(m=>(m.tool_calls || []).map(c=>c.id)));
+  return {message_roles:request.messages.map(m=>m.role),
+    user_turns:request.messages.filter(m=>m.role==='user').length,
+    tool_result_count:toolResults.length,
+    fixture_ids_present:ids.filter(id=>records.some(r=>r.id===id)).length,
+    exact_prior_note_result_preserved:savedEvidence ? toolResults.some(m=>
+      m.tool_call_id===savedEvidence.call_id && hash(m.content)===savedEvidence.content_sha256)
+      : Boolean(noteResult && toolResults.some(m=>
+        m.tool_call_id===noteResult.tool_call_id && m.content===noteResult.content)),
+    comparison_source:savedEvidence?'prior_turn_saved_tool_result':'prior_outbound_request',
+    orphan_tool_results:toolResults.filter(m=>!callIds.has(m.tool_call_id)).length,
+    messages_sha256:hash(request.messages),
+    compact_schemas_sha256:hash(request.tools),
+    offered_tools:(request.tools || []).map(s=>s.function.name),
+    thinking:request.chat_template_kwargs?.enable_thinking,
+    forced_tool_choice:request.tool_choice || null};
+}
+
+async function completion(body) {
+  const started=performance.now();
+  let buffer='',firstDelta=null,firstTool=null,usage={},finish=null,content='',reasoningChars=0;
+  const calls=new Map();
+  const response=await fetch(upstream,{method:'POST',headers:{'Content-Type':'application/json'},
+    body:JSON.stringify(body),signal:AbortSignal.timeout(90000)});
+  if(!response.ok) throw Error(`Inference HTTP ${response.status}`);
+  const consume = frame => {
+    const raw=frame.split('\n').filter(l=>l.startsWith('data:')).map(l=>l.slice(5).trimStart()).join('\n');
+    if(!raw || raw==='[DONE]') return;
+    const p=JSON.parse(raw);
+    if(p.usage) usage=p.usage;
+    for(const c of p.choices || []) {
+      if(c.finish_reason) finish=c.finish_reason;
+      const d=c.delta || {};
+      if(d.content || d.reasoning_content || d.reasoning || d.tool_calls?.length)
+        firstDelta ??= (performance.now()-started)/1000;
+      reasoningChars+=String(d.reasoning_content || d.reasoning || '').length;
+      content+=d.content || '';
+      for(const part of d.tool_calls || []) {
+        firstTool ??= (performance.now()-started)/1000;
+        const v=calls.get(part.index) || {function:{name:'',arguments:''}};
+        v.function.name+=part.function?.name || '';
+        v.function.arguments+=part.function?.arguments || '';
+        calls.set(part.index,v);
+      }
+    }
+  };
+  for await(const chunk of response.body) {
+    buffer+=Buffer.from(chunk).toString('utf8');
+    let end;
+    while((end=buffer.indexOf('\n\n'))>=0) {consume(buffer.slice(0,end));buffer=buffer.slice(end+2);}
+  }
+  if(buffer.trim()) consume(buffer);
+  const endThink=content.indexOf('</think>');
+  const unparsedThinking=endThink>=0 || content.includes('<think>');
+  const seconds=(performance.now()-started)/1000;
+  return {calls:[...calls.values()],metrics:{seconds,first_delta_s:firstDelta,first_tool_delta_s:firstTool,
+    input_tokens:usage.prompt_tokens ?? null,output_tokens:usage.completion_tokens ?? null,
+    generation_tok_s:usage.completion_tokens && firstDelta!==null && seconds>firstDelta
+      ? usage.completion_tokens/(seconds-firstDelta) : null,
+    finish_reason:finish,reasoning_chars:reasoningChars,thinking_in_content:unparsedThinking,
+    content_chars:content.length}};
+}
+
+async function main() {
+  const stamp=new Date().toISOString().replace(/[:.]/g,'-');
+  const formatting=process.env.EXPERIMENT==='result_format';
+  const prefix=formatting?'result-format':'schema-thinking';
+  const file=path.join(root,'reports',`${prefix}-${stamp}.json`);
+  const cases=(process.env.PROBE_CASES || 'original,typo,drinks,schedule_words,quoted,negative,subset,single').split(',');
+  const fullSchemas=formatting?[]:JSON.parse(execFileSync('/home/pewds/odysseus-cookbook-fresh/.venv/bin/python',[
+    '-c','import json; from src.tool_schemas import FUNCTION_TOOL_SCHEMAS; print(json.dumps(FUNCTION_TOOL_SCHEMAS))'
+  ],{cwd:root,maxBuffer:4*1024*1024,encoding:'utf8'}));
+  const report={status:'running',scope:'Actual 7011 fixture history audit; direct proposal replay does not execute tools.',
+    experiment:prefix,model:'odysseus-qwen3.5-tools-pre-heretic',temperature:0,max_tokens:2048,cases,runs:[]};
+  const save=()=>fs.writeFileSync(file,JSON.stringify(report,null,2)+'\n');
+  let captured=[];
+  const server=http.createServer(async(req,res)=>{
+    if(req.method!=='POST' || req.url!=='/v1/chat/completions') {res.writeHead(404).end();return;}
+    try {
+      let raw=''; for await(const c of req) {raw+=c;if(raw.length>2*1024*1024)throw Error('Request too large');}
+      const body=JSON.parse(raw);
+      if(body.model!==report.model) {res.writeHead(400).end();return;}
+      captured.push(structuredClone(body));
+      const result=await fetch(upstream,{method:'POST',headers:{'Content-Type':'application/json'},
+        body:raw,signal:AbortSignal.timeout(90000)});
+      res.writeHead(result.status,{'Content-Type':result.headers.get('content-type') || 'text/event-stream'});
+      for await(const c of result.body) res.write(c);
+      res.end();
+    } catch {if(!res.headersSent)res.writeHead(502);res.end();}
+  });
+  await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));
+  const local=`http://127.0.0.1:${server.address().port}/v1/chat/completions`;
+  const endpointId=crypto.randomUUID();
+  const endpointName=`[schema-thinking-fixture] ${endpointId}`;
+  const endpointDB=(operation)=>execFileSync('/home/pewds/odysseus-cookbook-fresh/.venv/bin/python',[
+    '-c', `import sqlite3,sys,json
+c=sqlite3.connect('/home/pewds/odysseus-cookbook-fresh/data/app.db')
+op,ident,name,url,model=sys.argv[1:]
+if op=='add':
+ c.execute('INSERT INTO model_endpoints (id,name,base_url,owner,is_enabled,cached_models,pinned_models,model_type,endpoint_kind,model_refresh_mode,supports_tools,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP)',(ident,name,url,'sft_alex_creator',1,json.dumps([model]),json.dumps([model]),'llm','local','manual',1))
+else:
+ c.execute('DELETE FROM model_endpoints WHERE id=? AND name=? AND owner=? AND base_url=?',(ident,name,'sft_alex_creator',url))
+c.commit()
+print(c.execute('SELECT count(*) FROM model_endpoints WHERE id=?',(ident,)).fetchone()[0])`,
+    operation,endpointId,endpointName,local.replace('/chat/completions',''),report.model,
+  ],{encoding:'utf8'}).trim();
+  save();
+  try {
+    if(endpointDB('add')!=='1') throw Error('Fixture proxy registration failed');
+    for(const [index,name] of cases.entries()) {
+      captured=[];
+      const uiFile=path.join(root,'reports',`${formatting?'format-ui':'schema-ui'}-${stamp}-${name}.json`);
+      await new Promise((resolve,reject)=>{
+        const p=spawn(process.execPath,['scripts/verify_multi_note_delete_followup.mjs'],{cwd:root,
+          env:{...process.env,ENDPOINT_URL:local,ENDPOINT_ID:endpointId,ROUTING_MODE:'recent_fixture_only',
+            FOLLOWUP_CASE:name,TITLE_STYLE:'plain',REPORT_PATH:uiFile,AUDIT_FINAL:'true'},
+          stdio:['ignore','pipe','pipe']});
+        p.stdout.resume();p.stderr.resume();p.on('error',reject);p.on('exit',resolve);
+      });
+      const ui=JSON.parse(fs.readFileSync(uiFile,'utf8'));
+      if(ui.error || !ui.outcome || !ui.outcome.unrelated_preserved ||
+          Object.keys(ui.cleanup).length!==4 || !Object.values(ui.cleanup).every(Boolean))
+        throw Error(`Invalid UI fixture capture: ${name}; see child report`);
+      const ids=Object.keys(ui.cleanup).filter(k=>k!=='session');
+      // Third user turn is the real follow-up; later rounds retain that request.
+      const firstIndex=captured.findIndex(r=>r.messages.filter(m=>m.role==='user').length===3);
+      if(firstIndex<0) throw Error(`No real outbound follow-up captured: ${name}`);
+      const request=captured[firstIndex];
+      const audit=auditHistory(request,captured.slice(0,firstIndex),ids,ui.prior_note_evidence);
+      const records=recordsIn(request.messages).filter(r=>ids.includes(r.id));
+      const expected=expectedNoteTitles(name,records.map(r=>r.title));
+      const run={case:name,ui_report:path.relative(root,uiFile),ui_outcome:ui.outcome,history:audit,variants:[]};
+      report.runs.push(run);save();
+      if(audit.fixture_ids_present!==3 || !audit.exact_prior_note_result_preserved || audit.orphan_tool_results)
+        throw Error(`History audit failed: ${name}`);
+      if(formatting) {
+        const noteIndex=request.messages.findIndex(m=>m.role==='tool' &&
+          m.tool_call_id===ui.prior_note_evidence.call_id);
+        const variants=['original','quoted','jsonl'];
+        const order=variants.slice(index%3).concat(variants.slice(0,index%3));
+        for(const variant of order) {
+          const body={...structuredClone(request),max_tokens:2048};
+          if(variant!=='original') body.messages[noteIndex].content=
+            reformatNoteResult(body.messages[noteIndex].content,variant);
+          const otherMessagesUnchanged=request.messages.every((m,i)=>i===noteIndex || hash(m)===hash(body.messages[i]));
+          const sameSchemas=hash(body.tools)===hash(request.tools);
+          if(!otherMessagesUnchanged || !sameSchemas || body.chat_template_kwargs.enable_thinking!==false)
+            throw Error('Non-format change in formatting comparison');
+          const result=await completion(body);
+          run.variants.push({variant,other_messages_unchanged:otherMessagesUnchanged,
+            schemas_unchanged:sameSchemas,lossless_result:true,
+            result_chars:body.messages[noteIndex].content.length,
+            ...scoreCalls(result.calls,records,expected),metrics:result.metrics});
+          save();
+        }
+        console.log(JSON.stringify({case:name,variants:run.variants.map(v=>({mode:v.variant,
+          exact:v.exact_target_proposal,seconds:v.metrics.seconds}))}));
+        continue;
+      }
+      const full=request.tools.map(s=>fullSchemas.find(f=>f.function.name===s.function.name));
+      if(full.some(s=>!s)) throw Error('Missing canonical full schema');
+      run.schema_comparison={compact_bytes:JSON.stringify(request.tools).length,full_bytes:JSON.stringify(full).length,
+        same_tool_names:JSON.stringify(full.map(s=>s.function.name))===JSON.stringify(request.tools.map(s=>s.function.name)),
+        full_schemas_sha256:hash(full)};
+      const variants=['compact_off','full_off','compact_on'];
+      const order=variants.slice(index%3).concat(variants.slice(0,index%3));
+      for(const variant of order) {
+        const body={...structuredClone(request),max_tokens:2048,
+          tools:variant==='full_off'?full:request.tools,
+          chat_template_kwargs:{...request.chat_template_kwargs,enable_thinking:variant==='compact_on'}};
+        const result=await completion(body);
+        run.variants.push({variant,messages_sha256:hash(body.messages),
+          ...scoreCalls(result.calls,records,expected),metrics:result.metrics});
+        save();
+      }
+      // Error-only progressive thinking replays the actual next model request,
+      // after successful partial effects and tool errors; it never re-executes them.
+      const second=captured.slice(firstIndex+1).find(r=>userText(r)===userText(request));
+      const failed=ui.turns.at(-1).errors.length>0;
+      run.progressive={triggered:failed};
+      if(failed && second) {
+        const remaining=ui.turns.at(-1).remaining_fixture_titles;
+        const result=await completion({...structuredClone(second),max_tokens:2048,
+          chat_template_kwargs:{...second.chat_template_kwargs,enable_thinking:true}});
+        run.progressive={triggered:true,...scoreCalls(result.calls,records,remaining.filter(t=>expected.includes(t))),
+          metrics:result.metrics,scope:'Error-round recovery proposal only; not executed or timed end-to-end.'};
+      }
+      save();
+      console.log(JSON.stringify({case:name,history_ok:true,variants:run.variants.map(v=>({mode:v.variant,
+        exact:v.exact_target_proposal,seconds:v.metrics.seconds})),progressive:run.progressive.triggered}));
+    }
+    report.status='measured';
+  } catch(e) {report.status='blocked';report.error=String(e.message).slice(0,300);
+    report.capture_diagnostic={requests:captured.length,user_turn_counts:captured.map(r=>r.messages.filter(m=>m.role==='user').length)};}
+  finally {captured=[];server.closeAllConnections();await new Promise(resolve=>server.close(resolve));
+    report.fixture_endpoint_removed=endpointDB('remove')==='0';save();}
+  console.log(JSON.stringify({report:file,status:report.status,completed:report.runs.length,error:report.error}));
+}
+
+if(process.argv[1] && path.resolve(process.argv[1])===fileURLToPath(import.meta.url)) await main();

scripts/compare_tool_routing.mjs

@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+/** Sequential, reproducible UI comparisons. Never changes the live default. */
+import fs from 'node:fs';
+import path from 'node:path';
+import {spawn} from 'node:child_process';
+
+const root = path.resolve(new URL('..', import.meta.url).pathname);
+const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+const reportPath = path.join(root, 'reports', `routing-comparison-${stamp}.json`);
+const endpoint = process.env.ENDPOINT_URL || 'http://100.67.207.85:19184/v1/chat/completions';
+const model = process.env.MODEL || 'odysseus-qwen3.5-tools-pre-heretic';
+const report = {status: 'running', model, thinking: false, repetitions: 3, runs: [],
+  coverage: '11 read conversation chains plus calendar/notes multi-delete; broader CRUD/mobile gate remains required',
+  promotion_eligible: false};
+const save = () => fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n');
+fs.mkdirSync(path.dirname(reportPath), {recursive: true});
+save();
+async function preflight() {
+  const response = await fetch(endpoint, {
+    method: 'POST', headers: {'Content-Type': 'application/json'},
+    body: JSON.stringify({model, messages: [{role: 'user', content: 'Reply OK.'}],
+      temperature: 0, max_tokens: 8, stream: false,
+      chat_template_kwargs: {enable_thinking: false}}),
+    signal: AbortSignal.timeout(10000),
+  });
+  if (!response.ok) throw Error(`Inference preflight HTTP ${response.status}`);
+  const body = await response.json();
+  if (!body.choices?.length) throw Error('Inference preflight returned no choices');
+}
+async function execute(script, env) {
+  return await new Promise((resolve, reject) => {
+    const child = spawn(process.execPath, [path.join(root, 'scripts', script)], {
+      cwd: root, env: {...process.env, ...env}, stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    // Child artifacts are authoritative; do not copy private console output.
+    child.stdout.resume(); child.stderr.resume();
+    child.on('error', reject); child.on('exit', resolve);
+  });
+}
+try {
+  for (let repeat = 1; repeat <= 3; repeat++) {
+    // Rotate ordering to reduce warm-cache/order bias. Run serially: mutation
+    // snapshots must never race another test's fixture creation or cleanup.
+    const modes = ['baseline', 'recent', 'all'];
+    const order = modes.slice(repeat - 1).concat(modes.slice(0, repeat - 1));
+    for (const mode of order) {
+      await preflight();
+      for (const suite of ['read', 'notes']) {
+        const childPath = path.join(root, 'reports', `routing-${stamp}-${mode}-${repeat}-${suite}.json`);
+        const code = await execute(suite === 'read'
+          ? 'verify_interleaved_tool_followups.mjs' : 'verify_multi_note_delete_followup.mjs', {
+          ROUTING_MODE: mode, REPORT_PATH: childPath,
+          OWNER: suite === 'read' ? 'pewds' : 'sft_alex_creator', KEEP_SESSION: 'false',
+        });
+        const result = JSON.parse(fs.readFileSync(childPath, 'utf8'));
+        report.runs.push({mode, repeat, suite, exit_code: code, status: result.status,
+          summary: result.summary || null, report: path.relative(root, childPath)});
+        save();
+        if (result.chains?.some(c => c.infrastructure_failure) || /PRECONDITION|Timeout|ECONN/.test(result.error || '')) {
+          throw Error(`Infrastructure failure in ${suite}; inspect ${childPath}`);
+        }
+      }
+    }
+  }
+  report.status = 'measured';
+} catch (error) {
+  report.status = 'blocked'; report.blocker = String(error.message).slice(0, 500);
+}
+save();
+console.log(JSON.stringify({report: reportPath, status: report.status, runs: report.runs.length}));
+if (report.status !== 'measured') process.exitCode = 1;

scripts/curate_public_search_seed_cases.py

@@ -0,0 +1,326 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import random
+import re
+import time
+from pathlib import Path
+from typing import Any
+
+from datasets import load_dataset
+
+from run_odysseus_search_teacher_pipeline import call_deepseek_json, db_deepseek_endpoint
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_OUT = REPO_ROOT / "data/evals/ody_public_search_seed_20260825/cases.json"
+DEFAULT_LOCAL_SEEDS = [
+    Path("/home/pewds/deep_research_task_ui_seeds.jsonl"),
+]
+
+
+QUESTION_RE = re.compile(r"\?$|^(?:who|what|when|where|why|how|which|can|does|do|is|are|was|were)\b", re.I)
+PRIVATE_RE = re.compile(
+    r"\b(my|our)\s+(?:email|inbox|calendar|notes?|documents?|files?|computer|desktop|downloads?|contacts?)\b|"
+    r"\b(?:send|delete|archive|mark|reply to|draft|schedule|remind me|open my)\b",
+    re.I,
+)
+TOO_CURRENT_RE = re.compile(r"\b(?:today|right now|current|latest|this week|this month|2026|2025)\b", re.I)
+
+
+def stable_id(prefix: str, value: Any) -> str:
+    text = json.dumps(value, sort_keys=True, ensure_ascii=True)
+    return f"{prefix}_{hashlib.sha256(text.encode('utf-8')).hexdigest()[:16]}"
+
+
+def clean_text(value: Any) -> str:
+    return re.sub(r"\s+", " ", str(value or "")).strip()
+
+
+def useful_question(text: str) -> bool:
+    q = clean_text(text)
+    if len(q) < 18 or len(q) > 240:
+        return False
+    if PRIVATE_RE.search(q):
+        return False
+    if not QUESTION_RE.search(q):
+        return False
+    if len(q.split()) < 5:
+        return False
+    return True
+
+
+def prompt_variant(question: str, source: str, index: int) -> str:
+    q = clean_text(question).rstrip("?")
+    variants = [
+        f"Search the web and answer this: {q}?",
+        f"Can you look up {q} and give me the answer?",
+        f"Find a reliable source for this and answer briefly: {q}?",
+        f"Use search to verify: {q}?",
+        f"I need a quick sourced answer: {q}?",
+    ]
+    if source == "hotpot_qa":
+        variants.extend([
+            f"Search for the two facts needed to answer this: {q}?",
+            f"Look this up and combine the evidence: {q}?",
+        ])
+    return variants[index % len(variants)]
+
+
+def add_candidate(out: list[dict[str, Any]], seen: set[str], *, source: str, question: str, answer: Any = "", family: str = "") -> None:
+    question = clean_text(question)
+    if not useful_question(question):
+        return
+    key = question.lower()
+    if key in seen:
+        return
+    seen.add(key)
+    idx = len(out)
+    out.append({
+        "source": source,
+        "source_id": stable_id(source, question),
+        "question": question,
+        "answer_hint": clean_text(answer)[:220],
+        "family": family or ("fresh_or_date_sensitive" if TOO_CURRENT_RE.search(question) else "public_fact_search"),
+        "user": prompt_variant(question, source, idx),
+    })
+
+
+def sample_nq_open(out: list[dict[str, Any]], seen: set[str], target: int, seed: int) -> None:
+    ds = load_dataset("nq_open", split="train", streaming=True)
+    rng = random.Random(seed)
+    for i, row in enumerate(ds):
+        if i > 250_000 or len(out) >= target:
+            break
+        if rng.random() > 0.045:
+            continue
+        add_candidate(
+            out,
+            seen,
+            source="nq_open",
+            question=row.get("question"),
+            answer=row.get("answer"),
+            family="simple_public_fact",
+        )
+
+
+def sample_hotpot(out: list[dict[str, Any]], seen: set[str], target: int, seed: int) -> None:
+    ds = load_dataset("hotpot_qa", "distractor", split="train", streaming=True)
+    rng = random.Random(seed + 17)
+    for i, row in enumerate(ds):
+        if i > 180_000 or len(out) >= target:
+            break
+        if rng.random() > 0.075:
+            continue
+        add_candidate(
+            out,
+            seen,
+            source="hotpot_qa",
+            question=row.get("question"),
+            answer=row.get("answer"),
+            family=f"multi_hop_{clean_text(row.get('type') or 'qa')}",
+        )
+
+
+def load_local(out: list[dict[str, Any]], seen: set[str], paths: list[Path], target: int) -> None:
+    for path in paths:
+        if not path.exists():
+            continue
+        for line in path.read_text(encoding="utf-8").splitlines():
+            if len(out) >= target:
+                return
+            if not line.strip():
+                continue
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            prompt = clean_text(row.get("prompt") or row.get("user") or row.get("question"))
+            if not prompt or PRIVATE_RE.search(prompt) or len(prompt) > 1600:
+                continue
+            key = prompt.lower()
+            if key in seen:
+                continue
+            seen.add(key)
+            out.append({
+                "source": f"local:{path.name}",
+                "source_id": clean_text(row.get("task_id") or row.get("id") or stable_id(path.name, prompt)),
+                "question": prompt,
+                "answer_hint": clean_text(row.get("reference_solution") or row.get("answer"))[:500],
+                "family": clean_text(row.get("task_family") or row.get("family") or "local_web_research"),
+                "user": prompt,
+            })
+
+
+def heuristic_rank(item: dict[str, Any]) -> float:
+    q = item["question"].lower()
+    score = 0.0
+    score += 1.0 if item["source"] == "nq_open" else 0.0
+    score += 1.4 if item["source"] == "hotpot_qa" else 0.0
+    score += 1.0 if item["source"].startswith("local:") else 0.0
+    score += 0.4 if 7 <= len(q.split()) <= 22 else 0.0
+    score += 0.5 if re.search(r"\b(which|compare|both|between|relationship|part of|head office)\b", q) else 0.0
+    score += 0.3 if item.get("answer_hint") else 0.0
+    score -= 0.7 if TOO_CURRENT_RE.search(q) else 0.0
+    score -= 0.8 if re.search(r"\b(song|lyrics|movie cast|episode)\b", q) else 0.0
+    return score
+
+
+def deepseek_audit(endpoint: dict[str, str], items: list[dict[str, Any]], batch_size: int) -> dict[str, dict[str, Any]]:
+    audits: dict[str, dict[str, Any]] = {}
+    for start in range(0, len(items), batch_size):
+        batch = items[start:start + batch_size]
+        payload = {
+            "task": "Audit public web-search SFT seed prompts. Pick prompts that are natural, generic, useful for teaching a web_search/web_fetch agent, and not private/user-data tasks.",
+            "current_date": "2026-08-25",
+            "rating_scale": "0 reject, 1 weak, 2 usable, 3 good, 4 excellent",
+            "reject_if": [
+                "requires private data, email, calendar, local files, account access, login, or sending/deleting actions",
+                "too broad for a 1-3 web tool trace unless it is a small minority of deep research seeds",
+                "answer is purely subjective or does not benefit from search",
+                "current/date-sensitive but lacks a stable phrasing or source date expectation",
+                "unsafe medical/legal/financial advice beyond general sourced information",
+            ],
+            "items": [
+                {
+                    "id": item["source_id"],
+                    "source": item["source"],
+                    "family": item["family"],
+                    "user": item["user"],
+                    "answer_hint": item.get("answer_hint") or "",
+                }
+                for item in batch
+            ],
+            "return_schema": {
+                "audits": [
+                    {"id": "string", "rating": 0, "keep": False, "family": "string", "reason": "string"}
+                ]
+            },
+        }
+        result = call_deepseek_json(endpoint, payload, max_tokens=5000, temperature=0.15, json_mode=True)
+        for audit in result.get("audits") or []:
+            if not isinstance(audit, dict):
+                continue
+            item_id = clean_text(audit.get("id"))
+            if item_id:
+                audits[item_id] = audit
+        print(json.dumps({"stage": "deepseek_audit", "start": start, "batch": len(batch), "audited": len(audits)}), flush=True)
+    return audits
+
+
+def build_cases(items: list[dict[str, Any]], audits: dict[str, dict[str, Any]], count: int) -> list[dict[str, Any]]:
+    ranked: list[tuple[float, dict[str, Any], dict[str, Any]]] = []
+    for item in items:
+        audit = audits.get(item["source_id"]) or {}
+        rating = float(audit.get("rating") or 0)
+        if audit and not audit.get("keep"):
+            continue
+        if rating < 2:
+            continue
+        ranked.append((rating * 10 + heuristic_rank(item), item, audit))
+    ranked.sort(key=lambda x: x[0], reverse=True)
+    cases = []
+    family_counts: dict[str, int] = {}
+    source_counts: dict[str, int] = {}
+    for _score, item, audit in ranked:
+        family = clean_text(audit.get("family") or item.get("family") or "web")
+        source = item["source"]
+        if family_counts.get(family, 0) >= max(40, count // 5):
+            continue
+        if source_counts.get(source, 0) >= max(80, int(count * 0.55)):
+            continue
+        cases.append({
+            "id": f"public_search_seed_{len(cases):04d}",
+            "kind": "web",
+            "family": family,
+            "source_dataset": source,
+            "source_id": item["source_id"],
+            "user": item["user"],
+            "expect_first_tool": "web_search",
+            "allow_web_search": True,
+            "forbidden_final": ["WEB SEARCH RESULTS", "```sources", "Here are links", "Web sources", "from the search results", "snippets"],
+            "why_search_needed": clean_text(audit.get("reason") or "public source-backed answer"),
+            "answer_hint": item.get("answer_hint") or "",
+        })
+        family_counts[family] = family_counts.get(family, 0) + 1
+        source_counts[source] = source_counts.get(source, 0) + 1
+        if len(cases) >= count:
+            break
+    return cases
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--count", type=int, default=500)
+    parser.add_argument("--candidate-count", type=int, default=900)
+    parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+    parser.add_argument("--seed", type=int, default=20260825)
+    parser.add_argument("--audit-batch-size", type=int, default=35)
+    parser.add_argument("--skip-deepseek", action="store_true")
+    parser.add_argument("--local-seed", action="append", type=Path, default=[])
+    args = parser.parse_args()
+
+    rng = random.Random(args.seed)
+    candidates: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    local_paths = args.local_seed or DEFAULT_LOCAL_SEEDS
+    load_local(candidates, seen, local_paths, min(args.candidate_count, 120))
+    sample_hotpot(candidates, seen, max(args.candidate_count // 2, 260), args.seed)
+    sample_nq_open(candidates, seen, args.candidate_count, args.seed)
+    rng.shuffle(candidates)
+    candidates.sort(key=heuristic_rank, reverse=True)
+    candidates = candidates[: args.candidate_count]
+
+    endpoint = db_deepseek_endpoint()
+    endpoint["model"] = args.__dict__.get("teacher_model") or endpoint.get("model") or "deepseek-chat"
+    if args.skip_deepseek:
+        audits = {
+            item["source_id"]: {
+                "id": item["source_id"],
+                "rating": 3,
+                "keep": True,
+                "family": item["family"],
+                "reason": "heuristic keep",
+            }
+            for item in candidates
+        }
+    else:
+        audits = deepseek_audit(endpoint, candidates, args.audit_batch_size)
+
+    cases = build_cases(candidates, audits, args.count)
+    payload = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "generator": Path(__file__).name,
+        "current_date": "2026-08-25",
+        "source_notes": [
+            "nq_open / Natural Questions: CC-BY-SA-3.0 on Hugging Face.",
+            "hotpot_qa: CC-BY-SA-4.0 on Hugging Face.",
+            "local research seeds are prompt seeds only; inspect before training if exporting outside this workspace.",
+        ],
+        "candidate_count": len(candidates),
+        "audit_count": len(audits),
+        "cases": cases,
+        "audit_summary": {
+            "accepted_cases": len(cases),
+            "sources": {source: sum(1 for c in cases if c.get("source_dataset") == source) for source in sorted({c.get("source_dataset") for c in cases})},
+            "families": {family: sum(1 for c in cases if c.get("family") == family) for family in sorted({c.get("family") for c in cases})},
+        },
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    (args.out.parent / "seed_audits.json").write_text(json.dumps({"audits": audits}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    (args.out.parent / "seed_candidates.jsonl").write_text(
+        "".join(json.dumps(item, ensure_ascii=False) + "\n" for item in candidates),
+        encoding="utf-8",
+    )
+    print(json.dumps({"cases": len(cases), "candidates": len(candidates), "out": str(args.out)}, indent=2))
+    if len(cases) < args.count:
+        raise RuntimeError(f"Only built {len(cases)} cases; requested {args.count}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/curate_sft_trace_run.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+
+def _domain_from_session_name(name: str) -> str:
+    if " email " in name:
+        return "email"
+    if " notes " in name:
+        return "notes"
+    if " calendar " in name:
+        return "calendar"
+    return "other"
+
+
+def load_passing_report_sessions(report_path: Path) -> dict[str, dict[str, Any]]:
+    payload = json.loads(report_path.read_text(encoding="utf-8"))
+    sessions: dict[str, dict[str, Any]] = {}
+    for row in payload.get("results") or []:
+        session_id = str(row.get("session_id") or "")
+        if row.get("pass") is True and session_id:
+            sessions[session_id] = row
+    return sessions
+
+
+def load_trace_rows(trace_path: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for line_no, line in enumerate(trace_path.read_text(encoding="utf-8").splitlines(), start=1):
+        if not line.strip():
+            continue
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError as exc:
+            raise ValueError(f"{trace_path}:{line_no}: invalid JSON: {exc}") from exc
+        rows.append(row)
+    return rows
+
+
+def row_runtime_revision(row: dict[str, Any]) -> str:
+    direct = str(row.get("runtime_revision") or "").strip()
+    if direct:
+        return direct
+    metadata = row.get("metadata") or {}
+    if isinstance(metadata, str):
+        try:
+            metadata = json.loads(metadata)
+        except json.JSONDecodeError:
+            metadata = {}
+    if isinstance(metadata, dict):
+        return str(metadata.get("runtime_revision") or "").strip()
+    return ""
+
+
+def curate_rows(
+    rows: list[dict[str, Any]],
+    passing_sessions: dict[str, dict[str, Any]],
+    *,
+    require_thinking: bool = False,
+    require_runtime_revision: bool = False,
+    expected_runtime_revision: str = "",
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    curated: list[dict[str, Any]] = []
+    seen_sessions: set[str] = set()
+    no_thinking_sessions: set[str] = set()
+    missing_runtime_revision_sessions: set[str] = set()
+    mismatched_runtime_revision_sessions: set[str] = set()
+    skipped_no_thinking = 0
+    skipped_missing_runtime_revision = 0
+    skipped_mismatched_runtime_revision = 0
+    duplicate_sessions = 0
+    expected_runtime_revision = str(expected_runtime_revision or "").strip()
+
+    for row in rows:
+        session_id = str(row.get("session_id") or "")
+        if session_id not in passing_sessions:
+            continue
+        if session_id in seen_sessions:
+            duplicate_sessions += 1
+            continue
+        if require_thinking and not str(row.get("thinking") or "").strip():
+            no_thinking_sessions.add(session_id)
+            skipped_no_thinking += 1
+            continue
+        runtime_revision = row_runtime_revision(row)
+        if require_runtime_revision and not runtime_revision:
+            missing_runtime_revision_sessions.add(session_id)
+            skipped_missing_runtime_revision += 1
+            continue
+        if expected_runtime_revision and runtime_revision != expected_runtime_revision:
+            mismatched_runtime_revision_sessions.add(session_id)
+            skipped_mismatched_runtime_revision += 1
+            continue
+        seen_sessions.add(session_id)
+        enriched = dict(row)
+        enriched["eval_case_id"] = passing_sessions[session_id].get("id")
+        enriched["eval_domain"] = passing_sessions[session_id].get("domain")
+        if runtime_revision:
+            enriched["runtime_revision"] = runtime_revision
+        curated.append(enriched)
+
+    missing_sessions = sorted(set(passing_sessions) - seen_sessions)
+    missing_without_reason = sorted(
+        set(missing_sessions)
+        - no_thinking_sessions
+        - missing_runtime_revision_sessions
+        - mismatched_runtime_revision_sessions
+    )
+    domains = Counter(str(row.get("eval_domain") or _domain_from_session_name(row.get("session_name") or "")) for row in curated)
+    summary = {
+        "rows": len(curated),
+        "report_passing_sessions": len(passing_sessions),
+        "missing_sessions": len(missing_sessions),
+        "missing_without_reason": len(missing_without_reason),
+        "duplicate_sessions_skipped": duplicate_sessions,
+        "skipped_no_thinking": skipped_no_thinking,
+        "skipped_missing_runtime_revision": skipped_missing_runtime_revision,
+        "skipped_mismatched_runtime_revision": skipped_mismatched_runtime_revision,
+        "expected_runtime_revision": expected_runtime_revision,
+        "domains": dict(sorted(domains.items())),
+        "rows_with_thinking": sum(1 for row in curated if str(row.get("thinking") or "").strip()),
+        "rows_with_tool_events": sum(1 for row in curated if row.get("tool_events")),
+        "rows_with_runtime_revision": sum(1 for row in curated if row_runtime_revision(row)),
+        "missing_session_ids": missing_sessions[:20],
+        "missing_without_reason_session_ids": missing_without_reason[:20],
+        "missing_runtime_revision_session_ids": sorted(missing_runtime_revision_sessions)[:20],
+        "mismatched_runtime_revision_session_ids": sorted(mismatched_runtime_revision_sessions)[:20],
+    }
+    return curated, summary
+
+
+def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Curate accepted Odysseus SFT traces for one eval report.")
+    parser.add_argument("--report", type=Path, required=True, help="Eval actual_results.json path.")
+    parser.add_argument("--trace", type=Path, required=True, help="Owner SFT trace JSONL path.")
+    parser.add_argument("--out", type=Path, required=True, help="Curated JSONL output path.")
+    parser.add_argument("--summary-out", type=Path, default=None, help="Optional summary JSON path.")
+    parser.add_argument("--require-thinking", action="store_true", help="Drop passing rows that lack thinking text.")
+    parser.add_argument("--require-runtime-revision", action="store_true", help="Drop passing rows that lack runtime revision provenance.")
+    parser.add_argument(
+        "--runtime-revision",
+        default=os.getenv("ODYSSEUS_RUNTIME_REVISION", ""),
+        help="Require this exact runtime revision. Defaults to ODYSSEUS_RUNTIME_REVISION.",
+    )
+    args = parser.parse_args()
+
+    passing_sessions = load_passing_report_sessions(args.report)
+    rows = load_trace_rows(args.trace)
+    expected_runtime_revision = str(args.runtime_revision or "").strip()
+    curated, summary = curate_rows(
+        rows,
+        passing_sessions,
+        require_thinking=args.require_thinking,
+        require_runtime_revision=args.require_runtime_revision or bool(expected_runtime_revision),
+        expected_runtime_revision=expected_runtime_revision,
+    )
+    write_jsonl(args.out, curated)
+    if args.summary_out:
+        args.summary_out.parent.mkdir(parents=True, exist_ok=True)
+        args.summary_out.write_text(json.dumps(summary, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
+    print(json.dumps(summary, indent=2, ensure_ascii=True))
+    return 0 if summary["missing_without_reason"] == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/eval_exact_file_routing.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""Run a small live-model evaluation for exact edit_file routing."""
+
+import argparse
+import asyncio
+import json
+import sys
+import tempfile
+from pathlib import Path
+
+import httpx
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from src.agent_loop import (
+    _WORKSPACE_AGENT_TOOLS,
+    _looks_like_exact_file_replacement,
+    stream_agent_loop,
+)
+
+
+EXACT_TEMPLATES = [
+    "In {path}, change status=old to status=new.",
+    "Replace `June 30` with `July 1` in {path}.",
+    "In {path}, update MODE=dev to MODE=prod.",
+    "Change ETA June 30 to ETA July 1 in {path}.",
+    "Replace owner=alice with owner=bob in {path}.",
+    "In {path}, change enabled=false to enabled=true.",
+    "Update color=red to color=green in {path}.",
+    "In {path}, replace port=8000 with port=9000.",
+    "Change queue=slow to queue=fast in {path}.",
+    "Replace draft with published in {path}.",
+    "In {path}, update retry=1 to retry=3.",
+    "Change region=west to region=east in {path}.",
+    "Replace level=info with level=warning in {path}.",
+    "In {path}, change feature=off to feature=on.",
+    "Update team=alpha to team=beta in {path}.",
+    "Replace pending with approved in {path}.",
+    "In {path}, change timeout=30 to timeout=60.",
+    "Change format=csv to format=json in {path}.",
+    "Replace stage=test with stage=production in {path}.",
+    "In {path}, update version=1 to version=2.",
+]
+
+CONTROL_PREFIXES = [
+    "Inspect {path}, then change old_value to new_value.",
+    "Read {path} first, then replace old_value with new_value.",
+    "Show the contents of {path}, then change old_value to new_value.",
+    "Open {path} and replace old_value with new_value.",
+    "Review {path} before changing old_value to new_value.",
+    "Use cat to inspect {path}, then replace old_value with new_value.",
+    "Examine {path}, then update old_value to new_value.",
+    "Look at {path} before replacing old_value with new_value.",
+    "Change old_value to new_value in {path} and verify the result.",
+    "Replace old_value with new_value in {path}, then run the tests.",
+]
+
+
+def _values(template: str) -> tuple[str, str]:
+    pairs = [
+        ("status=old", "status=new"), ("June 30", "July 1"),
+        ("MODE=dev", "MODE=prod"), ("ETA June 30", "ETA July 1"),
+        ("owner=alice", "owner=bob"), ("enabled=false", "enabled=true"),
+        ("color=red", "color=green"), ("port=8000", "port=9000"),
+        ("queue=slow", "queue=fast"), ("draft", "published"),
+        ("retry=1", "retry=3"), ("region=west", "region=east"),
+        ("level=info", "level=warning"), ("feature=off", "feature=on"),
+        ("team=alpha", "team=beta"), ("pending", "approved"),
+        ("timeout=30", "timeout=60"), ("format=csv", "format=json"),
+        ("stage=test", "stage=production"), ("version=1", "version=2"),
+    ]
+    return pairs[EXACT_TEMPLATES.index(template)]
+
+
+def _event(chunk: str):
+    if not chunk.startswith("data: ") or chunk.startswith("data: [DONE]"):
+        return None
+    try:
+        return json.loads(chunk[6:])
+    except json.JSONDecodeError:
+        return None
+
+
+async def _run_case(endpoint: str, model: str, owner: str, prompt: str, path: Path, expected: str):
+    chunks = []
+    starts = []
+    outputs = []
+    stream = stream_agent_loop(
+        endpoint,
+        model,
+        [{"role": "user", "content": prompt}],
+        temperature=0.2,
+        max_tokens=1024,
+        max_rounds=4,
+        max_tool_calls=4,
+        owner=owner,
+        workspace=str(path.parent),
+        relevant_tools=set(_WORKSPACE_AGENT_TOOLS),
+    )
+    async for chunk in stream:
+        chunks.append(chunk)
+        event = _event(chunk)
+        if not event:
+            continue
+        if event.get("type") == "tool_start":
+            starts.append(event.get("tool"))
+        elif event.get("type") == "tool_output":
+            outputs.append(event)
+    actual = path.read_text() if path.exists() else ""
+    return {
+        "classifier_exact": _looks_like_exact_file_replacement(prompt),
+        "tool_sequence": starts,
+        "tool_outputs": outputs,
+        "first_tool": starts[0] if starts else None,
+        "content_ok": actual == expected,
+        "actual_content": actual,
+        "response": "".join(
+            event.get("delta", "")
+            for chunk in chunks
+            if (event := _event(chunk)) and isinstance(event.get("delta"), str)
+        ),
+    }
+
+
+async def main(args):
+    models_url = args.endpoint.rstrip("/") + "/models"
+    try:
+        models_response = httpx.get(models_url, timeout=10)
+    except httpx.ConnectError:
+        # The same eval may run on the host or inside the backend container.
+        # Docker's host alias is container-only; use the host-published loopback
+        # endpoint when the evaluator is running outside Docker.
+        if "host.docker.internal" not in args.endpoint:
+            raise
+        args.endpoint = args.endpoint.replace("host.docker.internal", "127.0.0.1")
+        models_response = httpx.get(args.endpoint.rstrip("/") + "/models", timeout=10)
+    models_response.raise_for_status()
+    advertised = {
+        item.get("id")
+        for item in models_response.json().get("data", [])
+        if isinstance(item, dict)
+    }
+    if args.model not in advertised:
+        raise SystemExit(
+            f"Requested model {args.model!r} is not advertised by the endpoint; "
+            f"available={sorted(name for name in advertised if name)}"
+        )
+
+    output = Path(args.output)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    records = []
+    with output.open("w") as handle, tempfile.TemporaryDirectory(prefix="ody-exact-edit-") as root:
+        def emit(record):
+            records.append(record)
+            handle.write(json.dumps(record) + "\n")
+            handle.flush()
+            print(json.dumps(record), flush=True)
+
+        root_path = Path(root)
+        for repetition in range(1, args.repetitions + 1):
+            exact_templates = EXACT_TEMPLATES[:args.exact_limit] if args.exact_limit else EXACT_TEMPLATES
+            for index, template in enumerate(exact_templates, 1):
+                old, new = _values(template)
+                path = root_path / f"exact_{index}.txt"
+                path.write_text(old + "\n")
+                prompt = template.format(path=path)
+                result = await _run_case(args.endpoint, args.model, args.owner, prompt, path, new + "\n")
+                emit({
+                    "kind": "exact", "case": index, "repetition": repetition,
+                    "model": args.label or args.model, "request_model": args.model,
+                    "prompt": prompt, **result,
+                })
+
+            control_templates = [] if args.skip_controls else (
+                CONTROL_PREFIXES[:args.control_limit] if args.control_limit else CONTROL_PREFIXES
+            )
+            for index, template in enumerate(control_templates, 1):
+                path = root_path / f"control_{index}.txt"
+                path.write_text("old_value\n")
+                prompt = template.format(path=path)
+                result = await _run_case(args.endpoint, args.model, args.owner, prompt, path, "new_value\n")
+                emit({
+                    "kind": "control", "case": index, "repetition": repetition,
+                    "model": args.label or args.model, "request_model": args.model,
+                    "prompt": prompt, **result,
+                })
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--endpoint", required=True)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--label")
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--owner", default="pewds")
+    parser.add_argument("--repetitions", type=int, default=2)
+    parser.add_argument("--exact-limit", type=int, default=0)
+    parser.add_argument("--control-limit", type=int, default=0)
+    parser.add_argument("--skip-controls", action="store_true")
+    asyncio.run(main(parser.parse_args()))

scripts/eval_odysseus_app_route_smoke.py

@@ -0,0 +1,400 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+import time
+import uuid
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+import sys
+
+import httpx
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from core.database import CalendarCal, CalendarEvent, Document, Note, ScheduledTask, SessionLocal
+from scripts.eval_odysseus_everyday_live_hard import load_cases, score_case, write_markdown
+from scripts.eval_odysseus_live_hard_examples import _parse_tool_args
+from scripts.eval_odysseus_tool_use import _cookie, _raise_for_status_with_body, _sse_events, _visible_event_text
+from scripts.ody_eval_email_fixture import email_fixture
+
+
+DEFAULT_OWNER = "pewds"
+DEFAULT_TZ = "Asia/Tokyo"
+DEFAULT_TZ_OFFSET_MIN = 540
+
+
+def _default_cases() -> list[dict[str, Any]]:
+    return [
+        {
+            "id": "no_tool_web_search_engine",
+            "kind": "negative_web",
+            "user": "Explain what a web search engine does without looking anything up.",
+            "expect_no_tool": True,
+            "must_answer_any": ["search", "pages"],
+        },
+        {
+            "id": "positive_latest_email",
+            "kind": "email",
+            "user": "What's my latest email?",
+            "expect_first_tool": "mcp__email__list_emails",
+            "forbidden_tools": ["web_search"],
+            "must_answer_any": ["latest email", "UID"],
+        },
+    ]
+
+
+def _ensure_calendar(db: Any, owner: str) -> CalendarCal:
+    cal = db.query(CalendarCal).filter(CalendarCal.owner == owner).first()
+    if cal:
+        return cal
+    cal = CalendarCal(id=f"ody-app-gate-cal-{uuid.uuid4().hex[:8]}", owner=owner, name="Odysseus App Gate", source="local")
+    db.add(cal)
+    db.commit()
+    db.refresh(cal)
+    return cal
+
+
+def _precreate_calendar(db: Any, owner: str, fixture: dict[str, str]) -> str:
+    cal = _ensure_calendar(db, owner)
+    uid = f"ody-app-gate-event-{uuid.uuid4().hex[:8]}"
+    event = CalendarEvent(
+        uid=uid,
+        calendar_id=cal.id,
+        summary=fixture["summary"],
+        dtstart=datetime.fromisoformat(fixture["dtstart"]),
+        dtend=datetime.fromisoformat(fixture["dtend"]),
+        all_day=False,
+        is_utc=False,
+        origin="local",
+        status="confirmed",
+    )
+    db.add(event)
+    db.commit()
+    return uid
+
+
+def _create_session(client: httpx.Client, args: argparse.Namespace, case: dict[str, Any]) -> str:
+    create = client.post(
+        args.base_url.rstrip("/") + "/api/session",
+        data={
+            "name": "[eval-app-route] " + case["id"],
+            "endpoint_url": args.endpoint,
+            "endpoint_id": args.endpoint_id,
+            "model": args.model,
+            "skip_validation": "true",
+            "rag": "false",
+        },
+        timeout=30,
+    )
+    _raise_for_status_with_body(create)
+    return create.json()["id"]
+
+
+def _seed_case_state(
+    case: dict[str, Any], args: argparse.Namespace, session_id: str, client: httpx.Client
+) -> dict[str, Any]:
+    state = {"precreated_event_uid": "", "active_document_id": "", "active_document_before": ""}
+    db = SessionLocal()
+    try:
+        if case.get("precreate_calendar_event"):
+            state["precreated_event_uid"] = _precreate_calendar(db, args.owner, case["precreate_calendar_event"])
+        if case.get("active_document"):
+            # Seed through the same authenticated app runtime being evaluated.
+            # Importing SessionLocal here may point at a different deployment's
+            # SQLite file, producing cross-database foreign-key failures or,
+            # worse, a fixture the live 7011 process can never see.
+            fixture = case["active_document"]
+            created = client.post(
+                args.base_url.rstrip("/") + "/api/document",
+                json={
+                    "session_id": session_id,
+                    "title": fixture["title"],
+                    "language": fixture["language"],
+                    "content": fixture["content"],
+                },
+                timeout=30,
+            )
+            _raise_for_status_with_body(created)
+            state["active_document_id"] = created.json()["id"]
+            state["active_document_before"] = fixture["content"]
+    finally:
+        db.close()
+    return state
+
+
+def _tool_calls(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    calls: list[dict[str, Any]] = []
+    for event in events:
+        if event.get("type") != "tool_start":
+            continue
+        calls.append({
+            "tool": event.get("tool"),
+            "args": _parse_tool_args(event.get("full_command") or event.get("command")),
+            "round": event.get("round"),
+        })
+    return calls
+
+
+def _tool_outputs(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    outputs: list[dict[str, Any]] = []
+    for event in events:
+        if event.get("type") != "tool_output":
+            continue
+        outputs.append({
+            "tool": event.get("tool"),
+            "output": event.get("output"),
+            "exit_code": event.get("exit_code"),
+        })
+    return outputs
+
+
+def _collect_and_cleanup(
+    case: dict[str, Any], args: argparse.Namespace, seeded: dict[str, Any], client: httpx.Client
+) -> dict[str, Any]:
+    result_state: dict[str, Any] = {}
+    active_after = ""
+    db = SessionLocal()
+    try:
+        marker_text = case.get("marker") or ""
+        if marker_text:
+            note = db.query(Note).filter(Note.owner == args.owner, Note.archived == False).filter(  # noqa: E712
+                (Note.title.contains(marker_text)) | (Note.content.contains(marker_text))
+            ).first()
+            task = db.query(ScheduledTask).filter(ScheduledTask.owner == args.owner).filter(
+                (ScheduledTask.name.contains(marker_text)) | (ScheduledTask.prompt.contains(marker_text))
+            ).first()
+            events = db.query(CalendarEvent).filter(CalendarEvent.summary.contains(marker_text)).all()
+            result_state["note_found"] = bool(note)
+            result_state["task_found"] = bool(task)
+            result_state["events"] = [
+                {
+                    "uid": event.uid,
+                    "summary": event.summary,
+                    "dtstart": event.dtstart.isoformat(),
+                    "is_utc": bool(event.is_utc),
+                    "status": event.status,
+                }
+                for event in events
+            ]
+            if note:
+                db.delete(note)
+            if task:
+                db.delete(task)
+            for event in events:
+                db.delete(event)
+        active_doc_id = seeded.get("active_document_id") or ""
+        if active_doc_id:
+            response = client.get(
+                args.base_url.rstrip("/") + f"/api/document/{active_doc_id}", timeout=15
+            )
+            if response.is_success:
+                active_after = response.json().get("current_content") or ""
+                result_state["active_document_changed"] = active_after != (seeded.get("active_document_before") or "")
+            with contextlib.suppress(Exception):
+                client.delete(
+                    args.base_url.rstrip("/") + f"/api/document/{active_doc_id}", timeout=15
+                )
+        db.commit()
+    finally:
+        db.close()
+    return {"state": result_state, "active_document_after": active_after}
+
+
+def _run_turn(client: httpx.Client, args: argparse.Namespace, case: dict[str, Any]) -> dict[str, Any]:
+    session_id = _create_session(client, args, case)
+    seeded = _seed_case_state(case, args, session_id, client)
+    events: list[dict[str, Any]] = []
+    prior_events: list[dict[str, Any]] = []
+    response_text: list[str] = []
+    stream_errors: list[dict[str, Any]] = []
+    error = None
+    started = time.time()
+
+    def _form_data(message: str, current_case: dict[str, Any]) -> dict[str, str]:
+        active_email = current_case.get("active_email") or {}
+        form_data = {
+            "message": message,
+            "session": session_id,
+            "mode": "agent",
+            "agent_prompt_mode": "auto",
+            "selected_endpoint_id": args.endpoint_id,
+            "selected_model": args.model,
+            "allow_web_search": (
+                "true"
+                if (
+                    current_case.get("kind") == "web"
+                    or current_case.get("allow_web_search") is True
+                    or current_case.get("expect_first_tool") == "web_search"
+                    or "web_search" in current_case.get("expect_first_tool_any", [])
+                )
+                else ""
+            ),
+            "client_runtime_context": json.dumps(
+                {"timezone": args.timezone, "tz_offset_min": args.tz_offset_min},
+                ensure_ascii=True,
+            ),
+        }
+        if active_email:
+            form_data.update({
+                "active_email_uid": str(active_email.get("uid") or ""),
+                "active_email_folder": str(active_email.get("folder") or "INBOX"),
+                "active_email_account": str(active_email.get("account") or ""),
+            })
+        return form_data
+
+    def _submit(message: str, current_case: dict[str, Any]) -> tuple[list[dict[str, Any]], list[str], list[dict[str, Any]]]:
+        turn_events: list[dict[str, Any]] = []
+        turn_text: list[str] = []
+        turn_stream_errors: list[dict[str, Any]] = []
+        with client.stream(
+            "POST",
+            args.base_url.rstrip("/") + "/api/chat_stream",
+            data=_form_data(message, current_case),
+            headers={
+                "Accept": "text/event-stream",
+                "X-Tz-Name": args.timezone,
+                "X-Tz-Offset": str(args.tz_offset_min),
+            },
+            timeout=args.timeout,
+        ) as response:
+            _raise_for_status_with_body(response)
+            for event in _sse_events(response):
+                turn_events.append(event)
+                if event.get("type") in {"error", "parse_error"}:
+                    turn_stream_errors.append(event)
+                text = _visible_event_text(event)
+                if text:
+                    if event.get("type") == "final_response":
+                        turn_text[:] = [text]
+                    else:
+                        turn_text.append(text)
+        return turn_events, turn_text, turn_stream_errors
+
+    try:
+        for prior in case.get("prior_turns", []):
+            if isinstance(prior, str):
+                prior_case = {"kind": "", "allow_web_search": False}
+                prior_message = prior
+            else:
+                prior_case = prior
+                prior_message = str(prior.get("user") or "")
+            if not prior_message:
+                continue
+            prior_turn_events, _, prior_turn_errors = _submit(prior_message, prior_case)
+            prior_events.extend(prior_turn_events)
+            stream_errors.extend(prior_turn_errors)
+        events, response_text, final_errors = _submit(case["user"], case)
+        stream_errors.extend(final_errors)
+    except Exception as exc:
+        error = repr(exc)
+
+    calls = _tool_calls(events)
+    cleanup = _collect_and_cleanup(case, args, seeded, client)
+    with contextlib.suppress(Exception):
+        client.delete(args.base_url.rstrip("/") + f"/api/session/{session_id}", timeout=15)
+    final_answer = "".join(response_text).strip()
+    result = {
+        "id": case["id"],
+        "kind": case.get("kind", ""),
+        "user": case["user"],
+        "marker": case.get("marker", ""),
+        "first_tool": calls[0]["tool"] if calls else None,
+        "first_tool_args": calls[0]["args"] if calls else None,
+        "tool_names": [call["tool"] for call in calls],
+        "tool_calls": calls,
+        "tool_outputs": _tool_outputs(events),
+        "final_answer": final_answer,
+        "answer": final_answer,
+        "precreated_event_uid": seeded.get("precreated_event_uid", ""),
+        "active_document_before": seeded.get("active_document_before", ""),
+        "active_document_after": cleanup["active_document_after"],
+        "state": cleanup["state"],
+        "stream_errors": stream_errors,
+        "prior_events": prior_events,
+        "events": events,
+        "elapsed_seconds": round(time.time() - started, 3),
+    }
+    passed, failures = score_case(case, result)
+    if error:
+        failures.append(f"exception: {error}")
+        passed = False
+    if stream_errors:
+        failures.append(f"stream errors: {len(stream_errors)}")
+        passed = False
+    result["pass"] = passed
+    result["failures"] = failures
+    return result
+
+
+def _output_paths(args: argparse.Namespace) -> tuple[Path, Path | None]:
+    if args.out_dir:
+        out_dir = Path(args.out_dir)
+        return out_dir / "actual_results.json", out_dir / "actual_results.md"
+    output = Path(args.output)
+    md = output.with_suffix(".md") if args.write_md else None
+    return output, md
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", required=True)
+    parser.add_argument("--endpoint", required=True)
+    parser.add_argument("--endpoint-id", required=True)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--cookie-file", default="data/sessions.json")
+    parser.add_argument("--output", default="data/evals/ody_app_route_smoke_results.json")
+    parser.add_argument("--out-dir", default="")
+    parser.add_argument("--cases-file", default="")
+    parser.add_argument("--email-fixture", action="store_true")
+    parser.add_argument("--owner", default=DEFAULT_OWNER)
+    parser.add_argument("--timezone", default=DEFAULT_TZ)
+    parser.add_argument("--tz-offset-min", type=int, default=DEFAULT_TZ_OFFSET_MIN)
+    parser.add_argument("--timeout", type=float, default=180)
+    parser.add_argument("--write-md", action="store_true")
+    args = parser.parse_args()
+    cases = load_cases(Path(args.cases_file)) if args.cases_file else _default_cases()
+    client = httpx.Client(
+        cookies={"odysseus_session": _cookie(Path(args.cookie_file), args.owner)},
+        follow_redirects=False,
+    )
+    try:
+        with email_fixture(args.email_fixture, owner=args.owner):
+            results = [_run_turn(client, args, case) for case in cases]
+    finally:
+        client.close()
+    summary = {
+        "total": len(results),
+        "passed": sum(1 for result in results if result["pass"]),
+    }
+    summary["failed"] = summary["total"] - summary["passed"]
+    payload = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "base_url": args.base_url,
+        "endpoint": args.endpoint,
+        "endpoint_id": args.endpoint_id,
+        "model": args.model,
+        "owner": args.owner,
+        "timezone": args.timezone,
+        "tz_offset_min": args.tz_offset_min,
+        "summary": summary,
+        "cases": cases,
+        "results": results,
+    }
+    json_path, md_path = _output_paths(args)
+    json_path.parent.mkdir(parents=True, exist_ok=True)
+    json_path.write_text(json.dumps(payload, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
+    if md_path is not None:
+        md_path.parent.mkdir(parents=True, exist_ok=True)
+        write_markdown(md_path, payload)
+    print(json.dumps({"summary": summary, "json": str(json_path), "md": str(md_path) if md_path else ""}, indent=2))
+    return 0 if summary["failed"] == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/eval_odysseus_contextual_tool_use.py

@@ -0,0 +1,944 @@
+#!/usr/bin/env python3
+"""Multi-turn Odysseus tool-use eval for contextual follow-up behavior."""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+import os
+import re
+import time
+import uuid
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+try:
+    from scripts.eval_odysseus_tool_use import (
+        _cookie,
+        _raise_for_status_with_body,
+        _sse_events,
+        _tool_approval_from_event,
+        _visible_event_text,
+    )
+except ModuleNotFoundError:
+    from eval_odysseus_tool_use import (
+        _cookie,
+        _raise_for_status_with_body,
+        _sse_events,
+        _tool_approval_from_event,
+        _visible_event_text,
+    )
+
+
+SCENARIOS: list[dict[str, Any]] = [
+    {
+        "scenario": "public_domain_art_links_followup",
+        "turns": [
+            {
+                "message": "What are some good sites for public domain art?",
+                "expected_tool": "no_tool",
+                "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"],
+            },
+            {
+                "message": "send links",
+                "expected_tool": "web_search",
+                "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"],
+            },
+        ],
+    },
+    {
+        "scenario": "notes_then_identity_boundary",
+        "turns": [
+            {
+                "message": "what are my notes?",
+                "expected_tool": "manage_notes",
+                "expected_action": "list",
+                "required_any": ["note", "[", "test scenario"],
+            },
+            {
+                "message": "who are you?",
+                "expected_tool": "no_tool",
+                "required_any": ["odysseus", "assistant"],
+            },
+        ],
+    },
+    {
+        "scenario": "email_followup_search",
+        "turns": [
+            {
+                "message": "what is my latest email?",
+                "expected_tool": "mcp__email__list_emails",
+                "required_any": ["email", "from", "subject", "latest"],
+            },
+            {
+                "message": "find emails from Runpod instead",
+                "expected_tool": "mcp__email__search_emails",
+                "required_any": ["runpod", "email", "no emails"],
+            },
+        ],
+    },
+    {
+        "scenario": "calendar_then_general_fact",
+        "turns": [
+            {
+                "message": "what is on my calendar?",
+                "expected_tool": "manage_calendar",
+                "expected_action": "list_events",
+                "required_any": ["event", "calendar", "found", "no events"],
+            },
+            {
+                "message": "what does VAT stand for?",
+                "expected_tool": "no_tool",
+                "required_any": ["value-added tax", "value added tax"],
+            },
+        ],
+    },
+    {
+        "scenario": "public_domain_art_typo_links_followup",
+        "turns": [
+            {
+                "message": "What are some good sites for public domain art?",
+                "expected_tool": "no_tool",
+                "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"],
+            },
+            {
+                "message": "sned links for those",
+                "expected_tool": "web_search",
+                "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"],
+            },
+        ],
+    },
+    {
+        "scenario": "notes_then_calendar_switch",
+        "turns": [
+            {
+                "message": "what are my notes?",
+                "expected_tool": "manage_notes",
+                "expected_action": "list",
+                "required_any": ["note", "[", "test scenario"],
+            },
+            {
+                "message": "what is on my calendar next week?",
+                "expected_tool": "manage_calendar",
+                "expected_action": "list_events",
+                "required_any": ["event", "calendar", "found", "no events"],
+            },
+        ],
+    },
+    {
+        "scenario": "email_then_ambiguous_links_clarify",
+        "turns": [
+            {
+                "message": "what is my latest email?",
+                "expected_tool": "mcp__email__list_emails",
+                "required_any": ["email", "from", "subject", "latest"],
+            },
+            {
+                "message": "send links",
+                "expected_tool": "no_tool",
+                "required_any": ["which links", "what links", "clarify", "what topic", "which topic"],
+            },
+        ],
+    },
+    {
+        "scenario": "web_answer_then_calendar_boundary",
+        "turns": [
+            {
+                "message": "What are some good sites for public domain art?",
+                "expected_tool": "no_tool",
+                "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"],
+            },
+            {
+                "message": "what is on my calendar?",
+                "expected_tool": "manage_calendar",
+                "expected_action": "list_events",
+                "required_any": ["event", "calendar", "found", "no events"],
+            },
+        ],
+    },
+    {
+        "scenario": "public_domain_art_sites_tail_followup",
+        "turns": [
+            {
+                "message": "What are some good sites for public domain art?",
+                "expected_tool": "no_tool",
+                "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"],
+            },
+            {
+                "message": "send links for the sites",
+                "expected_tool": "web_search",
+                "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"],
+            },
+        ],
+    },
+    {
+        "scenario": "public_domain_art_typo_bare_links_followup",
+        "turns": [
+            {
+                "message": "What are some good sites for public domain art?",
+                "expected_tool": "no_tool",
+                "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"],
+            },
+            {
+                "message": "sned links",
+                "expected_tool": "web_search",
+                "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"],
+            },
+        ],
+    },
+    {
+        "scenario": "public_domain_art_bare_websites_followup",
+        "turns": [
+            {
+                "message": "What are some good sites for public domain art?",
+                "expected_tool": "no_tool",
+                "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"],
+            },
+            {
+                "message": "for the websites",
+                "expected_tool": "web_search",
+                "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"],
+            },
+        ],
+    },
+    {
+        "scenario": "email_then_typo_links_tail_clarify",
+        "turns": [
+            {
+                "message": "what is my latest email?",
+                "expected_tool": "mcp__email__list_emails",
+                "required_any": ["email", "from", "subject", "latest"],
+            },
+            {
+                "message": "sned links for those",
+                "expected_tool": "no_tool",
+                "required_any": ["which links", "what links", "clarify", "what topic", "which topic", "topic"],
+            },
+        ],
+    },
+    {
+        "scenario": "email_then_bare_websites_clarify",
+        "turns": [
+            {
+                "message": "what is my latest email?",
+                "expected_tool": "mcp__email__list_emails",
+                "required_any": ["email", "from", "subject", "latest"],
+            },
+            {
+                "message": "for the websites",
+                "expected_tool": "no_tool",
+                "required_any": ["which links", "what links", "clarify", "what topic", "which topic", "topic", "website"],
+            },
+        ],
+    },
+    {
+        "scenario": "notes_then_typo_links_tail_clarify",
+        "turns": [
+            {
+                "message": "what are my notes?",
+                "expected_tool": "manage_notes",
+                "expected_action": "list",
+                "required_any": ["note", "[", "test scenario"],
+            },
+            {
+                "message": "sned links for those",
+                "expected_tool": "no_tool",
+                "required_any": ["which links", "what links", "clarify", "what topic", "which topic", "topic"],
+            },
+        ],
+    },
+    {
+        "scenario": "notes_crud_followthrough",
+        "fixture_prefix": "ODY-EVAL-CRUD-NOTES-",
+        "turns": [
+            {
+                "message": "Create a note titled ODY-EVAL-CRUD-NOTES-FLOW with content alpha checkpoint.",
+                "expected_tool": "manage_notes",
+                "expected_actions": ["add", "create"],
+                "required_all": ["created", "ody-eval-crud-notes-flow"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Update that note so its content says beta checkpoint.",
+                "expected_tool": "manage_notes",
+                "expected_action": "update",
+                "required_all": ["updated", "note"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Delete that note.",
+                "expected_tool": "manage_notes",
+                "expected_action": "delete",
+                "required_all": ["deleted", "note"],
+                "max_tool_count": 1,
+            },
+        ],
+    },
+    {
+        "scenario": "calendar_crud_followthrough",
+        "fixture_prefix": "ODY-EVAL-CRUD-CALENDAR-",
+        "turns": [
+            {
+                "message": (
+                    "Create a calendar event titled ODY-EVAL-CRUD-CALENDAR-FLOW "
+                    "on 2026-08-25 from 10:00 to 10:30 at Test Lab."
+                ),
+                "expected_tool": "manage_calendar",
+                "expected_actions": ["create_event", "create"],
+                "required_all": ["created", "event", "ody-eval-crud-calendar-flow"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Update that calendar event location to Blue Room.",
+                "expected_tool": "manage_calendar",
+                "expected_actions": ["update_event", "update"],
+                "required_all": ["updated", "event"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Delete that calendar event.",
+                "expected_tool": "manage_calendar",
+                "expected_actions": ["delete_event", "delete"],
+                "required_all": ["deleted", "event"],
+                "max_tool_count": 1,
+            },
+        ],
+    },
+    {
+        "scenario": "memory_crud_followthrough",
+        "fixture_prefix": "ODY-EVAL-CRUD-MEMORY-",
+        "turns": [
+            {
+                "message": "Remember this temporary eval fact: ODY-EVAL-CRUD-MEMORY-FLOW alpha checkpoint.",
+                "expected_tool": "manage_memory",
+                "expected_action": "add",
+                "required_all": ["memory", "added"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Update that memory to say ODY-EVAL-CRUD-MEMORY-FLOW beta checkpoint.",
+                "expected_tool": "manage_memory",
+                "expected_action": "edit",
+                "required_all": ["memory", "updated"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Delete that memory.",
+                "expected_tool": "manage_memory",
+                "expected_action": "delete",
+                "required_all": ["memory", "deleted"],
+                "max_tool_count": 1,
+            },
+        ],
+    },
+    {
+        "scenario": "memory_add_one_call_efficiency",
+        "fixture_prefix": "ODY-EVAL-CRUD-MEMORY-",
+        "turns": [
+            {
+                "message": "Remember this temporary eval fact: ODY-EVAL-CRUD-MEMORY-ONECALL alpha checkpoint.",
+                "expected_tool": "manage_memory",
+                "expected_action": "add",
+                "required_all": ["memory", "added"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Delete that memory.",
+                "expected_tool": "manage_memory",
+                "expected_action": "delete",
+                "required_all": ["memory", "deleted"],
+                "max_tool_count": 1,
+            },
+        ],
+    },
+    {
+        "scenario": "memory_add_wording_variants_efficiency",
+        "fixture_prefix": "ODY-EVAL-CRUD-MEMORY-",
+        "turns": [
+            {
+                "message": "Save this as a memory: ODY-EVAL-CRUD-MEMORY-VAR-A alpha checkpoint.",
+                "expected_tool": "manage_memory",
+                "expected_action": "add",
+                "required_all": ["memory", "added"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Delete that memory.",
+                "expected_tool": "manage_memory",
+                "expected_action": "delete",
+                "required_all": ["memory", "deleted"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Add to memory that ODY-EVAL-CRUD-MEMORY-VAR-B beta checkpoint is temporary.",
+                "expected_tool": "manage_memory",
+                "expected_action": "add",
+                "required_all": ["memory", "added"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Remove that memory.",
+                "expected_tool": "manage_memory",
+                "expected_action": "delete",
+                "required_all": ["memory", "deleted"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Please remember: ODY-EVAL-CRUD-MEMORY-VAR-C gamma checkpoint.",
+                "expected_tool": "manage_memory",
+                "expected_action": "add",
+                "required_all": ["memory", "added"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Forget that memory.",
+                "expected_tool": "manage_memory",
+                "expected_action": "delete",
+                "required_all": ["memory", "deleted"],
+                "max_tool_count": 1,
+            },
+        ],
+    },
+    {
+        "scenario": "memory_no_tool_boundary",
+        "turns": [
+            {
+                "message": "do you remember what VAT stands for?",
+                "expected_tool": "no_tool",
+                "required_any": ["value-added tax", "value added tax"],
+            },
+            {
+                "message": "what should I remember before buying public domain art?",
+                "expected_tool": "no_tool",
+                "required_any": ["license", "copyright", "public domain", "source"],
+            },
+            {
+                "message": "remind me what Sweden is bordered by",
+                "expected_tool": "no_tool",
+                "required_any": ["norway", "finland"],
+            },
+            {
+                "message": "what does it mean to remember something in a computer?",
+                "expected_tool": "no_tool",
+                "required_any": ["store", "storage", "memory", "data", "information"],
+            },
+        ],
+    },
+    {
+        "scenario": "tasks_crud_followthrough",
+        "fixture_prefix": "ODY-EVAL-CRUD-TASKS-",
+        "turns": [
+            {
+                "message": (
+                    "Create a scheduled task named ODY-EVAL-CRUD-TASKS-FLOW that runs daily at 09:00 UTC "
+                    "and has prompt alpha checkpoint."
+                ),
+                "expected_tool": "manage_tasks",
+                "expected_action": "create",
+                "required_all": ["created", "task", "ody-eval-crud-tasks-flow"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Update that task prompt to beta checkpoint.",
+                "expected_tool": "manage_tasks",
+                "expected_action": "edit",
+                "required_all": ["updated", "task"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Delete that task.",
+                "expected_tool": "manage_tasks",
+                "expected_action": "delete",
+                "required_all": ["deleted", "task"],
+                "max_tool_count": 1,
+            },
+        ],
+    },
+    {
+        "scenario": "documents_create_delete_followthrough",
+        "fixture_prefix": "ODY-EVAL-CRUD-DOCUMENTS-",
+        "turns": [
+            {
+                "message": (
+                    "Create an editor document titled ODY-EVAL-CRUD-DOCUMENTS-FLOW "
+                    "with markdown content alpha checkpoint."
+                ),
+                "expected_tool": "create_document",
+                "required_all": ["document", "ody-eval-crud-documents-flow"],
+                "max_tool_count": 1,
+            },
+            {
+                "message": "Delete that document.",
+                "expected_tool": "manage_documents",
+                "expected_action": "delete",
+                "required_all": ["deleted", "document"],
+                "max_tool_count": 1,
+            },
+        ],
+    },
+]
+
+
+TOOL_ALIASES = {
+    "mcp_email_list_emails": "mcp__email__list_emails",
+    "mcp_email_search_emails": "mcp__email__search_emails",
+    "list_emails": "mcp__email__list_emails",
+    "search_emails": "mcp__email__search_emails",
+}
+
+
+def malformed_text_surface(response_text: str) -> bool:
+    value = (response_text or "").lower()
+    if any(
+        marker in value
+        for marker in (
+            "<function",
+            "<parameter",
+            "function=",
+            "parameter=",
+            "</parameter",
+            '"function"',
+            "tool_call",
+            "|end|",
+            "web search results and fetched content",
+            "search results summary:",
+        )
+    ):
+        return True
+    return any(
+        re.search(pattern, response_text or "")
+        for pattern in (
+            r"\bIamOdysseus\b",
+            r"\bSwedenisin\b",
+            r"\bVATstands\b",
+            r"\bpublic\s+domain\s+ar\b(?!t)",
+            r"\bthe\s+me\s+open\s+access\b",
+            r"\bar\s+institute\b(?!t)",
+            r"\bpublicdomainar\b",
+            r"\bTheMeOpenAccess\b",
+            r"\bCanyouclarifywhich\b",
+            r"\bwan me\s+link\b",
+            r"\bwhat you wan me\b",
+        )
+    )
+
+
+def canonical_tool(tool: str | None) -> str | None:
+    if not tool:
+        return tool
+    return TOOL_ALIASES.get(tool, tool)
+
+
+def parse_action(command: str | None) -> str:
+    if not command:
+        return ""
+    try:
+        parsed = json.loads(command)
+    except json.JSONDecodeError:
+        parsed = command
+    if isinstance(parsed, dict):
+        return str(parsed.get("action") or "")
+    if isinstance(parsed, str):
+        return parsed.strip().splitlines()[0] if parsed.strip() else ""
+    return ""
+
+
+def _fixture_owner() -> str:
+    return os.environ.get("ODY_EVAL_OWNER", "pewds")
+
+
+def _cleanup_crud_fixtures() -> None:
+    """Remove only eval-owned CRUD artifacts created by this script."""
+    owner = _fixture_owner()
+    try:
+        from core.database import (
+            CalendarCal,
+            CalendarEvent,
+            Document,
+            DocumentVersion,
+            Note,
+            ScheduledTask,
+            SessionLocal,
+        )
+    except Exception as exc:
+        print(json.dumps({"cleanup_warning": f"database import failed: {exc!r}"}), flush=True)
+    else:
+        db = SessionLocal()
+        try:
+            notes_q = db.query(Note).filter(Note.title.like("ODY-EVAL-CRUD-%"))
+            if owner:
+                notes_q = notes_q.filter(Note.owner == owner)
+            for note in notes_q.all():
+                db.delete(note)
+
+            events_q = db.query(CalendarEvent).filter(CalendarEvent.summary.like("ODY-EVAL-CRUD-%"))
+            if owner:
+                events_q = events_q.join(CalendarCal, CalendarEvent.calendar_id == CalendarCal.id).filter(
+                    CalendarCal.owner == owner
+                )
+            for event in events_q.all():
+                db.delete(event)
+
+            cals_q = db.query(CalendarCal).filter(CalendarCal.name.like("ODY-EVAL-CRUD-%"))
+            if owner:
+                cals_q = cals_q.filter(CalendarCal.owner == owner)
+            for calendar in cals_q.all():
+                db.delete(calendar)
+
+            docs_q = db.query(Document).filter(Document.title.like("ODY-EVAL-CRUD-%"))
+            if owner:
+                docs_q = docs_q.filter(Document.owner == owner)
+            for doc in docs_q.all():
+                db.query(DocumentVersion).filter(DocumentVersion.document_id == doc.id).delete()
+                db.delete(doc)
+
+            tasks_q = db.query(ScheduledTask).filter(ScheduledTask.name.like("ODY-EVAL-CRUD-%"))
+            if owner:
+                tasks_q = tasks_q.filter(ScheduledTask.owner == owner)
+            tasks_q.delete(synchronize_session=False)
+            db.commit()
+        except Exception as exc:
+            db.rollback()
+            print(json.dumps({"cleanup_warning": repr(exc)}), flush=True)
+        finally:
+            db.close()
+
+    try:
+        from src.constants import MEMORY_FILE
+        memory_path = Path(MEMORY_FILE)
+        if memory_path.exists():
+            entries = json.loads(memory_path.read_text(encoding="utf-8"))
+            if isinstance(entries, list):
+                filtered = [
+                    entry
+                    for entry in entries
+                    if not (
+                        isinstance(entry, dict)
+                        and "ODY-EVAL-CRUD-MEMORY-" in str(entry.get("text") or "")
+                        and (not owner or entry.get("owner") == owner)
+                    )
+                ]
+                if len(filtered) != len(entries):
+                    memory_path.write_text(json.dumps(filtered, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
+    except Exception as exc:
+        print(json.dumps({"cleanup_warning": f"memory cleanup failed: {exc!r}"}), flush=True)
+
+
+@contextlib.contextmanager
+def _crud_fixture_cleanup(enabled: bool):
+    if enabled:
+        _cleanup_crud_fixtures()
+    try:
+        yield
+    finally:
+        if enabled:
+            _cleanup_crud_fixtures()
+
+
+def output_ok(event: dict[str, Any]) -> bool:
+    if event.get("exit_code") not in (0, None):
+        return False
+    text = str(event.get("output") or "")
+    return not text.lstrip().lower().startswith("error")
+
+
+def event_action(event: dict[str, Any]) -> str:
+    return parse_action(str(event.get("command") or ""))
+
+
+def create_session(client: httpx.Client, args, name: str) -> str:
+    response = client.post(
+        args.base_url.rstrip("/") + "/api/session",
+        data={
+            "name": name,
+            "endpoint_url": args.selected_endpoint_url or args.endpoint,
+            "model": args.selected_model or args.model,
+            "skip_validation": "true",
+            "rag": "false",
+            **({"endpoint_id": args.endpoint_id} if args.endpoint_id else {}),
+        },
+        timeout=30,
+    )
+    _raise_for_status_with_body(response)
+    return response.json()["id"]
+
+
+def run_turn(client: httpx.Client, args, session_id: str, spec: dict[str, Any]) -> dict[str, Any]:
+    started = time.monotonic()
+    events: list[dict[str, Any]] = []
+    text: list[str] = []
+    errors: list[dict[str, Any]] = []
+    approval_turns = 0
+    turn_data = {
+        "message": spec["message"],
+        "session": session_id,
+        "mode": "agent",
+        "agent_prompt_mode": args.prompt_mode,
+        **({"selected_endpoint_id": args.endpoint_id} if args.endpoint_id else {}),
+        **({"selected_endpoint_url": args.selected_endpoint_url} if args.selected_endpoint_url else {}),
+        **({"selected_model": args.selected_model} if args.selected_model else {}),
+    }
+    try:
+        while True:
+            approval = None
+            with client.stream(
+                "POST",
+                args.base_url.rstrip("/") + "/api/chat_stream",
+                data=turn_data,
+                headers={"Accept": "text/event-stream"},
+                timeout=args.timeout,
+            ) as response:
+                _raise_for_status_with_body(response)
+                for event in _sse_events(response):
+                    events.append(event)
+                    if event.get("type") == "error":
+                        errors.append(event)
+                    visible = _visible_event_text(event)
+                    if visible:
+                        if event.get("type") == "final_response":
+                            text[:] = [visible]
+                        else:
+                            text.append(visible)
+                    approval = approval or _tool_approval_from_event(event)
+            if not args.auto_approve or not approval or approval_turns >= 3:
+                break
+            approval_turns += 1
+            turn_data = {
+                **turn_data,
+                "tool_approval_id": approval["approval_id"],
+                "tool_approval_decision": "approve",
+            }
+    except Exception as exc:
+        errors.append({"type": "client_exception", "error": repr(exc)})
+
+    starts = [event for event in events if event.get("type") == "tool_start"]
+    outputs = [event for event in events if event.get("type") == "tool_output"]
+    metrics = [
+        event.get("data")
+        for event in events
+        if event.get("type") == "metrics" and isinstance(event.get("data"), dict)
+    ]
+    snapshots = [
+        {
+            key: event.get(key)
+            for key in (
+                "round",
+                "model",
+                "messages",
+                "tools",
+                "temperature",
+                "max_tokens",
+                "agent_prompt_mode",
+            )
+        }
+        for event in events
+        if event.get("type") == "model_request_snapshot"
+    ]
+    metric_tool_events = [
+        tool_event
+        for metric in metrics
+        for tool_event in (metric.get("tool_events") or [])
+        if isinstance(tool_event, dict)
+    ]
+    summarized_tool_events = [
+        {
+            "tool": canonical_tool(str(event.get("tool") or "")),
+            "command": str(event.get("command") or ""),
+            "exit_code": event.get("exit_code"),
+            "output_preview": str(event.get("output") or "")[:500],
+        }
+        for event in [*outputs, *metric_tool_events]
+        if isinstance(event, dict)
+    ]
+    observed_events = metric_tool_events or outputs or starts
+    first = observed_events[0] if observed_events else {}
+    first_tool = canonical_tool(first.get("tool"))
+    first_action = parse_action(first.get("command"))
+    response_text = "".join(text).strip()
+    if not response_text and metrics:
+        round_texts = metrics[-1].get("round_texts") or []
+        response_text = next((str(item).strip() for item in reversed(round_texts) if str(item).strip()), "")
+
+    expected_tool = spec["expected_tool"]
+    expected_action = spec.get("expected_action") or ""
+    expected_actions = [str(item) for item in (spec.get("expected_actions") or [])]
+    max_tool_count = spec.get("max_tool_count")
+    if expected_action and not expected_actions:
+        expected_actions = [expected_action]
+    if expected_tool == "no_tool":
+        tool_ok = not observed_events
+        execution_ok = bool(response_text) and not errors
+    else:
+        tool_ok = first_tool == expected_tool
+        executed = [
+            event
+            for event in [*outputs, *metric_tool_events]
+            if canonical_tool(str(event.get("tool") or "")) == expected_tool
+            and (not expected_actions or event_action(event) in expected_actions)
+            and output_ok(event)
+        ]
+        execution_ok = bool(executed) and not errors
+    action_ok = not expected_actions or first_action in expected_actions
+    lower_response = response_text.lower()
+    required_any = [str(item).lower() for item in spec.get("required_any") or []]
+    required_all = [str(item).lower() for item in spec.get("required_all") or []]
+    response_quality_ok = bool(response_text) and (
+        not required_any or any(item in lower_response for item in required_any)
+    ) and all(item in lower_response for item in required_all)
+    if malformed_text_surface(response_text):
+        response_quality_ok = False
+    tool_efficiency_ok = True
+    if isinstance(max_tool_count, int):
+        tool_efficiency_ok = len(observed_events) <= max_tool_count
+
+    latest_metrics = metrics[-1] if metrics else {}
+    usage_buckets = latest_metrics.get("usage_buckets") if isinstance(latest_metrics, dict) else None
+    return {
+        "message": spec["message"],
+        "expected_tool": expected_tool,
+        "expected_action": expected_action,
+        "expected_actions": expected_actions,
+        "first_tool": first_tool,
+        "first_action": first_action,
+        "tool_count": len(observed_events),
+        "tool_ok": bool(tool_ok),
+        "action_ok": bool(action_ok),
+        "execution_ok": bool(execution_ok),
+        "response_quality_ok": bool(response_quality_ok),
+        "tool_efficiency_ok": bool(tool_efficiency_ok),
+        "max_tool_count": max_tool_count,
+        "stream_errors": errors,
+        "response": response_text[:2000],
+        "input_tokens": latest_metrics.get("input_tokens"),
+        "output_tokens": latest_metrics.get("output_tokens"),
+        "tokens_per_second": latest_metrics.get("tokens_per_second"),
+        "request_context_tokens": latest_metrics.get("request_context_tokens"),
+        "usage_buckets": usage_buckets if isinstance(usage_buckets, list) else [],
+        "tool_events": summarized_tool_events,
+        "elapsed_seconds": round(time.monotonic() - started, 3),
+        "approval_turns": approval_turns,
+        "model_request_snapshots": snapshots,
+    }
+
+
+def write_output(path: Path, records: list[dict[str, Any]], model: str) -> None:
+    turns = [turn for record in records for turn in record["turns"]]
+    summary = {
+        "model": model,
+        "scenarios": len(records),
+        "turns": len(turns),
+        "tool_success": sum(turn["tool_ok"] for turn in turns),
+        "action_success": sum(turn["action_ok"] for turn in turns),
+        "execution_success": sum(turn["execution_ok"] for turn in turns),
+        "response_quality_success": sum(turn["response_quality_ok"] for turn in turns),
+        "tool_efficiency_success": sum(turn.get("tool_efficiency_ok", True) for turn in turns),
+        "stream_errors": sum(bool(turn["stream_errors"]) for turn in turns),
+        "records": records,
+    }
+    tmp = path.with_name(path.name + ".tmp")
+    tmp.write_text(json.dumps(summary, indent=2, ensure_ascii=True) + "\n")
+    tmp.replace(path)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", default="http://127.0.0.1:7011")
+    parser.add_argument("--endpoint", default="http://host.docker.internal:18052/v1")
+    parser.add_argument("--endpoint-id", default="v8c1000")
+    parser.add_argument("--model", default="qwen35-9b-tool-router-v15-regular-chat-boundary-final")
+    parser.add_argument("--selected-endpoint-url", default="")
+    parser.add_argument("--selected-model", default="")
+    parser.add_argument("--cookie-file", default="data/sessions.json")
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--prompt-mode", default="compact")
+    parser.add_argument("--timeout", type=float, default=180.0)
+    parser.add_argument("--cases", default="")
+    parser.add_argument("--no-auto-approve", dest="auto_approve", action="store_false")
+    parser.add_argument("--keep-sessions", action="store_true")
+    args = parser.parse_args()
+
+    selected = {item.strip() for item in args.cases.split(",") if item.strip()}
+    scenarios = [case for case in SCENARIOS if not selected or case["scenario"] in selected]
+    unknown = selected - {case["scenario"] for case in SCENARIOS}
+    if unknown:
+        raise SystemExit(f"Unknown scenario(s): {', '.join(sorted(unknown))}")
+
+    output = Path(args.output)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    client = httpx.Client(
+        cookies={"odysseus_session": _cookie(Path(args.cookie_file))},
+        follow_redirects=False,
+    )
+    records: list[dict[str, Any]] = []
+    try:
+        needs_crud_cleanup = any(str(case.get("fixture_prefix") or "").startswith("ODY-EVAL-CRUD-") for case in scenarios)
+        with _crud_fixture_cleanup(needs_crud_cleanup):
+            for scenario in scenarios:
+                session_id = create_session(
+                    client,
+                    args,
+                    "[eval-context] "
+                    + scenario["scenario"]
+                    + " "
+                    + time.strftime("%Y%m%d-%H%M%S")
+                    + "-"
+                    + uuid.uuid4().hex[:6],
+                )
+                turns = []
+                try:
+                    for spec in scenario["turns"]:
+                        turn = run_turn(client, args, session_id, spec)
+                        turns.append(turn)
+                        print(
+                            json.dumps(
+                                {
+                                    "scenario": scenario["scenario"],
+                                    **{
+                                        key: turn.get(key)
+                                        for key in (
+                                            "message",
+                                            "expected_tool",
+                                            "first_tool",
+                                            "expected_action",
+                                            "expected_actions",
+                                            "first_action",
+                                            "tool_ok",
+                                            "action_ok",
+                                            "execution_ok",
+                                            "response_quality_ok",
+                                            "tool_efficiency_ok",
+                                            "max_tool_count",
+                                            "tool_count",
+                                            "input_tokens",
+                                            "output_tokens",
+                                            "elapsed_seconds",
+                                            "stream_errors",
+                                        )
+                                    },
+                                },
+                                ensure_ascii=True,
+                            ),
+                            flush=True,
+                        )
+                finally:
+                    if args.keep_sessions:
+                        print(json.dumps({"kept_session": session_id, "scenario": scenario["scenario"]}), flush=True)
+                    else:
+                        try:
+                            client.delete(args.base_url.rstrip("/") + f"/api/session/{session_id}", timeout=15)
+                        except Exception:
+                            pass
+                records.append({"scenario": scenario["scenario"], "turns": turns})
+                write_output(output, records, args.selected_model or args.model)
+    finally:
+        client.close()
+    write_output(output, records, args.selected_model or args.model)
+    summary = json.loads(output.read_text())
+    print("SUMMARY", json.dumps({k: v for k, v in summary.items() if k != "records"}))
+
+
+if __name__ == "__main__":
+    main()