odysseus/scripts/odysseus_remaining_tool_audit.py

#!/usr/bin/env python3
"""Audit the remaining Odysseus tools with isolated, resumable sessions.

This uses the same curation contract as ``odysseus_domain_audit.py`` but
creates one session per tool. Prompts prefer read-only behavior, but mutating
email prompts target synthetic SFT fixture accounts only so they can produce
real reviewable action traces.
"""

from __future__ import annotations

import argparse
import contextlib
import json
import os
import re
import sys
import time
import uuid
from pathlib import Path

import httpx

ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
    sys.path.insert(0, str(ROOT))

from scripts.odysseus_domain_audit import (  # noqa: E402
    Case,
    _create_session,
    _durable_tool_events,
    _event_text,
    _history_pairs,
    _render_prompt,
    _run_turn,
    _session_payload,
    score_case,
)
from scripts.odysseus_related_flow_audit import (  # noqa: E402
    _flow_has_good_training_shape,
    _latest_assistant_text,
    _login_cookie,
    _safe_metadata,
)


# These are intentionally excluded from this job because they already have
# dedicated 20-case coverage in the domain audit or the earlier email/search
# runs. Aliases are omitted; each canonical runtime tool is tested once.
REMAINING_TOOLS = (
    "bash", "python", "read_file", "write_file", "edit_file", "apply_patch",
    "grep", "glob", "ls", "get_workspace", "host_shell", "manage_bg_jobs",
    "manage_contact", "resolve_contact", "manage_session", "list_sessions",
    "search_chats", "web_fetch", "private_browser", "youtube_tool",
    "ask_user", "update_plan",
    "trigger_research", "manage_research", "chat_with_model", "ask_teacher",
    "pipeline", "list_models", "create_session", "send_to_session",
    "download_model", "serve_model", "serve_preset", "adopt_served_model",
    "stop_served_model", "tail_serve_output", "list_served_models",
    "list_downloads", "list_cached_models", "list_cookbook_servers",
    "list_serve_presets", "cancel_download",
    "manage_endpoints", "manage_mcp", "api_call", "app_api", "manage_settings",
    "manage_webhooks", "manage_tokens", "download_attachment", "scan_spam",
    "block_sender", "manage_email_state", "scan_email_unsubscribes",
    "unsubscribe_email", "draft_email", "draft_email_reply", "ai_draft_email_reply",
    "bulk_email",
)

# These are intentionally unavailable to ``sft_*`` owners under the current
# workspace-isolation policy. They are still listed in REMAINING_TOOLS so the
# matrix documents the full catalog, but are audited separately as policy
# checks rather than spending 20 live turns on guaranteed unavailable tools.
SFT_POLICY_DISABLED_TOOLS = frozenset({
    "python", "read_file", "write_file", "edit_file", "apply_patch",
    "grep", "glob", "ls", "get_workspace", "host_shell", "manage_bg_jobs",
})


def _tool_prompts(tool: str, marker: str) -> list[str]:
    """Return exactly 20 prompts for a canonical tool.

    The prompts exercise discovery, repeated use, and follow-up wording.
    """
    safe = {
        "bash": ["Run a read-only shell check for audit marker {marker}", "Show the current working directory without changing files"],
        "python": ["Compute 2 + 2 in Python", "Parse this audit marker as plain text: {marker}"],
        "read_file": ["Read the audit fixture file if it exists: /tmp/audit-{marker}.txt", "Read one harmless system metadata file such as /etc/hostname"],
        "write_file": ["Preview creating a fixture file named /tmp/audit-{marker}.txt; do not write it", "Explain the exact write_file arguments for a fixture file without executing"],
        "edit_file": ["Preview changing /tmp/audit-{marker}.txt; do not edit files", "Explain the exact edit_file arguments for a fixture replacement without executing"],
        "apply_patch": ["Preview a patch to a fixture file; do not apply it", "Explain how to apply a no-op patch to an audit fixture without executing"],
        "grep": ["Search the workspace for the audit marker {marker}", "Search source files for the word fixture"],
        "glob": ["Find markdown files in the workspace", "Find files matching **/*audit*"],
        "ls": ["List the workspace directory", "List the temporary audit directory"],
        "get_workspace": ["What is the active workspace folder?", "Show the workspace path used by this chat"],
        "host_shell": ["Preview a read-only host check for the audit marker; do not change anything", "Show the host operating system name without modifying it"],
        "manage_bg_jobs": ["List my background jobs", "Show whether any background jobs are running"],
        "manage_contact": ["Search my address book contacts for Priya Shah", "List my address-book contacts"],
        "resolve_contact": ["Find the email address for Casey Morgan", "Resolve Priya Shah in my contacts"],
        "manage_session": [
            "Rename this current audit chat to manage-session-audit-{marker}",
            "Archive this current audit chat",
            "Unarchive this current audit chat",
        ],
        "list_sessions": ["List my chats", "Show recent chat sessions"],
        "search_chats": ["Search past chats for audit marker {marker}", "Find previous chats mentioning calendar tools"],
        "web_fetch": ["Read the text of https://example.com", "Fetch https://www.rfc-editor.org/rfc/rfc9110"],
        "private_browser": ["Open https://example.com in the private browser and inspect its title", "Open https://www.w3.org and report the visible heading"],
        "youtube_tool": ["Find the metadata for YouTube video https://www.youtube.com/watch?v=dQw4w9WgXcQ", "Read the latest available metadata for that YouTube video"],
        "ask_user": [
            "Ask me which day next month my dinner reservation should be saved for; do not guess the date",
            "Ask me to choose whether to keep, archive, or delete a suspicious email; do not take action",
        ],
        "update_plan": [
            "Make a short plan for testing Odysseus SFT rows and write it to the plan panel",
            "Update the plan panel with inspect marked done and patch still pending",
        ],
        "trigger_research": ["Start a small research job about the history of SearXNG", "Research the difference between PDF and HTML extraction"],
        "manage_research": ["List my saved research reports", "Search saved research for SearXNG"],
        "chat_with_model": ["Ask another model for a one-sentence definition of SFT", "Compare another model's answer about tool calling"],
        "ask_teacher": ["Ask the teacher how to validate a tool trace", "Ask the teacher for one concise SFT quality check"],
        "pipeline": ["Describe a two-step analysis pipeline without running it", "Preview a pipeline that summarizes then checks a result"],
        "list_models": ["List available models", "Show the configured model endpoints"],
        "create_session": ["Preview creating a chat named audit-{marker}; do not create it", "Explain the arguments for a new chat without creating one"],
        "send_to_session": ["Preview sending a message to another chat; do not send it", "Explain how cross-chat messaging works without sending"],
        "download_model": ["Preview a download of Qwen/Qwen3-0.6B; do not start it", "Explain which server would receive a model download without starting one"],
        "serve_model": ["Preview serving a tiny local model; do not launch a server", "Explain the safe arguments for a model server dry run without launching it"],
        "serve_preset": ["Preview launching a saved serve preset; do not launch it", "List what a serve preset would do without starting it"],
        "adopt_served_model": ["Preview adopting an existing model server; do not change tracking", "Explain how an existing server would be adopted without registering it"],
        "stop_served_model": ["Preview stopping a model server; do not stop anything", "Explain how to identify a model server before stopping it"],
        "tail_serve_output": ["List model servers before reading any logs", "Explain how to inspect serve output without changing a server"],
        "list_served_models": ["List currently running Cookbook model servers", "Show what is serving in Cookbook right now"],
        "list_downloads": ["List active Cookbook downloads", "Show current model download progress"],
        "list_cached_models": ["List cached models on disk", "Show downloaded models already available locally"],
        "list_cookbook_servers": ["List configured Cookbook servers", "Show the current default Cookbook server"],
        "list_serve_presets": ["List saved Cookbook serve presets", "Show available serve presets without launching one"],
        "cancel_download": ["List downloads before considering cancellation; do not cancel anything", "Explain how to cancel a download without executing cancellation"],
        "manage_endpoints": ["List configured API endpoints", "Show enabled endpoints without changing them"],
        "manage_mcp": ["List configured MCP servers", "Show available MCP tools without changing configuration"],
        "api_call": ["Preview a harmless GET integration request without sending it", "Explain how to inspect a configured integration safely"],
        "app_api": ["List allowed internal API endpoints for cookbook state", "Preview reading a harmless internal status endpoint"],
        "manage_settings": ["Show available settings without changing them", "Read the current search setting without modifying it"],
        "manage_webhooks": ["List configured webhooks", "Show webhook status without changing anything"],
        "manage_tokens": ["List API tokens without creating or deleting one", "Explain token management without changing tokens"],
        "download_attachment": ["Open attachment 0 from email UID 112 and summarize it", "Read the creator payout sample attachment from email UID 112"],
        "scan_spam": ["Scan my inbox for likely spam without deleting or blocking anything", "Run a spam scan on recent inbox messages without taking action"],
        "block_sender": ["Block sender alerts@secure-rowan-login.co but do not delete existing messages", "Block sender notice@creator-awards.example.net and leave existing messages alone"],
        "manage_email_state": ["List blocked senders and reversible email state without changing it", "Show my blocked email senders without changing anything"],
        "scan_email_unsubscribes": ["Scan recent email headers for unsubscribe candidates", "Find newsletter unsubscribe candidates in my inbox"],
        "unsubscribe_email": ["Unsubscribe from email UID 162 using method 0", "Use unsubscribe method 0 for email UID 163", "Unsubscribe from UID 162 using method 0"],
        "draft_email": ["Create a reviewable draft email to Casey Morgan saying hello", "Draft an email to Priya Shah saying I will review the agenda", "Create a reviewable email draft to Marco Wells saying I saw the playbook"],
        "draft_email_reply": ["Create a reply draft for email UID 10 saying thanks for the next steps", "Draft a reply to UID 104 saying I received the invoice backup", "Create a reply draft to email UID 123 saying I saw the playbook"],
        "ai_draft_email_reply": ["Create an AI reply draft for email UID 10", "Use AI Reply to draft a response to email UID 104", "Create an AI reply draft for email UID 123"],
        "bulk_email": ["Mark emails UID 162 and UID 163 as read", "Mark UIDs 162 and 163 unread in one bulk action"],
    }
    variants = safe[tool]
    prompts = []
    fixture_mutating = {
        "unsubscribe_email",
        "draft_email",
        "draft_email_reply",
        "ai_draft_email_reply",
        "bulk_email",
        "block_sender",
    }
    for index in range(20):
        base = variants[index % len(variants)]
        if tool in fixture_mutating:
            qualifier = " Use the synthetic SFT fixture only and report the result."
        else:
            qualifier = (" Use the tool directly and report the result." if index % 2 == 0
                          else " Keep this read-only and concise.")
        prompts.append(base + qualifier)
    return prompts


EXPECTED_TOOL_ALIASES = {
    "draft_email_reply": ("draft_email_reply", "ui_control"),
    "ai_draft_email_reply": ("ai_draft_email_reply", "draft_email_reply", "ui_control"),
}


def tool_matrix() -> dict[str, list[Case]]:
    matrix = {}
    for tool in REMAINING_TOOLS:
        prompts = _tool_prompts(tool, "{marker}")
        expected_tools = EXPECTED_TOOL_ALIASES.get(tool, (tool,))
        matrix[tool] = [Case(f"{tool}_{i:02d}", prompt, expected_tools, "", False,
                       tool in {"download_model", "serve_model", "serve_preset", "adopt_served_model", "stop_served_model", "cancel_download", "bulk_email"})
                        for i, prompt in enumerate(prompts, 1)]
    return matrix


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--base-url", default="http://127.0.0.1:7011")
    parser.add_argument("--cookie", default=os.environ.get("ODY_COOKIE", ""))
    parser.add_argument("--username", default="sft_alex_creator")
    parser.add_argument("--password", default="SftDemo!2026")
    parser.add_argument("--endpoint-url", default="")
    parser.add_argument("--endpoint-id", default="")
    parser.add_argument("--model", default="")
    parser.add_argument("--owner", default="sft_alex_creator")
    parser.add_argument("--tools", default="all")
    parser.add_argument("--out-dir", type=Path, default=ROOT / "tmp" / "remaining-tool-audit")
    parser.add_argument("--timeout", type=float, default=180)
    parser.add_argument("--delete-bad", action="store_true")
    parser.add_argument("--limit", type=int, default=20)
    parser.add_argument("--include-policy-disabled", action="store_true",
                        help="Also run tools hidden from sft_* owners (expected to fail policy checks)")
    parser.add_argument(
        "--workspace",
        default="",
        help="Workspace/cwd to bind for workspace/file/shell tool cases.",
    )
    parser.add_argument(
        "--client-runtime-context",
        default="",
        help="Optional JSON object passed as client_runtime_context.",
    )
    args = parser.parse_args()
    if args.client_runtime_context:
        try:
            args.client_runtime_context = json.loads(args.client_runtime_context)
        except json.JSONDecodeError as exc:
            raise SystemExit(f"--client-runtime-context must be valid JSON: {exc}") from exc
        if not isinstance(args.client_runtime_context, dict):
            raise SystemExit("--client-runtime-context must decode to a JSON object")
    else:
        args.client_runtime_context = None
    cookie = args.cookie or _login_cookie(args.base_url, args.username, args.password)
    requested = list(REMAINING_TOOLS) if args.tools == "all" else [x.strip() for x in args.tools.split(",") if x.strip()]
    unknown = sorted(set(requested) - set(REMAINING_TOOLS))
    if unknown:
        parser.error(f"unknown tools: {', '.join(unknown)}")
    skipped_policy = []
    if not args.include_policy_disabled and str(args.owner).startswith("sft_"):
        skipped_policy = [tool for tool in requested if tool in SFT_POLICY_DISABLED_TOOLS]
        requested = [tool for tool in requested if tool not in SFT_POLICY_DISABLED_TOOLS]
    matrix = tool_matrix()
    args.out_dir.mkdir(parents=True, exist_ok=True)
    marker = f"{time.strftime('%Y%m%d_%H%M%S')}-{uuid.uuid4().hex[:8]}"
    rows = []
    with httpx.Client(cookies={"odysseus_session": cookie}, follow_redirects=True) as client:
        for tool in requested:
            sid = _create_session(client, args, f"tool-{tool}-{marker}")
            turns = []
            path = args.out_dir / f"{tool}_{sid}.json"
            for case in matrix[tool][:args.limit]:
                prompt = _render_prompt(case.prompt, marker)
                try:
                    events = _run_turn(client, args, sid, prompt)
                    durable = _session_payload(client, args.base_url, sid)
                    tool_events = _durable_tool_events(durable)
                    if tool_events:
                        events += [{"type": "metrics", "data": {"tool_events": tool_events}}]
                    durable_response = _latest_assistant_text(durable) or _event_text(events)
                    result = score_case(case, events, durable_response)
                    result["events"] = events
                except Exception as exc:
                    result = {"case_id": case.id, "prompt": prompt, "pass": False, "errors": [repr(exc)], "events": []}
                turns.append(result)
                print(f"{tool}: {case.id} {'PASS' if result.get('pass') else 'FAIL'}", flush=True)
                partial_history = {}
                with contextlib.suppress(Exception):
                    partial_history = _session_payload(client, args.base_url, sid)
                partial_payload = {
                    "tool": tool,
                    "marker": marker,
                    "session_id": sid,
                    "owner": args.owner,
                    "turns": turns,
                    "history": partial_history,
                    "partial": True,
                }
                path.write_text(json.dumps(partial_payload, ensure_ascii=False, indent=2), encoding="utf-8")
            history = _session_payload(client, args.base_url, sid)
            shape_ok, shape_reasons = _flow_has_good_training_shape(history, len(turns))
            passed = sum(bool(turn.get("pass")) for turn in turns)
            payload = {"tool": tool, "marker": marker, "session_id": sid, "owner": args.owner,
                       "turns": turns, "history": history, "passed": passed,
                       "shape_ok": shape_ok, "shape_reasons": shape_reasons,
                       "deterministic_pass": bool(turns) and passed == len(turns) and shape_ok,
                       "partial": False}
            path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
            verdict = "keep" if payload["deterministic_pass"] else "repair"
            payload["verdict"] = verdict
            if args.delete_bad and verdict != "keep":
                payload["deleted"] = client.delete(f"{args.base_url.rstrip('/')}/api/session/{sid}", timeout=30).is_success
                path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
            rows.append({"tool": tool, "session_id": sid, "passed": passed, "turns": len(turns),
                         "shape_ok": shape_ok, "shape_reasons": shape_reasons,
                         "verdict": verdict, "artifact": str(path), "deleted": payload.get("deleted", False)})
    stamp = time.strftime("%Y%m%d_%H%M%S")
    summary = {"marker": marker, "tools": rows, "skipped_policy_tools": skipped_policy,
               "matrix_size": {tool: len(matrix[tool]) for tool in requested},
               "policy_matrix_size": {tool: len(matrix[tool]) for tool in skipped_policy}}
    (args.out_dir / f"summary_{stamp}.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
    keep = args.out_dir / f"sft_keep_{stamp}.jsonl"
    repair = args.out_dir / f"repair_queue_{stamp}.jsonl"
    with keep.open("w", encoding="utf-8") as keep_file, repair.open("w", encoding="utf-8") as repair_file:
        for row in rows:
            artifact = json.loads(Path(row["artifact"]).read_text(encoding="utf-8"))
            pairs = _history_pairs(artifact.get("history") or {})
            for index, turn in enumerate(artifact["turns"]):
                if not turn.get("pass"):
                    continue
                user, assistant = pairs[index] if index < len(pairs) else ({}, {})
                keep_file.write(json.dumps({"tool": artifact["tool"], "session_id": artifact["session_id"],
                                            "case_id": turn["case_id"], "messages":[
                                                {"role":"user", "content": user.get("content") or turn.get("prompt", "")},
                                                {"role":"assistant", "content": assistant.get("content") or turn.get("response", "")},
                                            ], "turn": turn,
                                            "thinking_preserved": bool(_safe_metadata(assistant).get("thinking")),
                                            "tool_events_preserved": bool(_safe_metadata(assistant).get("tool_events"))}, ensure_ascii=False) + "\n")
            if row["verdict"] != "keep":
                repair_file.write(json.dumps({"tool": artifact["tool"], "session_id": artifact["session_id"],
                                              "turns": artifact["turns"],
                                              "shape_reasons": artifact.get("shape_reasons") or [],
                                              "artifact": row["artifact"],
                                              "deleted": row.get("deleted", False)}, ensure_ascii=False) + "\n")
    print(json.dumps({"summary": str(args.out_dir / f"summary_{stamp}.json"), "keep": str(keep), "repair": str(repair)}, indent=2))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())