odysseus/scripts/eval_odysseus_app_route_smoke.py

#!/usr/bin/env python3
from __future__ import annotations

import argparse
import contextlib
import json
import time
import uuid
from datetime import datetime
from pathlib import Path
from typing import Any
import sys

import httpx

REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
    sys.path.insert(0, str(REPO_ROOT))

from core.database import CalendarCal, CalendarEvent, Document, Note, ScheduledTask, SessionLocal
from scripts.eval_odysseus_everyday_live_hard import load_cases, score_case, write_markdown
from scripts.eval_odysseus_live_hard_examples import _parse_tool_args
from scripts.eval_odysseus_tool_use import _cookie, _raise_for_status_with_body, _sse_events, _visible_event_text
from scripts.ody_eval_email_fixture import email_fixture


DEFAULT_OWNER = "pewds"
DEFAULT_TZ = "Asia/Tokyo"
DEFAULT_TZ_OFFSET_MIN = 540


def _default_cases() -> list[dict[str, Any]]:
    return [
        {
            "id": "no_tool_web_search_engine",
            "kind": "negative_web",
            "user": "Explain what a web search engine does without looking anything up.",
            "expect_no_tool": True,
            "must_answer_any": ["search", "pages"],
        },
        {
            "id": "positive_latest_email",
            "kind": "email",
            "user": "What's my latest email?",
            "expect_first_tool": "mcp__email__list_emails",
            "forbidden_tools": ["web_search"],
            "must_answer_any": ["latest email", "UID"],
        },
    ]


def _ensure_calendar(db: Any, owner: str) -> CalendarCal:
    cal = db.query(CalendarCal).filter(CalendarCal.owner == owner).first()
    if cal:
        return cal
    cal = CalendarCal(id=f"ody-app-gate-cal-{uuid.uuid4().hex[:8]}", owner=owner, name="Odysseus App Gate", source="local")
    db.add(cal)
    db.commit()
    db.refresh(cal)
    return cal


def _precreate_calendar(db: Any, owner: str, fixture: dict[str, str]) -> str:
    cal = _ensure_calendar(db, owner)
    uid = f"ody-app-gate-event-{uuid.uuid4().hex[:8]}"
    event = CalendarEvent(
        uid=uid,
        calendar_id=cal.id,
        summary=fixture["summary"],
        dtstart=datetime.fromisoformat(fixture["dtstart"]),
        dtend=datetime.fromisoformat(fixture["dtend"]),
        all_day=False,
        is_utc=False,
        origin="local",
        status="confirmed",
    )
    db.add(event)
    db.commit()
    return uid


def _create_session(client: httpx.Client, args: argparse.Namespace, case: dict[str, Any]) -> str:
    create = client.post(
        args.base_url.rstrip("/") + "/api/session",
        data={
            "name": "[eval-app-route] " + case["id"],
            "endpoint_url": args.endpoint,
            "endpoint_id": args.endpoint_id,
            "model": args.model,
            "skip_validation": "true",
            "rag": "false",
        },
        timeout=30,
    )
    _raise_for_status_with_body(create)
    return create.json()["id"]


def _seed_case_state(
    case: dict[str, Any], args: argparse.Namespace, session_id: str, client: httpx.Client
) -> dict[str, Any]:
    state = {"precreated_event_uid": "", "active_document_id": "", "active_document_before": ""}
    db = SessionLocal()
    try:
        if case.get("precreate_calendar_event"):
            state["precreated_event_uid"] = _precreate_calendar(db, args.owner, case["precreate_calendar_event"])
        if case.get("active_document"):
            # Seed through the same authenticated app runtime being evaluated.
            # Importing SessionLocal here may point at a different deployment's
            # SQLite file, producing cross-database foreign-key failures or,
            # worse, a fixture the live 7011 process can never see.
            fixture = case["active_document"]
            created = client.post(
                args.base_url.rstrip("/") + "/api/document",
                json={
                    "session_id": session_id,
                    "title": fixture["title"],
                    "language": fixture["language"],
                    "content": fixture["content"],
                },
                timeout=30,
            )
            _raise_for_status_with_body(created)
            state["active_document_id"] = created.json()["id"]
            state["active_document_before"] = fixture["content"]
    finally:
        db.close()
    return state


def _tool_calls(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
    calls: list[dict[str, Any]] = []
    for event in events:
        if event.get("type") != "tool_start":
            continue
        calls.append({
            "tool": event.get("tool"),
            "args": _parse_tool_args(event.get("full_command") or event.get("command")),
            "round": event.get("round"),
        })
    return calls


def _tool_outputs(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
    outputs: list[dict[str, Any]] = []
    for event in events:
        if event.get("type") != "tool_output":
            continue
        outputs.append({
            "tool": event.get("tool"),
            "output": event.get("output"),
            "exit_code": event.get("exit_code"),
        })
    return outputs


def _collect_and_cleanup(
    case: dict[str, Any], args: argparse.Namespace, seeded: dict[str, Any], client: httpx.Client
) -> dict[str, Any]:
    result_state: dict[str, Any] = {}
    active_after = ""
    db = SessionLocal()
    try:
        marker_text = case.get("marker") or ""
        if marker_text:
            note = db.query(Note).filter(Note.owner == args.owner, Note.archived == False).filter(  # noqa: E712
                (Note.title.contains(marker_text)) | (Note.content.contains(marker_text))
            ).first()
            task = db.query(ScheduledTask).filter(ScheduledTask.owner == args.owner).filter(
                (ScheduledTask.name.contains(marker_text)) | (ScheduledTask.prompt.contains(marker_text))
            ).first()
            events = db.query(CalendarEvent).filter(CalendarEvent.summary.contains(marker_text)).all()
            result_state["note_found"] = bool(note)
            result_state["task_found"] = bool(task)
            result_state["events"] = [
                {
                    "uid": event.uid,
                    "summary": event.summary,
                    "dtstart": event.dtstart.isoformat(),
                    "is_utc": bool(event.is_utc),
                    "status": event.status,
                }
                for event in events
            ]
            if note:
                db.delete(note)
            if task:
                db.delete(task)
            for event in events:
                db.delete(event)
        active_doc_id = seeded.get("active_document_id") or ""
        if active_doc_id:
            response = client.get(
                args.base_url.rstrip("/") + f"/api/document/{active_doc_id}", timeout=15
            )
            if response.is_success:
                active_after = response.json().get("current_content") or ""
                result_state["active_document_changed"] = active_after != (seeded.get("active_document_before") or "")
            with contextlib.suppress(Exception):
                client.delete(
                    args.base_url.rstrip("/") + f"/api/document/{active_doc_id}", timeout=15
                )
        db.commit()
    finally:
        db.close()
    return {"state": result_state, "active_document_after": active_after}


def _run_turn(client: httpx.Client, args: argparse.Namespace, case: dict[str, Any]) -> dict[str, Any]:
    session_id = _create_session(client, args, case)
    seeded = _seed_case_state(case, args, session_id, client)
    events: list[dict[str, Any]] = []
    prior_events: list[dict[str, Any]] = []
    response_text: list[str] = []
    stream_errors: list[dict[str, Any]] = []
    error = None
    started = time.time()

    def _form_data(message: str, current_case: dict[str, Any]) -> dict[str, str]:
        active_email = current_case.get("active_email") or {}
        form_data = {
            "message": message,
            "session": session_id,
            "mode": "agent",
            "agent_prompt_mode": "auto",
            "selected_endpoint_id": args.endpoint_id,
            "selected_model": args.model,
            "allow_web_search": (
                "true"
                if (
                    current_case.get("kind") == "web"
                    or current_case.get("allow_web_search") is True
                    or current_case.get("expect_first_tool") == "web_search"
                    or "web_search" in current_case.get("expect_first_tool_any", [])
                )
                else ""
            ),
            "client_runtime_context": json.dumps(
                {"timezone": args.timezone, "tz_offset_min": args.tz_offset_min},
                ensure_ascii=True,
            ),
        }
        if active_email:
            form_data.update({
                "active_email_uid": str(active_email.get("uid") or ""),
                "active_email_folder": str(active_email.get("folder") or "INBOX"),
                "active_email_account": str(active_email.get("account") or ""),
            })
        return form_data

    def _submit(message: str, current_case: dict[str, Any]) -> tuple[list[dict[str, Any]], list[str], list[dict[str, Any]]]:
        turn_events: list[dict[str, Any]] = []
        turn_text: list[str] = []
        turn_stream_errors: list[dict[str, Any]] = []
        with client.stream(
            "POST",
            args.base_url.rstrip("/") + "/api/chat_stream",
            data=_form_data(message, current_case),
            headers={
                "Accept": "text/event-stream",
                "X-Tz-Name": args.timezone,
                "X-Tz-Offset": str(args.tz_offset_min),
            },
            timeout=args.timeout,
        ) as response:
            _raise_for_status_with_body(response)
            for event in _sse_events(response):
                turn_events.append(event)
                if event.get("type") in {"error", "parse_error"}:
                    turn_stream_errors.append(event)
                text = _visible_event_text(event)
                if text:
                    if event.get("type") == "final_response":
                        turn_text[:] = [text]
                    else:
                        turn_text.append(text)
        return turn_events, turn_text, turn_stream_errors

    try:
        for prior in case.get("prior_turns", []):
            if isinstance(prior, str):
                prior_case = {"kind": "", "allow_web_search": False}
                prior_message = prior
            else:
                prior_case = prior
                prior_message = str(prior.get("user") or "")
            if not prior_message:
                continue
            prior_turn_events, _, prior_turn_errors = _submit(prior_message, prior_case)
            prior_events.extend(prior_turn_events)
            stream_errors.extend(prior_turn_errors)
        events, response_text, final_errors = _submit(case["user"], case)
        stream_errors.extend(final_errors)
    except Exception as exc:
        error = repr(exc)

    calls = _tool_calls(events)
    cleanup = _collect_and_cleanup(case, args, seeded, client)
    with contextlib.suppress(Exception):
        client.delete(args.base_url.rstrip("/") + f"/api/session/{session_id}", timeout=15)
    final_answer = "".join(response_text).strip()
    result = {
        "id": case["id"],
        "kind": case.get("kind", ""),
        "user": case["user"],
        "marker": case.get("marker", ""),
        "first_tool": calls[0]["tool"] if calls else None,
        "first_tool_args": calls[0]["args"] if calls else None,
        "tool_names": [call["tool"] for call in calls],
        "tool_calls": calls,
        "tool_outputs": _tool_outputs(events),
        "final_answer": final_answer,
        "answer": final_answer,
        "precreated_event_uid": seeded.get("precreated_event_uid", ""),
        "active_document_before": seeded.get("active_document_before", ""),
        "active_document_after": cleanup["active_document_after"],
        "state": cleanup["state"],
        "stream_errors": stream_errors,
        "prior_events": prior_events,
        "events": events,
        "elapsed_seconds": round(time.time() - started, 3),
    }
    passed, failures = score_case(case, result)
    if error:
        failures.append(f"exception: {error}")
        passed = False
    if stream_errors:
        failures.append(f"stream errors: {len(stream_errors)}")
        passed = False
    result["pass"] = passed
    result["failures"] = failures
    return result


def _output_paths(args: argparse.Namespace) -> tuple[Path, Path | None]:
    if args.out_dir:
        out_dir = Path(args.out_dir)
        return out_dir / "actual_results.json", out_dir / "actual_results.md"
    output = Path(args.output)
    md = output.with_suffix(".md") if args.write_md else None
    return output, md


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--base-url", required=True)
    parser.add_argument("--endpoint", required=True)
    parser.add_argument("--endpoint-id", required=True)
    parser.add_argument("--model", required=True)
    parser.add_argument("--cookie-file", default="data/sessions.json")
    parser.add_argument("--output", default="data/evals/ody_app_route_smoke_results.json")
    parser.add_argument("--out-dir", default="")
    parser.add_argument("--cases-file", default="")
    parser.add_argument("--email-fixture", action="store_true")
    parser.add_argument("--owner", default=DEFAULT_OWNER)
    parser.add_argument("--timezone", default=DEFAULT_TZ)
    parser.add_argument("--tz-offset-min", type=int, default=DEFAULT_TZ_OFFSET_MIN)
    parser.add_argument("--timeout", type=float, default=180)
    parser.add_argument("--write-md", action="store_true")
    args = parser.parse_args()
    cases = load_cases(Path(args.cases_file)) if args.cases_file else _default_cases()
    client = httpx.Client(
        cookies={"odysseus_session": _cookie(Path(args.cookie_file), args.owner)},
        follow_redirects=False,
    )
    try:
        with email_fixture(args.email_fixture, owner=args.owner):
            results = [_run_turn(client, args, case) for case in cases]
    finally:
        client.close()
    summary = {
        "total": len(results),
        "passed": sum(1 for result in results if result["pass"]),
    }
    summary["failed"] = summary["total"] - summary["passed"]
    payload = {
        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
        "base_url": args.base_url,
        "endpoint": args.endpoint,
        "endpoint_id": args.endpoint_id,
        "model": args.model,
        "owner": args.owner,
        "timezone": args.timezone,
        "tz_offset_min": args.tz_offset_min,
        "summary": summary,
        "cases": cases,
        "results": results,
    }
    json_path, md_path = _output_paths(args)
    json_path.parent.mkdir(parents=True, exist_ok=True)
    json_path.write_text(json.dumps(payload, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
    if md_path is not None:
        md_path.parent.mkdir(parents=True, exist_ok=True)
        write_markdown(md_path, payload)
    print(json.dumps({"summary": summary, "json": str(json_path), "md": str(md_path) if md_path else ""}, indent=2))
    return 0 if summary["failed"] == 0 else 1


if __name__ == "__main__":
    raise SystemExit(main())