Squash Odysseus development history

scripts/eval_odysseus_app_route_smoke.py

@@ -0,0 +1,400 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+import time
+import uuid
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+import sys
+
+import httpx
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from core.database import CalendarCal, CalendarEvent, Document, Note, ScheduledTask, SessionLocal
+from scripts.eval_odysseus_everyday_live_hard import load_cases, score_case, write_markdown
+from scripts.eval_odysseus_live_hard_examples import _parse_tool_args
+from scripts.eval_odysseus_tool_use import _cookie, _raise_for_status_with_body, _sse_events, _visible_event_text
+from scripts.ody_eval_email_fixture import email_fixture
+
+
+DEFAULT_OWNER = "pewds"
+DEFAULT_TZ = "Asia/Tokyo"
+DEFAULT_TZ_OFFSET_MIN = 540
+
+
+def _default_cases() -> list[dict[str, Any]]:
+    return [
+        {
+            "id": "no_tool_web_search_engine",
+            "kind": "negative_web",
+            "user": "Explain what a web search engine does without looking anything up.",
+            "expect_no_tool": True,
+            "must_answer_any": ["search", "pages"],
+        },
+        {
+            "id": "positive_latest_email",
+            "kind": "email",
+            "user": "What's my latest email?",
+            "expect_first_tool": "mcp__email__list_emails",
+            "forbidden_tools": ["web_search"],
+            "must_answer_any": ["latest email", "UID"],
+        },
+    ]
+
+
+def _ensure_calendar(db: Any, owner: str) -> CalendarCal:
+    cal = db.query(CalendarCal).filter(CalendarCal.owner == owner).first()
+    if cal:
+        return cal
+    cal = CalendarCal(id=f"ody-app-gate-cal-{uuid.uuid4().hex[:8]}", owner=owner, name="Odysseus App Gate", source="local")
+    db.add(cal)
+    db.commit()
+    db.refresh(cal)
+    return cal
+
+
+def _precreate_calendar(db: Any, owner: str, fixture: dict[str, str]) -> str:
+    cal = _ensure_calendar(db, owner)
+    uid = f"ody-app-gate-event-{uuid.uuid4().hex[:8]}"
+    event = CalendarEvent(
+        uid=uid,
+        calendar_id=cal.id,
+        summary=fixture["summary"],
+        dtstart=datetime.fromisoformat(fixture["dtstart"]),
+        dtend=datetime.fromisoformat(fixture["dtend"]),
+        all_day=False,
+        is_utc=False,
+        origin="local",
+        status="confirmed",
+    )
+    db.add(event)
+    db.commit()
+    return uid
+
+
+def _create_session(client: httpx.Client, args: argparse.Namespace, case: dict[str, Any]) -> str:
+    create = client.post(
+        args.base_url.rstrip("/") + "/api/session",
+        data={
+            "name": "[eval-app-route] " + case["id"],
+            "endpoint_url": args.endpoint,
+            "endpoint_id": args.endpoint_id,
+            "model": args.model,
+            "skip_validation": "true",
+            "rag": "false",
+        },
+        timeout=30,
+    )
+    _raise_for_status_with_body(create)
+    return create.json()["id"]
+
+
+def _seed_case_state(
+    case: dict[str, Any], args: argparse.Namespace, session_id: str, client: httpx.Client
+) -> dict[str, Any]:
+    state = {"precreated_event_uid": "", "active_document_id": "", "active_document_before": ""}
+    db = SessionLocal()
+    try:
+        if case.get("precreate_calendar_event"):
+            state["precreated_event_uid"] = _precreate_calendar(db, args.owner, case["precreate_calendar_event"])
+        if case.get("active_document"):
+            # Seed through the same authenticated app runtime being evaluated.
+            # Importing SessionLocal here may point at a different deployment's
+            # SQLite file, producing cross-database foreign-key failures or,
+            # worse, a fixture the live 7011 process can never see.
+            fixture = case["active_document"]
+            created = client.post(
+                args.base_url.rstrip("/") + "/api/document",
+                json={
+                    "session_id": session_id,
+                    "title": fixture["title"],
+                    "language": fixture["language"],
+                    "content": fixture["content"],
+                },
+                timeout=30,
+            )
+            _raise_for_status_with_body(created)
+            state["active_document_id"] = created.json()["id"]
+            state["active_document_before"] = fixture["content"]
+    finally:
+        db.close()
+    return state
+
+
+def _tool_calls(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    calls: list[dict[str, Any]] = []
+    for event in events:
+        if event.get("type") != "tool_start":
+            continue
+        calls.append({
+            "tool": event.get("tool"),
+            "args": _parse_tool_args(event.get("full_command") or event.get("command")),
+            "round": event.get("round"),
+        })
+    return calls
+
+
+def _tool_outputs(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    outputs: list[dict[str, Any]] = []
+    for event in events:
+        if event.get("type") != "tool_output":
+            continue
+        outputs.append({
+            "tool": event.get("tool"),
+            "output": event.get("output"),
+            "exit_code": event.get("exit_code"),
+        })
+    return outputs
+
+
+def _collect_and_cleanup(
+    case: dict[str, Any], args: argparse.Namespace, seeded: dict[str, Any], client: httpx.Client
+) -> dict[str, Any]:
+    result_state: dict[str, Any] = {}
+    active_after = ""
+    db = SessionLocal()
+    try:
+        marker_text = case.get("marker") or ""
+        if marker_text:
+            note = db.query(Note).filter(Note.owner == args.owner, Note.archived == False).filter(  # noqa: E712
+                (Note.title.contains(marker_text)) | (Note.content.contains(marker_text))
+            ).first()
+            task = db.query(ScheduledTask).filter(ScheduledTask.owner == args.owner).filter(
+                (ScheduledTask.name.contains(marker_text)) | (ScheduledTask.prompt.contains(marker_text))
+            ).first()
+            events = db.query(CalendarEvent).filter(CalendarEvent.summary.contains(marker_text)).all()
+            result_state["note_found"] = bool(note)
+            result_state["task_found"] = bool(task)
+            result_state["events"] = [
+                {
+                    "uid": event.uid,
+                    "summary": event.summary,
+                    "dtstart": event.dtstart.isoformat(),
+                    "is_utc": bool(event.is_utc),
+                    "status": event.status,
+                }
+                for event in events
+            ]
+            if note:
+                db.delete(note)
+            if task:
+                db.delete(task)
+            for event in events:
+                db.delete(event)
+        active_doc_id = seeded.get("active_document_id") or ""
+        if active_doc_id:
+            response = client.get(
+                args.base_url.rstrip("/") + f"/api/document/{active_doc_id}", timeout=15
+            )
+            if response.is_success:
+                active_after = response.json().get("current_content") or ""
+                result_state["active_document_changed"] = active_after != (seeded.get("active_document_before") or "")
+            with contextlib.suppress(Exception):
+                client.delete(
+                    args.base_url.rstrip("/") + f"/api/document/{active_doc_id}", timeout=15
+                )
+        db.commit()
+    finally:
+        db.close()
+    return {"state": result_state, "active_document_after": active_after}
+
+
+def _run_turn(client: httpx.Client, args: argparse.Namespace, case: dict[str, Any]) -> dict[str, Any]:
+    session_id = _create_session(client, args, case)
+    seeded = _seed_case_state(case, args, session_id, client)
+    events: list[dict[str, Any]] = []
+    prior_events: list[dict[str, Any]] = []
+    response_text: list[str] = []
+    stream_errors: list[dict[str, Any]] = []
+    error = None
+    started = time.time()
+
+    def _form_data(message: str, current_case: dict[str, Any]) -> dict[str, str]:
+        active_email = current_case.get("active_email") or {}
+        form_data = {
+            "message": message,
+            "session": session_id,
+            "mode": "agent",
+            "agent_prompt_mode": "auto",
+            "selected_endpoint_id": args.endpoint_id,
+            "selected_model": args.model,
+            "allow_web_search": (
+                "true"
+                if (
+                    current_case.get("kind") == "web"
+                    or current_case.get("allow_web_search") is True
+                    or current_case.get("expect_first_tool") == "web_search"
+                    or "web_search" in current_case.get("expect_first_tool_any", [])
+                )
+                else ""
+            ),
+            "client_runtime_context": json.dumps(
+                {"timezone": args.timezone, "tz_offset_min": args.tz_offset_min},
+                ensure_ascii=True,
+            ),
+        }
+        if active_email:
+            form_data.update({
+                "active_email_uid": str(active_email.get("uid") or ""),
+                "active_email_folder": str(active_email.get("folder") or "INBOX"),
+                "active_email_account": str(active_email.get("account") or ""),
+            })
+        return form_data
+
+    def _submit(message: str, current_case: dict[str, Any]) -> tuple[list[dict[str, Any]], list[str], list[dict[str, Any]]]:
+        turn_events: list[dict[str, Any]] = []
+        turn_text: list[str] = []
+        turn_stream_errors: list[dict[str, Any]] = []
+        with client.stream(
+            "POST",
+            args.base_url.rstrip("/") + "/api/chat_stream",
+            data=_form_data(message, current_case),
+            headers={
+                "Accept": "text/event-stream",
+                "X-Tz-Name": args.timezone,
+                "X-Tz-Offset": str(args.tz_offset_min),
+            },
+            timeout=args.timeout,
+        ) as response:
+            _raise_for_status_with_body(response)
+            for event in _sse_events(response):
+                turn_events.append(event)
+                if event.get("type") in {"error", "parse_error"}:
+                    turn_stream_errors.append(event)
+                text = _visible_event_text(event)
+                if text:
+                    if event.get("type") == "final_response":
+                        turn_text[:] = [text]
+                    else:
+                        turn_text.append(text)
+        return turn_events, turn_text, turn_stream_errors
+
+    try:
+        for prior in case.get("prior_turns", []):
+            if isinstance(prior, str):
+                prior_case = {"kind": "", "allow_web_search": False}
+                prior_message = prior
+            else:
+                prior_case = prior
+                prior_message = str(prior.get("user") or "")
+            if not prior_message:
+                continue
+            prior_turn_events, _, prior_turn_errors = _submit(prior_message, prior_case)
+            prior_events.extend(prior_turn_events)
+            stream_errors.extend(prior_turn_errors)
+        events, response_text, final_errors = _submit(case["user"], case)
+        stream_errors.extend(final_errors)
+    except Exception as exc:
+        error = repr(exc)
+
+    calls = _tool_calls(events)
+    cleanup = _collect_and_cleanup(case, args, seeded, client)
+    with contextlib.suppress(Exception):
+        client.delete(args.base_url.rstrip("/") + f"/api/session/{session_id}", timeout=15)
+    final_answer = "".join(response_text).strip()
+    result = {
+        "id": case["id"],
+        "kind": case.get("kind", ""),
+        "user": case["user"],
+        "marker": case.get("marker", ""),
+        "first_tool": calls[0]["tool"] if calls else None,
+        "first_tool_args": calls[0]["args"] if calls else None,
+        "tool_names": [call["tool"] for call in calls],
+        "tool_calls": calls,
+        "tool_outputs": _tool_outputs(events),
+        "final_answer": final_answer,
+        "answer": final_answer,
+        "precreated_event_uid": seeded.get("precreated_event_uid", ""),
+        "active_document_before": seeded.get("active_document_before", ""),
+        "active_document_after": cleanup["active_document_after"],
+        "state": cleanup["state"],
+        "stream_errors": stream_errors,
+        "prior_events": prior_events,
+        "events": events,
+        "elapsed_seconds": round(time.time() - started, 3),
+    }
+    passed, failures = score_case(case, result)
+    if error:
+        failures.append(f"exception: {error}")
+        passed = False
+    if stream_errors:
+        failures.append(f"stream errors: {len(stream_errors)}")
+        passed = False
+    result["pass"] = passed
+    result["failures"] = failures
+    return result
+
+
+def _output_paths(args: argparse.Namespace) -> tuple[Path, Path | None]:
+    if args.out_dir:
+        out_dir = Path(args.out_dir)
+        return out_dir / "actual_results.json", out_dir / "actual_results.md"
+    output = Path(args.output)
+    md = output.with_suffix(".md") if args.write_md else None
+    return output, md
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", required=True)
+    parser.add_argument("--endpoint", required=True)
+    parser.add_argument("--endpoint-id", required=True)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--cookie-file", default="data/sessions.json")
+    parser.add_argument("--output", default="data/evals/ody_app_route_smoke_results.json")
+    parser.add_argument("--out-dir", default="")
+    parser.add_argument("--cases-file", default="")
+    parser.add_argument("--email-fixture", action="store_true")
+    parser.add_argument("--owner", default=DEFAULT_OWNER)
+    parser.add_argument("--timezone", default=DEFAULT_TZ)
+    parser.add_argument("--tz-offset-min", type=int, default=DEFAULT_TZ_OFFSET_MIN)
+    parser.add_argument("--timeout", type=float, default=180)
+    parser.add_argument("--write-md", action="store_true")
+    args = parser.parse_args()
+    cases = load_cases(Path(args.cases_file)) if args.cases_file else _default_cases()
+    client = httpx.Client(
+        cookies={"odysseus_session": _cookie(Path(args.cookie_file), args.owner)},
+        follow_redirects=False,
+    )
+    try:
+        with email_fixture(args.email_fixture, owner=args.owner):
+            results = [_run_turn(client, args, case) for case in cases]
+    finally:
+        client.close()
+    summary = {
+        "total": len(results),
+        "passed": sum(1 for result in results if result["pass"]),
+    }
+    summary["failed"] = summary["total"] - summary["passed"]
+    payload = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "base_url": args.base_url,
+        "endpoint": args.endpoint,
+        "endpoint_id": args.endpoint_id,
+        "model": args.model,
+        "owner": args.owner,
+        "timezone": args.timezone,
+        "tz_offset_min": args.tz_offset_min,
+        "summary": summary,
+        "cases": cases,
+        "results": results,
+    }
+    json_path, md_path = _output_paths(args)
+    json_path.parent.mkdir(parents=True, exist_ok=True)
+    json_path.write_text(json.dumps(payload, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
+    if md_path is not None:
+        md_path.parent.mkdir(parents=True, exist_ok=True)
+        write_markdown(md_path, payload)
+    print(json.dumps({"summary": summary, "json": str(json_path), "md": str(md_path) if md_path else ""}, indent=2))
+    return 0 if summary["failed"] == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())