Squash Odysseus development history

scripts/analyze_odysseus_eval_targets.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Rank next Odysseus tool-router improvement targets from eval artifacts."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+
+def _load(path: Path) -> dict[str, Any]:
+    with path.open("r", encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def _metric(record: dict[str, Any], key: str, default: Any = None) -> Any:
+    metrics = record.get("metrics") or {}
+    return metrics.get(key, default)
+
+
+def _tool_rounds(record: dict[str, Any]) -> int:
+    metrics = record.get("metrics") or {}
+    usage = metrics.get("usage_buckets") or []
+    round_models = metrics.get("round_models") or []
+    if usage:
+        return len(usage)
+    if round_models:
+        return len(round_models)
+    snapshots = record.get("model_request_snapshots") or []
+    if snapshots:
+        return len(snapshots)
+    return 0
+
+
+def _is_infra_failure_error(error: dict[str, Any]) -> bool:
+    if not isinstance(error, dict):
+        return False
+    status = error.get("status")
+    text = " ".join(
+        str(error.get(key) or "")
+        for key in ("error", "message", "detail", "type")
+    ).lower()
+    if status in {502, 503, 504, 520, 521, 522, 523, 524}:
+        return True
+    return bool(
+        "cannot reach" in text
+        or "connection refused" in text
+        or "connection reset" in text
+        or "connect timeout" in text
+        or "read timeout" in text
+        or "unreachable" in text
+        or "cooldown active" in text
+        or "upstream protocol error" in text
+        or ("upstream" in text and "failed" in text)
+    )
+
+
+def _record_has_infra_error(record: dict[str, Any]) -> bool:
+    if record.get("infra_failure") is True:
+        return True
+    errors = list(record.get("stream_errors") or [])
+    stream_exception = record.get("stream_exception")
+    if isinstance(stream_exception, dict):
+        errors.append(stream_exception)
+    return any(_is_infra_failure_error(error) for error in errors)
+
+
+def _record_status(record: dict[str, Any]) -> str:
+    if _record_has_infra_error(record):
+        return "infra"
+    if not record.get("native_call_ok"):
+        return "routing"
+    if not record.get("command_contract_ok"):
+        return "contract"
+    if not record.get("tool_invocation_ok"):
+        return "invocation"
+    if not record.get("command_outcome_ok"):
+        return "outcome"
+    if not record.get("response_quality_ok"):
+        return "response"
+    if record.get("duplicate_textual_call"):
+        return "duplicate_text"
+    if record.get("repetitive_tool_call"):
+        return "repeat"
+    return "pass"
+
+
+def _first_output(record: dict[str, Any]) -> dict[str, Any]:
+    outputs = record.get("tool_outputs") or []
+    return outputs[0] if outputs else {}
+
+
+def _print_row(record: dict[str, Any]) -> None:
+    case = record.get("case")
+    status = _record_status(record)
+    first_tool = record.get("first_tool")
+    expected = record.get("expected_tool")
+    output = _first_output(record)
+    input_tokens = _metric(record, "input_tokens")
+    response_time = _metric(record, "response_time")
+    elapsed = record.get("elapsed_seconds")
+    rounds = _tool_rounds(record)
+    exit_code = output.get("exit_code")
+    print(
+        f"- {case}: status={status}, expected={expected}, first={first_tool}, "
+        f"rounds={rounds}, input={input_tokens}, response={response_time}s, "
+        f"elapsed={elapsed}s, exit={exit_code}"
+    )
+
+
+def _top(records: list[dict[str, Any]], key, limit: int) -> list[dict[str, Any]]:
+    return sorted(records, key=key, reverse=True)[:limit]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("artifact", type=Path)
+    parser.add_argument("--limit", type=int, default=12)
+    args = parser.parse_args()
+
+    artifact = _load(args.artifact)
+    records = list(artifact.get("records") or [])
+    infra = [record for record in records if _record_has_infra_error(record)]
+    evaluable = [record for record in records if not _record_has_infra_error(record)]
+    failed = [record for record in evaluable if _record_status(record) != "pass"]
+    slow = _top(
+        [record for record in evaluable if _metric(record, "response_time") is not None],
+        lambda record: float(_metric(record, "response_time", 0) or 0),
+        args.limit,
+    )
+    token_heavy = _top(
+        [record for record in evaluable if _metric(record, "input_tokens") is not None],
+        lambda record: int(_metric(record, "input_tokens", 0) or 0),
+        args.limit,
+    )
+    multi_round = _top(
+        [record for record in evaluable if _tool_rounds(record) > 1],
+        lambda record: (_tool_rounds(record), float(_metric(record, "response_time", 0) or 0)),
+        args.limit,
+    )
+
+    print(f"artifact: {args.artifact}")
+    print(f"model: {artifact.get('model')}")
+    print(f"cases: {artifact.get('cases', len(records))}")
+    print(f"infra: {len(infra)}")
+    print(f"evaluable: {len(evaluable)}")
+    print(f"failures: {len(failed)}")
+    print()
+
+    print("failures:")
+    if failed:
+        for record in failed:
+            _print_row(record)
+    else:
+        print("- none")
+    print()
+
+    print(f"slowest_{len(slow)}:")
+    for record in slow:
+        _print_row(record)
+    print()
+
+    print(f"token_heaviest_{len(token_heavy)}:")
+    for record in token_heavy:
+        _print_row(record)
+    print()
+
+    print(f"multi_round_{len(multi_round)}:")
+    if multi_round:
+        for record in multi_round:
+            _print_row(record)
+    else:
+        print("- none")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())