Squash Odysseus development history

tests/test_curate_sft_trace_run.py

@@ -0,0 +1,107 @@
+import json
+
+from scripts.curate_sft_trace_run import (
+    curate_rows,
+    load_passing_report_sessions,
+    load_trace_rows,
+    row_runtime_revision,
+    write_jsonl,
+)
+
+
+def test_curate_rows_keeps_only_report_passing_sessions_and_enriches_rows() -> None:
+    passing = {
+        "s1": {"id": "email_000", "domain": "email"},
+        "s2": {"id": "notes_000", "domain": "notes"},
+    }
+    rows = [
+        {"session_id": "s1", "session_name": "SFT trace batch sft_maya_ops email 000", "thinking": "t", "tool_events": [{}]},
+        {"session_id": "failed", "session_name": "SFT trace batch sft_maya_ops email 999", "thinking": "bad"},
+        {"session_id": "s2", "session_name": "SFT trace batch sft_maya_ops notes 000", "thinking": "", "tool_events": [{}]},
+        {"session_id": "s1", "session_name": "duplicate", "thinking": "dup"},
+    ]
+
+    curated, summary = curate_rows(rows, passing)
+
+    assert [row["session_id"] for row in curated] == ["s1", "s2"]
+    assert curated[0]["eval_case_id"] == "email_000"
+    assert curated[1]["eval_domain"] == "notes"
+    assert summary["rows"] == 2
+    assert summary["duplicate_sessions_skipped"] == 1
+    assert summary["domains"] == {"email": 1, "notes": 1}
+    assert summary["rows_with_thinking"] == 1
+    assert summary["missing_sessions"] == 0
+
+
+def test_curate_rows_can_require_thinking() -> None:
+    passing = {
+        "s1": {"id": "email_000", "domain": "email"},
+        "s2": {"id": "calendar_000", "domain": "calendar"},
+    }
+    rows = [
+        {"session_id": "s1", "session_name": "SFT trace batch sft_maya_ops email 000", "thinking": "t"},
+        {"session_id": "s2", "session_name": "SFT trace batch sft_maya_ops calendar 000", "thinking": ""},
+    ]
+
+    curated, summary = curate_rows(rows, passing, require_thinking=True)
+
+    assert [row["session_id"] for row in curated] == ["s1"]
+    assert summary["skipped_no_thinking"] == 1
+    assert summary["missing_sessions"] == 1
+    assert summary["missing_without_reason"] == 0
+    assert summary["missing_session_ids"] == ["s2"]
+
+
+def test_curate_rows_requires_runtime_revision_when_requested() -> None:
+    passing = {
+        "s1": {"id": "email_000", "domain": "email"},
+        "s2": {"id": "notes_000", "domain": "notes"},
+        "s3": {"id": "calendar_000", "domain": "calendar"},
+    }
+    rows = [
+        {"session_id": "s1", "runtime_revision": "revision-current"},
+        {"session_id": "s2"},
+        {"session_id": "s3", "metadata": {"runtime_revision": "revision-old"}},
+    ]
+
+    curated, summary = curate_rows(
+        rows,
+        passing,
+        require_runtime_revision=True,
+        expected_runtime_revision="revision-current",
+    )
+
+    assert [row["session_id"] for row in curated] == ["s1"]
+    assert curated[0]["runtime_revision"] == "revision-current"
+    assert summary["skipped_missing_runtime_revision"] == 1
+    assert summary["skipped_mismatched_runtime_revision"] == 1
+    assert summary["missing_without_reason"] == 0
+    assert summary["rows_with_runtime_revision"] == 1
+    assert summary["missing_runtime_revision_session_ids"] == ["s2"]
+    assert summary["mismatched_runtime_revision_session_ids"] == ["s3"]
+
+
+def test_row_runtime_revision_accepts_metadata_json_string() -> None:
+    assert row_runtime_revision({"metadata": "{\"runtime_revision\":\"revision-meta\"}"}) == "revision-meta"
+
+
+def test_load_and_write_jsonl_round_trip(tmp_path) -> None:
+    report = tmp_path / "actual_results.json"
+    report.write_text(
+        json.dumps(
+            {
+                "results": [
+                    {"session_id": "s1", "id": "email_000", "domain": "email", "pass": True},
+                    {"session_id": "s2", "id": "email_001", "domain": "email", "pass": False},
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+    trace = tmp_path / "trace.jsonl"
+    write_jsonl(trace, [{"session_id": "s1", "assistant": "ok"}])
+
+    assert load_passing_report_sessions(report) == {
+        "s1": {"session_id": "s1", "id": "email_000", "domain": "email", "pass": True}
+    }
+    assert load_trace_rows(trace) == [{"session_id": "s1", "assistant": "ok"}]