mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-09-16 13:12:20 +02:00
108 lines
4.0 KiB
Python
108 lines
4.0 KiB
Python
import json
|
|
|
|
from scripts.curate_sft_trace_run import (
|
|
curate_rows,
|
|
load_passing_report_sessions,
|
|
load_trace_rows,
|
|
row_runtime_revision,
|
|
write_jsonl,
|
|
)
|
|
|
|
|
|
def test_curate_rows_keeps_only_report_passing_sessions_and_enriches_rows() -> None:
|
|
passing = {
|
|
"s1": {"id": "email_000", "domain": "email"},
|
|
"s2": {"id": "notes_000", "domain": "notes"},
|
|
}
|
|
rows = [
|
|
{"session_id": "s1", "session_name": "SFT trace batch sft_maya_ops email 000", "thinking": "t", "tool_events": [{}]},
|
|
{"session_id": "failed", "session_name": "SFT trace batch sft_maya_ops email 999", "thinking": "bad"},
|
|
{"session_id": "s2", "session_name": "SFT trace batch sft_maya_ops notes 000", "thinking": "", "tool_events": [{}]},
|
|
{"session_id": "s1", "session_name": "duplicate", "thinking": "dup"},
|
|
]
|
|
|
|
curated, summary = curate_rows(rows, passing)
|
|
|
|
assert [row["session_id"] for row in curated] == ["s1", "s2"]
|
|
assert curated[0]["eval_case_id"] == "email_000"
|
|
assert curated[1]["eval_domain"] == "notes"
|
|
assert summary["rows"] == 2
|
|
assert summary["duplicate_sessions_skipped"] == 1
|
|
assert summary["domains"] == {"email": 1, "notes": 1}
|
|
assert summary["rows_with_thinking"] == 1
|
|
assert summary["missing_sessions"] == 0
|
|
|
|
|
|
def test_curate_rows_can_require_thinking() -> None:
|
|
passing = {
|
|
"s1": {"id": "email_000", "domain": "email"},
|
|
"s2": {"id": "calendar_000", "domain": "calendar"},
|
|
}
|
|
rows = [
|
|
{"session_id": "s1", "session_name": "SFT trace batch sft_maya_ops email 000", "thinking": "t"},
|
|
{"session_id": "s2", "session_name": "SFT trace batch sft_maya_ops calendar 000", "thinking": ""},
|
|
]
|
|
|
|
curated, summary = curate_rows(rows, passing, require_thinking=True)
|
|
|
|
assert [row["session_id"] for row in curated] == ["s1"]
|
|
assert summary["skipped_no_thinking"] == 1
|
|
assert summary["missing_sessions"] == 1
|
|
assert summary["missing_without_reason"] == 0
|
|
assert summary["missing_session_ids"] == ["s2"]
|
|
|
|
|
|
def test_curate_rows_requires_runtime_revision_when_requested() -> None:
|
|
passing = {
|
|
"s1": {"id": "email_000", "domain": "email"},
|
|
"s2": {"id": "notes_000", "domain": "notes"},
|
|
"s3": {"id": "calendar_000", "domain": "calendar"},
|
|
}
|
|
rows = [
|
|
{"session_id": "s1", "runtime_revision": "revision-current"},
|
|
{"session_id": "s2"},
|
|
{"session_id": "s3", "metadata": {"runtime_revision": "revision-old"}},
|
|
]
|
|
|
|
curated, summary = curate_rows(
|
|
rows,
|
|
passing,
|
|
require_runtime_revision=True,
|
|
expected_runtime_revision="revision-current",
|
|
)
|
|
|
|
assert [row["session_id"] for row in curated] == ["s1"]
|
|
assert curated[0]["runtime_revision"] == "revision-current"
|
|
assert summary["skipped_missing_runtime_revision"] == 1
|
|
assert summary["skipped_mismatched_runtime_revision"] == 1
|
|
assert summary["missing_without_reason"] == 0
|
|
assert summary["rows_with_runtime_revision"] == 1
|
|
assert summary["missing_runtime_revision_session_ids"] == ["s2"]
|
|
assert summary["mismatched_runtime_revision_session_ids"] == ["s3"]
|
|
|
|
|
|
def test_row_runtime_revision_accepts_metadata_json_string() -> None:
|
|
assert row_runtime_revision({"metadata": "{\"runtime_revision\":\"revision-meta\"}"}) == "revision-meta"
|
|
|
|
|
|
def test_load_and_write_jsonl_round_trip(tmp_path) -> None:
|
|
report = tmp_path / "actual_results.json"
|
|
report.write_text(
|
|
json.dumps(
|
|
{
|
|
"results": [
|
|
{"session_id": "s1", "id": "email_000", "domain": "email", "pass": True},
|
|
{"session_id": "s2", "id": "email_001", "domain": "email", "pass": False},
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
trace = tmp_path / "trace.jsonl"
|
|
write_jsonl(trace, [{"session_id": "s1", "assistant": "ok"}])
|
|
|
|
assert load_passing_report_sessions(report) == {
|
|
"s1": {"session_id": "s1", "id": "email_000", "domain": "email", "pass": True}
|
|
}
|
|
assert load_trace_rows(trace) == [{"session_id": "s1", "assistant": "ok"}]
|