Squash Odysseus development history

2026-10-09 16:32:21 +02:00 · 2026-09-11 06:04:19 +00:00
parent e5c99a5eee
commit 6ee6502010
2050 changed files with 538359 additions and 57745 deletions
@@ -0,0 +1,294 @@
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+from scripts import run_sft_overnight_fixture_flows as sft_flows
+
+
+def test_load_existing_results_keeps_only_clean_allowed_rows(tmp_path: Path) -> None:
+    out_dir = tmp_path / "run"
+    out_dir.mkdir()
+    payload = {
+        "results": [
+            {
+                "id": "email_list_000",
+                "domain": "email",
+                "pass": True,
+                "tool_names": ["mcp__email__list_emails"],
+                "session_id": "old",
+            },
+            {
+                "id": "email_list_000",
+                "domain": "email",
+                "pass": True,
+                "tool_names": ["mcp__email__list_emails"],
+                "session_id": "new",
+            },
+            {
+                "id": "email_today_001",
+                "domain": "email",
+                "pass": True,
+                "tool_names": ["mcp__email__list_emails", "manage_memory"],
+            },
+            {
+                "id": "email_draft_reply_004",
+                "domain": "email",
+                "pass": True,
+                "tool_names": ["ui_control", "ui_control"],
+            },
+            {
+                "id": "notes_002",
+                "domain": "notes",
+                "pass": False,
+                "tool_names": ["manage_notes"],
+            },
+            {
+                "id": "calendar_003",
+                "domain": "calendar",
+                "pass": True,
+                "tool_names": ["manage_calendar"],
+            },
+        ],
+    }
+    (out_dir / "actual_results.json").write_text(json.dumps(payload), encoding="utf-8")
+
+    rows = sft_flows.load_existing_results(
+        out_dir,
+        allowed_ids={"email_list_000", "email_today_001", "email_draft_reply_004", "notes_002"},
+    )
+
+    assert [row["id"] for row in rows] == ["email_list_000"]
+    assert rows[0]["session_id"] == "new"
+
+
+def test_quarantine_sft_rows_moves_matching_session_to_trash(tmp_path: Path, monkeypatch) -> None:
+    monkeypatch.setattr(sft_flows, "DATA_DIR", tmp_path)
+    trace_dir = tmp_path / "sft_traces"
+    trace_dir.mkdir()
+    path = trace_dir / "maya.jsonl"
+    path.write_text(
+        "\n".join(
+            [
+                json.dumps({"session_id": "keep", "assistant": "ok"}),
+                json.dumps({"session_id": "drop", "assistant": "bad"}),
+                "{not-json",
+            ]
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    removed = sft_flows.quarantine_sft_rows("maya", "drop", "duplicate draft")
+
+    assert removed == 1
+    kept = path.read_text(encoding="utf-8").splitlines()
+    assert json.loads(kept[0])["session_id"] == "keep"
+    assert kept[1] == "{not-json"
+    trash_rows = [
+        json.loads(line)
+        for line in (trace_dir / "maya.jsonl.trash").read_text(encoding="utf-8").splitlines()
+    ]
+    assert trash_rows == [
+        {
+            "session_id": "drop",
+            "assistant": "bad",
+            "deleted_from_training": True,
+            "delete_reason": "duplicate draft",
+        }
+    ]
+
+
+def test_score_rejects_duplicate_email_reply_draft_ui_control() -> None:
+    case = {
+        "id": "email_draft_reply_004",
+        "expected_tools": ["ui_control"],
+        "must_contain_any": ["draft"],
+    }
+    events = [
+        {"type": "tool_start", "tool": "ui_control"},
+        {"type": "tool_start", "tool": "ui_control"},
+    ]
+
+    passed, failures = sft_flows.score(case, events, "Draft opened.", {})
+
+    assert not passed
+    assert "duplicate_reply_draft_ui_control" in failures
+
+
+def test_clean_counts_by_domain_counts_only_passing_known_domains() -> None:
+    assert sft_flows.clean_counts_by_domain(
+        [
+            {"domain": "email", "pass": True},
+            {"domain": "email", "pass": False},
+            {"domain": "notes", "pass": True},
+            {"domain": "calendar", "pass": True},
+            {"domain": "other", "pass": True},
+        ]
+    ) == {"email": 1, "notes": 1, "calendar": 1}
+
+
+def test_build_cases_uses_owner_specific_fixture_titles() -> None:
+    maya_text = "\n".join([
+        sft_flows.build_email_case(1, "sft_maya_ops")["must_contain_any"][0],
+        sft_flows.build_note_case(1, "sft_maya_ops")["user"],
+        sft_flows.build_calendar_case(1, "sft_maya_ops")["user"],
+        " ".join(sft_flows.build_calendar_case(0, "sft_maya_ops")["must_contain_any"]),
+    ])
+    jules_text = "\n".join([
+        sft_flows.build_email_case(1, "sft_jules_research")["must_contain_any"][0],
+        sft_flows.build_note_case(1, "sft_jules_research")["user"],
+        sft_flows.build_calendar_case(1, "sft_jules_research")["user"],
+        " ".join(sft_flows.build_calendar_case(0, "sft_jules_research")["must_contain_any"]),
+    ])
+    nora_text = "\n".join([
+        sft_flows.build_email_case(1, "sft_nora_design")["must_contain_any"][0],
+        sft_flows.build_note_case(1, "sft_nora_design")["user"],
+        sft_flows.build_calendar_case(0, "sft_nora_design")["must_contain_any"][0],
+    ])
+    omar_text = "\n".join([
+        sft_flows.build_email_case(1, "sft_omar_finance")["must_contain_any"][0],
+        sft_flows.build_note_case(1, "sft_omar_finance")["user"],
+        sft_flows.build_calendar_case(0, "sft_omar_finance")["must_contain_any"][0],
+    ])
+
+    assert "creator operations" in maya_text
+    assert "Customer success summary" in maya_text
+    assert "Billing" in maya_text
+    assert "research synthesis" in jules_text
+    assert "Appendix cleanup" in jules_text
+    assert "License" in jules_text
+    assert "product design" in nora_text
+    assert "Settings cleanup" in nora_text
+    assert "Onboarding" in nora_text
+    assert "finance planning" in omar_text
+    assert "Contractor list" in omar_text
+    assert "Leadership" in omar_text
+    assert "OVN-JULES" in sft_flows.build_note_case(2, "sft_jules_research")["user"]
+
+
+def test_email_draft_reply_cases_do_not_invite_signature_memory_lookup() -> None:
+    case = sft_flows.build_email_case(4, "sft_nora_design")
+
+    assert case["id"] == "email_draft_reply_004"
+    assert "No signature needed" in case["user"]
+    assert "manage_memory" in case["forbidden_tools"]
+
+
+def test_note_update_cases_use_owner_name() -> None:
+    assert "owner is Nora" in sft_flows.build_note_case(3, "sft_nora_design")["user"]
+    assert "owner is Omar" in sft_flows.build_note_case(3, "sft_omar_finance")["user"]
+
+
+def test_seed_case_removes_stale_marker_rows_before_seeding(monkeypatch) -> None:
+    marker = "OVN-NORA-NOTE-018"
+    stale_note = SimpleNamespace(id="stale", owner="sft_nora_design", title=marker, content=f"{marker} old")
+    created = []
+    deleted = []
+    commits = []
+
+    class _Query:
+        def __init__(self, model):
+            self.model = model
+
+        def filter(self, *_args, **_kwargs):
+            return self
+
+        def join(self, *_args, **_kwargs):
+            return self
+
+        def all(self):
+            if self.model is sft_flows.Note:
+                return [stale_note]
+            return []
+
+        def first(self):
+            return SimpleNamespace(id="cal")
+
+    class _DB:
+        def query(self, model):
+            return _Query(model)
+
+        def add(self, obj):
+            created.append(obj)
+
+        def delete(self, obj):
+            deleted.append(obj)
+
+        def commit(self):
+            commits.append(True)
+
+        def close(self):
+            pass
+
+    monkeypatch.setattr(sft_flows, "SessionLocal", lambda: _DB())
+
+    seeded = sft_flows.seed_case(
+        "sft_nora_design",
+        {
+            "marker": marker,
+            "seed_note": {"title": marker, "content": f"{marker} initial"},
+        },
+    )
+
+    assert deleted == [stale_note]
+    assert len(created) == 1
+    assert created[0].title == marker
+    assert seeded["note_id"].startswith("sft-overnight-note-")
+    assert len(commits) >= 2
+
+
+def test_write_curated_trace_outputs_exports_run_specific_jsonl(tmp_path: Path, monkeypatch) -> None:
+    monkeypatch.setattr(sft_flows, "DATA_DIR", tmp_path)
+    trace_dir = tmp_path / "sft_traces"
+    trace_dir.mkdir()
+    (trace_dir / "maya.jsonl").write_text(
+        "\n".join(
+            [
+                json.dumps({"session_id": "s1", "session_name": "SFT trace batch maya email 000", "thinking": "t", "tool_events": [{}]}),
+                json.dumps({"session_id": "s2", "session_name": "SFT trace batch maya notes 000", "thinking": "", "tool_events": [{}]}),
+                json.dumps({"session_id": "old", "session_name": "SFT trace batch maya email 999", "thinking": "old"}),
+            ]
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    out_dir = tmp_path / "evals" / "run-1"
+    out_dir.mkdir(parents=True)
+    args = SimpleNamespace(owner="maya", out_dir=out_dir)
+    results = [
+        {"session_id": "s1", "id": "email_000", "domain": "email", "pass": True},
+        {"session_id": "s2", "id": "notes_000", "domain": "notes", "pass": True},
+        {"session_id": "failed", "id": "calendar_000", "domain": "calendar", "pass": False},
+    ]
+
+    info = sft_flows.write_curated_trace_outputs(args, results)
+
+    assert info["skipped"] is False
+    curated_path = trace_dir / "maya.run-1.curated.jsonl"
+    thinking_path = trace_dir / "maya.run-1.curated_thinking.jsonl"
+    assert [json.loads(line)["session_id"] for line in curated_path.read_text(encoding="utf-8").splitlines()] == ["s1", "s2"]
+    assert [json.loads(line)["session_id"] for line in thinking_path.read_text(encoding="utf-8").splitlines()] == ["s1"]
+    assert json.loads((out_dir / "curated_trace_summary.json").read_text(encoding="utf-8"))["rows"] == 2
+    thinking_summary = json.loads((out_dir / "curated_thinking_trace_summary.json").read_text(encoding="utf-8"))
+    assert thinking_summary["skipped_no_thinking"] == 1
+    assert thinking_summary["missing_without_reason"] == 0
+
+
+def test_atomic_write_text_preserves_existing_file_on_write_failure(tmp_path: Path, monkeypatch) -> None:
+    path = tmp_path / "actual_results.json"
+    path.write_text("previous", encoding="utf-8")
+    real_write_text = Path.write_text
+
+    def fail_tmp_write(self, *args, **kwargs):
+        if self.name.startswith(".actual_results.json."):
+            raise OSError("disk full")
+        return real_write_text(self, *args, **kwargs)
+
+    monkeypatch.setattr(Path, "write_text", fail_tmp_write)
+
+    try:
+        sft_flows.atomic_write_text(path, "next")
+    except OSError:
+        pass
+
+    assert path.read_text(encoding="utf-8") == "previous"