import json import pytest from scripts.generate_sft_environment_expansion import validate_case from scripts.run_sft_environment_expansion import score_turn def tool_start(action: str) -> dict: return { "type": "tool_start", "tool": "manage_calendar", "command": json.dumps({"action": action}), } def test_calendar_create_requires_create_action_not_any_calendar_call(): turn = { "prompt": "Add a one-hour prep event tomorrow at 9:00 AM.", "expected_tools": ["manage_calendar"], } failures = score_turn(turn, [tool_start("list_calendars")], "Which calendar?") assert any(item.startswith("missing_tool_action") for item in failures) def test_calendar_create_accepts_create_event_action(): turn = { "prompt": "Add a one-hour prep event tomorrow at 9:00 AM.", "expected_tools": ["manage_calendar"], } assert score_turn(turn, [tool_start("create_event")], "Done.") == [] def test_explicit_expected_actions_override_inference(): turn = { "prompt": "Handle the prep entry.", "expected_tools": ["manage_calendar"], "expected_actions": {"manage_calendar": ["delete_event"]}, } failures = score_turn(turn, [tool_start("list_events")], "Found it.") assert any(item.startswith("missing_tool_action") for item in failures) def test_compact_ui_control_command_extracts_action_name(): turn = { "prompt": "Open my calendar.", "expected_tools": ["ui_control"], "expected_actions": {"ui_control": ["open_panel"]}, } events = [{"type": "tool_start", "tool": "ui_control", "command": "open_panel calendar"}] assert score_turn(turn, events, "Opened.") == [] def test_mcp_expected_tool_name_matches_runtime_short_name(): turn = {"prompt": "Search my email.", "expected_tools": ["mcp__email__search_emails"]} events = [{"type": "tool_start", "tool": "search_emails", "command": "{}"}] assert score_turn(turn, events, "Found it.") == [] def test_generated_case_rejects_multiple_tool_families_in_one_turn(): seed = { "seed_family_id": "seed-1", "source_session_id": "session-1", "split": "train", "tools": ["bash", "manage_memory", "web_search"], } raw = { "owner": "sft_maya_ops", "turns": [ {"prompt": "Check the host and list my memories.", "expected_tools": ["bash", "manage_memory"]}, {"prompt": "Search for current operations news.", "expected_tools": ["web_search"]}, {"prompt": "Check the host name.", "expected_tools": ["bash"]}, ], } with pytest.raises(ValueError, match="invalid expected tools"): validate_case(seed, "sft_maya_ops", {"events": []}, raw, 1) def test_generated_case_normalizes_single_expected_tool_string(): seed = { "seed_family_id": "seed-1", "source_session_id": "session-1", "split": "train", "tools": ["manage_calendar"], } raw = { "owner": "sft_maya_ops", "turns": [ {"prompt": "Open my calendar for September 2026.", "expected_tools": "ui_control"}, {"prompt": "Show my events in September 2026.", "expected_tools": "manage_calendar"}, {"prompt": "Open the month view again.", "expected_tools": "ui_control"}, ], } case = validate_case(seed, "sft_maya_ops", {"events": []}, raw, 1) assert case["turns"][0]["expected_tools"] == ["ui_control"] def test_generated_case_rejects_action_contract_for_another_tool(): seed = { "seed_family_id": "seed-1", "source_session_id": "session-1", "split": "train", "tools": ["mcp__email__search_emails", "mcp__email__read_email"], } raw = { "owner": "sft_maya_ops", "turns": [ { "prompt": "Find Iris's email and open the thread.", "expected_tools": ["mcp__email__search_emails"], "expected_actions": {"mcp__email__read_email": ["read"]}, }, {"prompt": "Search again.", "expected_tools": ["mcp__email__search_emails"]}, {"prompt": "Read UID 1.", "expected_tools": ["mcp__email__read_email"]}, ], } with pytest.raises(ValueError, match="outside expected_tools"): validate_case(seed, "sft_maya_ops", {"events": []}, raw, 1) def test_generated_case_rejects_visible_fixture_marker_language(): seed = { "seed_family_id": "seed-1", "source_session_id": "session-1", "split": "train", "tools": ["manage_memory"], } raw = { "owner": "sft_maya_ops", "turns": [ {"prompt": "Save a temporary marker-scoped memory named {marker}.", "expected_tools": ["manage_memory"]}, {"prompt": "List my saved memories.", "expected_tools": ["manage_memory"]}, {"prompt": "Delete that memory.", "expected_tools": ["manage_memory"]}, ], } with pytest.raises(ValueError, match="empty or meta prompt"): validate_case(seed, "sft_maya_ops", {"events": []}, raw, 1) def test_generated_case_rejects_two_mutations_in_one_turn(): seed = { "seed_family_id": "seed-1", "source_session_id": "session-1", "split": "train", "tools": ["manage_memory"], } raw = { "owner": "sft_maya_ops", "turns": [ {"prompt": "Create a memory, then delete it.", "expected_tools": ["manage_memory"]}, {"prompt": "List my memories.", "expected_tools": ["manage_memory"]}, {"prompt": "Search for my preference.", "expected_tools": ["manage_memory"]}, ], } with pytest.raises(ValueError, match="multiple mutations"): validate_case(seed, "sft_maya_ops", {"events": []}, raw, 1)