from scripts.odysseus_domain_audit import Case, _durable_tool_events, prompt_matrix, score_case from scripts.odysseus_related_flow_audit import ( Flow, FlowTurn, _flow_has_good_training_shape, _provider_failure, _score_turn, ) from scripts.odysseus_remaining_tool_audit import REMAINING_TOOLS, SFT_POLICY_DISABLED_TOOLS, tool_matrix from src.tool_index import ToolIndex from src.agent_loop import _SFT_DISABLED_WORKSPACE_TOOLS, _STATEFUL_TOOL_CARRYOVER_DOMAINS def test_domain_matrix_has_twenty_cases_per_domain(): matrix = prompt_matrix() assert set(matrix) == {"skills", "tasks", "theme", "memory", "documents", "cookbook"} assert {name: len(cases) for name, cases in matrix.items()} == { "skills": 20, "tasks": 20, "theme": 20, "memory": 20, "documents": 20, "cookbook": 20, } assert len({case.id for cases in matrix.values() for case in cases}) == 120 def test_cookbook_cases_reject_mutating_tools(): case = Case("cookbook_test", "find a model", ("search_hf_models",), dry_run=True) events = [{"type": "tool_start", "tool": "serve_model"}, {"type": "tool_output", "tool": "serve_model", "output": "started"}] result = score_case(case, events, "I started it") assert result["dry_run_ok"] is False assert result["pass"] is False def test_score_rejects_raw_tool_dump_and_unavailable_claim(): case = Case("memory_test", "list memories", ("manage_memory",)) events = [{"type": "tool_start", "tool": "manage_memory"}, {"type": "tool_output", "tool": "manage_memory", "output": "ok"}] result = score_case(case, events, 'I don\'t have a manage_memory tool.') assert result["response_ok"] is False assert result["pass"] is False def test_score_accepts_clean_named_tool_trace(): case = Case("tasks_test", "list tasks", ("manage_tasks",)) events = [{"type": "tool_start", "tool": "manage_tasks"}, {"type": "tool_output", "tool": "manage_tasks", "output": "Found 0 tasks"}] result = score_case(case, events, "You have no scheduled tasks.") assert result["pass"] is True def test_skill_keyword_fallback_keeps_registry_tool_available(): matching = [tools for keywords, tools in ToolIndex._KEYWORD_HINTS.items() if any(keyword in "list my skills" for keyword in keywords)] assert matching assert any("manage_skills" in tools for tools in matching) def test_stateful_domains_carry_one_round_after_use(): assert _STATEFUL_TOOL_CARRYOVER_DOMAINS["manage_skills"] == "skills" assert _STATEFUL_TOOL_CARRYOVER_DOMAINS["manage_memory"] == "memory" assert _STATEFUL_TOOL_CARRYOVER_DOMAINS["manage_documents"] == "documents" assert _STATEFUL_TOOL_CARRYOVER_DOMAINS["manage_tasks"] == "notes_calendar_tasks" assert _STATEFUL_TOOL_CARRYOVER_DOMAINS["ui_control"] == "ui" assert _STATEFUL_TOOL_CARRYOVER_DOMAINS["list_served_models"] == "cookbook" def test_durable_history_is_the_tool_event_source_for_scoring(): history = { "history": [{ "role": "assistant", "content": "Done.", "metadata": {"tool_events": [{"tool": "manage_tasks", "command": '{"action":"list"}'}]}, }] } assert _durable_tool_events(history)[0]["tool"] == "manage_tasks" def test_related_flow_shape_rejects_false_failure_claims(): history = { "history": [ {"role": "user", "content": "Update that skill."}, { "role": "assistant", "content": "The skill registry may have failed, but I updated it.", "metadata": {"tool_events": [{"tool": "manage_skills"}]}, }, ] } ok, reasons = _flow_has_good_training_shape(history, 1) assert ok is False assert reasons == ["turn 1 contains a false/ambiguous failure claim"] def test_related_flow_shape_accepts_verifiability_language_with_cant(): history = { "history": [ {"role": "user", "content": "Ask the teacher to check the rewrite."}, { "role": "assistant", "content": ( "The teacher used the tool result. If the evidence can't be " "checked independently of the assistant's claim, it is not grounded." ), "metadata": {"tool_events": [{"tool": "ask_teacher", "output": "Valid review"}]}, }, ] } ok, reasons = _flow_has_good_training_shape(history, 1) assert ok is True assert reasons == [] def test_related_flow_shape_rejects_persisted_tool_errors(): history = { "history": [ {"role": "user", "content": "Draft a reply."}, { "role": "assistant", "content": "Reply draft opened. Nothing has been sent.", "metadata": { "tool_events": [ { "tool": "mcp__email__draft_email_reply", "output": "Error: [Errno 111] Connection refused", } ] }, }, ] } ok, reasons = _flow_has_good_training_shape(history, 1) assert ok is False assert reasons == ["turn 1 has failed tool output from mcp__email__draft_email_reply"] def test_related_flow_shape_rejects_missing_just_saved_memory(): history = { "history": [ {"role": "user", "content": "Find the memory you just saved about marker audit-123."}, { "role": "assistant", "content": "No memories found matching 'audit-123'.", "metadata": {"tool_events": [{"tool": "manage_memory", "output": "No memories found"}]}, }, ] } ok, reasons = _flow_has_good_training_shape(history, 1) assert ok is False assert reasons == ["turn 1 failed to find the just-saved memory"] def test_related_flow_shape_rejects_repeated_explicit_tool_loop(): history = { "history": [ {"role": "user", "content": "Show which agent tools are currently disabled."}, { "role": "assistant", "content": ( "No agent tools are currently disabled, so every core tool domain is enabled. " "No agent tools are currently disabled, so every core tool domain is enabled." ), "metadata": { "tool_events": [ {"tool": "manage_settings", "command": '{"action":"list_tools"}', "output": "ok"}, {"tool": "manage_settings", "command": '{"action":"list_tools"}', "output": "ok"}, ], "round_texts": [ "No agent tools are currently disabled.", "No agent tools are currently disabled.", ], }, }, ] } ok, reasons = _flow_has_good_training_shape(history, 1) assert ok is False assert "turn 1 repeated 1 identical tool call(s)" in reasons assert "turn 1 has multiple non-empty assistant rounds" in reasons assert "turn 1 appears to concatenate repeated assistant answers" in reasons def test_related_flow_shape_accepts_cumulative_progress_rounds(): preamble = "I'll check the inbox first." final = preamble + " The inbox is clean; Junk contains one suspicious message." history = { "history": [ {"role": "user", "content": "Check Inbox and Junk for spam."}, { "role": "assistant", "content": final, "metadata": { "tool_events": [ {"tool": "mcp__email__scan_spam", "command": '{"folder":"INBOX"}', "output": "clean"}, {"tool": "mcp__email__scan_spam", "command": '{"folder":"Junk"}', "output": "one candidate"}, ], "round_texts": [preamble, final], }, }, ] } ok, reasons = _flow_has_good_training_shape(history, 1) assert ok is True assert reasons == [] def test_related_flow_shape_rejects_missing_just_created_chat(): history = { "history": [ {"role": "user", "content": "Find the audit helper abc chat in my chat list."}, { "role": "assistant", "content": "No sessions found matching 'audit abc'.", "metadata": {"tool_events": [{"tool": "list_sessions", "output": "No sessions found"}]}, }, ] } ok, reasons = _flow_has_good_training_shape(history, 1) assert ok is False assert reasons == ["turn 1 failed to find the just-created chat"] def test_related_flow_shape_rejects_raw_calendar_event_dump(): history = { "history": [ {"role": "user", "content": "What events are visible for the next week?"}, { "role": "assistant", "content": ( "Here are your events (2):\n" "- [Stretch](#event-a) — 2026-08-28T07:30:00Z -> 2026-08-28T08:00:00Z #health\n" "- [Haircut](#event-b) — 2026-08-28T09:00:00Z -> 2026-08-28T10:00:00Z #personal" ), "metadata": {"tool_events": [{"tool": "manage_calendar", "output": "ok"}]}, }, ] } ok, reasons = _flow_has_good_training_shape(history, 1) assert ok is False assert reasons == ["turn 1 appears to preserve a raw harness dump"] def test_related_flow_score_requires_compound_tools_for_notes_handoff(): flow = Flow( "ui_calendar_notes_context", "notes", "UI panel context handoff", (FlowTurn("open_notes", "Open notes and create a note", ("ui_control", "manage_notes")),), ) events = [ {"type": "tool_start", "tool": "ui_control", "command": "open_panel notes"}, {"type": "tool_output", "tool": "ui_control", "output": "Opening notes panel"}, ] result = _score_turn(flow, flow.turns[0], events, "The notes panel is open.") assert result["pass"] is False assert result["missing_required_tools"] == ["manage_notes"] def test_related_flow_classifies_provider_status_as_infrastructure_failure(): events = [{"type": "error", "error": "Read timeout", "status": 504}] assert _provider_failure(events) is True def test_related_flow_classifies_provider_cooldown_as_infrastructure_failure(): events = [{ "type": "error", "error": "Upstream https://openrouter.ai unreachable (cooldown active)", "status": 503, }] assert _provider_failure(events, "The model provider returned no usable output.") is True def test_related_flow_classifies_missing_enabled_endpoint_as_infrastructure_failure(): events = [{ "type": "tool_output", "tool": "generate_image", "output": "Error: No enabled endpoints found", "exit_code": 0, }] assert _provider_failure(events) is True def test_related_flow_does_not_classify_tool_miss_as_provider_failure(): events = [{"type": "tool_output", "tool": "cancel_download", "output": "No download found"}] assert _provider_failure(events, "I could not find that download.") is False def test_sft_workspace_filter_keeps_private_skill_registry(): assert "manage_skills" not in _SFT_DISABLED_WORKSPACE_TOOLS def test_remaining_tool_matrix_has_twenty_cases_per_tool(): matrix = tool_matrix() assert set(matrix) == set(REMAINING_TOOLS) assert all(len(cases) == 20 for cases in matrix.values()) assert len({case.id for cases in matrix.values() for case in cases}) == 20 * len(REMAINING_TOOLS) def test_remaining_mutation_cases_are_dry_run(): matrix = tool_matrix() for tool in ("serve_model", "download_model", "stop_served_model", "bulk_email"): assert all(case.dry_run for case in matrix[tool]) def test_workspace_tools_are_explicitly_separated_from_sft_audit(): assert SFT_POLICY_DISABLED_TOOLS <= set(REMAINING_TOOLS) def test_remaining_tool_policy_disabled_list_documents_sft_workspace_gap(): assert "bash" not in SFT_POLICY_DISABLED_TOOLS for tool in ("python", "read_file", "write_file", "edit_file", "apply_patch"): assert tool in SFT_POLICY_DISABLED_TOOLS def test_related_flow_shape_rejects_failed_to_tool_output(): from scripts.odysseus_related_flow_audit import _flow_has_good_training_shape history = { "history": [ {"role": "user", "content": "Send it"}, { "role": "assistant", "content": "Done", "metadata": { "tool_events": [{ "tool": "send_to_session", "output": "Failed to send to session: provider rejected credentials", }] }, }, ] } ok, reasons = _flow_has_good_training_shape(history, 1) assert ok is False assert any("failed tool output" in reason for reason in reasons) def test_related_flow_shape_rejects_unconfigured_teacher_output(): history = { "history": [ {"role": "user", "content": "Ask the teacher"}, { "role": "assistant", "content": "The teacher is unavailable.", "metadata": { "tool_events": [{ "tool": "ask_teacher", "output": "No teacher model configured. Specify a model name.", "exit_code": None, }] }, }, ] } ok, reasons = _flow_has_good_training_shape(history, 1) assert ok is False assert reasons == ["turn 1 has failed tool output from ask_teacher"]