jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-02 21:12:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

Squash Odysseus development history

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-09-11 06:04:19 +00:00
parent e5c99a5eee
commit 6ee6502010
2050 changed files with 538359 additions and 57745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/eval_odysseus_crud.py
+925
View File
@@ -0,0 +1,925 @@
#!/usr/bin/env python3
"""Exercise disposable CRUD workflows through the real Odysseus chat route.
The model must choose and execute the tools. This runner never mutates the
database directly: each fixture is uniquely tagged, and cleanup is requested
through the model before the final verification turn.
"""
from __future__ import annotations
import argparse
import contextlib
import json
import signal
import time
import uuid
from pathlib import Path
import httpx
def cookie(path: Path) -> str:
sessions = json.loads(path.read_text())
now = time.time()
for token, row in sessions.items():
if row.get("username") == "pewds" and row.get("expiry", 0) > now:
return token
raise RuntimeError("No valid pewds Odysseus session cookie found")
def events(response: httpx.Response):
for line in response.iter_lines():
if not line.startswith("data: "):
continue
payload = line[6:]
if payload == "[DONE]":
continue
try:
yield json.loads(payload)
except json.JSONDecodeError:
continue
def tool_ok(name: str | None, expected: set[str]) -> bool:
aliases = {
"mcp__contacts__manage_contact": "manage_contact",
"mcp__email__list_emails": "list_emails",
}
return aliases.get(name, name) in expected
def output_ok(event: dict) -> bool:
if event.get("exit_code") not in (0, None):
return False
output = event.get("output")
return not isinstance(output, str) or not output.lstrip().lower().startswith("error")
def visible_event_text(event: dict) -> str:
"""Collect text from both streaming deltas and replacement final events."""
if isinstance(event.get("delta"), str):
return event["delta"]
if event.get("type") == "final_response" and isinstance(event.get("content"), str):
return event["content"]
return ""
def approval_from_event(event: dict) -> dict | None:
"""Extract an opaque exact-approval payload from any SSE wrapper."""
candidates = [event, event.get("data"), event.get("ask_user")]
for candidate in candidates:
if not isinstance(candidate, dict):
continue
approval = candidate.get("ask_user") if isinstance(candidate.get("ask_user"), dict) else candidate
if (
isinstance(approval, dict)
and approval.get("kind") == "tool_approval"
and approval.get("approval_id")
):
return approval
return None
@contextlib.contextmanager
def hard_timeout(seconds: float | None, label: str):
if not seconds or seconds <= 0:
yield
return
def _raise_timeout(signum, frame): # type: ignore[no-untyped-def]
raise TimeoutError(f"{label} exceeded hard timeout {seconds}s")
previous = signal.signal(signal.SIGALRM, _raise_timeout)
signal.setitimer(signal.ITIMER_REAL, seconds)
try:
yield
finally:
signal.setitimer(signal.ITIMER_REAL, 0)
signal.signal(signal.SIGALRM, previous)
def parse_command(command: str | None) -> tuple[str, dict | str | None]:
if not command:
return "", None
try:
parsed = json.loads(command)
except json.JSONDecodeError:
parsed = command
if isinstance(parsed, dict):
return str(parsed.get("action") or ""), parsed
if isinstance(parsed, str):
return parsed.strip().splitlines()[0] if parsed.strip() else "", parsed
return "", parsed
def build_summary(records: list[dict], model: str, tag: str) -> dict:
"""Build the same scorecard for complete and checkpointed eval runs."""
turns = [turn for workflow in records for turn in workflow.get("turns", [])]
return {
"model": model,
"tag": tag,
"workflows": len(records),
"turns": len(turns),
"native_success": sum(bool(turn.get("native_call_ok")) for turn in turns),
"first_action_success": sum(bool(turn.get("first_action_ok")) for turn in turns),
"tool_count_success": sum(bool(turn.get("tool_count_ok")) for turn in turns),
"exact_arg_success": sum(bool(turn.get("exact_args_ok", True)) for turn in turns),
"exact_arg_checked": sum(bool(turn.get("expected_exact_args")) for turn in turns),
"execution_success": sum(bool(turn.get("execution_ok")) for turn in turns),
"cleanup_or_verify_turns": sum(
bool(turn.get("native_call_ok")) and bool(turn.get("execution_ok"))
for turn in turns
if turn.get("cleanup_or_verify_turn")
),
"duplicate_textual_calls": sum(bool(turn.get("duplicate_textual_call")) for turn in turns),
"stream_errors": sum(bool(turn["stream_errors"]) for turn in turns),
"records": records,
}
def write_checkpoint(output: Path, records: list[dict], model: str, tag: str) -> None:
"""Persist progress atomically after every completed turn.
A hard timeout, killed terminal, or backend restart should leave a usable
scorecard instead of an empty/missing result file. The temporary sibling is
replaced only after the JSON has been fully written.
"""
checkpoint = output.with_name(output.name + ".tmp")
checkpoint.write_text(
json.dumps(build_summary(records, model, tag), indent=2, ensure_ascii=True) + "\n"
)
checkpoint.replace(output)
def infra_record(message: str, expected: set[str], exc: BaseException, cleanup: bool = False) -> dict:
return {
"message": message,
"expected_tools": sorted(expected),
"expected_first_action": None,
"max_tool_calls": None,
"expected_exact_args": {},
"tools": [],
"tool_events": [],
"approval_tool_events": [],
"first_action": "",
"native_call_ok": False,
"first_action_ok": False,
"tool_count_ok": False,
"exact_args_ok": False,
"exact_arg_failures": [
{
"field": "*",
"expected": "turn could run",
"actual": repr(exc),
}
],
"approval_required": False,
"execution_ok": False,
"duplicate_textual_call": False,
"stream_errors": [{"type": "infra_exception", "error": repr(exc)}],
"tool_outputs": [],
"response": "",
"elapsed_seconds": 0,
"approval_turns": 0,
"cleanup_or_verify_turn": cleanup,
"infra_failure": True,
}
def turn(
client: httpx.Client,
args,
session_id: str,
message: str,
expected: set[str],
expected_first_action: str | tuple[str, ...] | None = None,
max_tool_calls: int | None = None,
expected_exact_args: dict[str, str] | None = None,
) -> dict:
started = time.monotonic()
captured = []
text = []
stream_exception = None
approval_turns = 0
turn_data = {
"message": message,
"session": session_id,
"mode": "agent",
"agent_prompt_mode": args.prompt_mode,
**({"selected_endpoint_id": args.endpoint_id} if args.endpoint_id else {}),
**({"selected_endpoint_url": args.selected_endpoint_url} if args.selected_endpoint_url else {}),
**({"selected_model": args.selected_model} if args.selected_model else {}),
}
try:
with hard_timeout(args.hard_turn_timeout, message[:80]):
while True:
approval = None
with client.stream(
"POST",
args.base_url.rstrip("/") + "/api/chat_stream",
data=turn_data,
headers={"Accept": "text/event-stream"},
timeout=args.timeout,
) as response:
response.raise_for_status()
for event in events(response):
captured.append(event)
approval = approval or approval_from_event(event)
visible_text = visible_event_text(event)
if visible_text:
if event.get("type") == "final_response":
# A continuation can replace the approval
# draft from the previous HTTP stream. Keep
# the evaluator's response metric aligned
# with the TUI/client rendering contract.
text[:] = [visible_text]
else:
text.append(visible_text)
if not getattr(args, "auto_approve", True) or not approval or approval_turns >= 3:
break
approval_turns += 1
turn_data = {
**turn_data,
"tool_approval_id": approval["approval_id"],
"tool_approval_decision": "approve",
}
except Exception as exc:
stream_exception = repr(exc)
starts = [e for e in captured if e.get("type") == "tool_start"]
outputs = [e for e in captured if e.get("type") == "tool_output"]
doc_updates = [e for e in captured if e.get("type") == "doc_update"]
errors = [e for e in captured if e.get("type") == "error"]
metric_events = [e for e in captured if e.get("type") == "metrics"]
latest_metrics = (metric_events[-1].get("data") or {}) if metric_events else {}
model_request_snapshots = [
{
key: event.get(key)
for key in (
"round",
"model",
"messages",
"tools",
"temperature",
"max_tokens",
"prompt_type",
"agent_prompt_mode",
)
}
for event in captured
if event.get("type") == "model_request_snapshot"
]
metrics_round_texts = [
str(item)[:2000]
for item in (latest_metrics.get("round_texts") or [])
if str(item).strip()
]
event_types = [str(e.get("type") or "") for e in captured]
if stream_exception:
errors.append({"type": "client_exception", "error": stream_exception})
rendered = "".join(text).strip()
if not rendered:
if metric_events:
rendered = next(
(str(item).strip() for item in reversed(metrics_round_texts) if str(item).strip()),
"",
)
tool_events = []
for idx, event in enumerate(starts):
command = event.get("command")
action, parsed = parse_command(command)
tool_events.append(
{
"index": idx,
"tool": event.get("tool"),
"command": command,
"action": action,
"parsed_command": parsed,
}
)
approval_events = []
if not tool_events:
for idx, event in enumerate(outputs):
ask_user = event.get("ask_user")
action_payload = ask_user.get("action") if isinstance(ask_user, dict) else None
if not isinstance(action_payload, dict):
continue
command = action_payload.get("content")
action, parsed = parse_command(command)
approval_events.append(
{
"index": idx,
"tool": action_payload.get("tool") or event.get("tool"),
"command": command,
"action": action,
"parsed_command": parsed,
"approval_required": True,
}
)
if approval_events:
tool_events = approval_events
rendered_lower = rendered.lower()
duplicate = any(
marker in rendered_lower
for marker in (
"manage_notes(",
"manage_calendar(",
"manage_memory(",
"manage_contact(",
'"function"',
"function=",
"<function",
"</function",
"parameter=",
"<parameter",
"</parameter",
"mcp__email__",
)
)
expected_actions = (
list(expected_first_action)
if isinstance(expected_first_action, tuple)
else expected_first_action
)
first_action_ok = expected_first_action is None or (
bool(tool_events)
and (
tool_events[0]["action"] == expected_first_action
if isinstance(expected_first_action, str)
else tool_events[0]["action"] in expected_first_action
)
)
exact_arg_failures = []
expected_exact_args = dict(expected_exact_args or {})
expected_arg_spec = dict(expected_exact_args)
required_executed_actions = expected_exact_args.pop("__actions_include", [])
if isinstance(required_executed_actions, str):
required_executed_actions = [required_executed_actions]
observed_actions = [str(event.get("action") or "") for event in tool_events]
missing_required_actions = [
action for action in required_executed_actions if action not in observed_actions
]
for missing_action in missing_required_actions:
exact_arg_failures.append(
{
"field": "__actions_include",
"expected": missing_action,
"actual": observed_actions,
}
)
parsed_first = tool_events[0].get("parsed_command") if tool_events else None
raw_first_command = tool_events[0].get("command") if tool_events else ""
required_command_substrings = expected_exact_args.pop("__command_contains", [])
if isinstance(required_command_substrings, str):
required_command_substrings = [required_command_substrings]
for expected_substring in required_command_substrings:
if str(expected_substring) not in str(raw_first_command or ""):
exact_arg_failures.append(
{
"field": "__command_contains",
"expected": expected_substring,
"actual": raw_first_command,
}
)
required_state_substrings = expected_exact_args.pop("__state_contains", [])
if isinstance(required_state_substrings, str):
required_state_substrings = [required_state_substrings]
state_parts = [str(raw_first_command or ""), rendered]
for event in outputs:
for key in (
"output",
"document_title",
"document_language",
"document_content",
"doc_id",
):
value = event.get(key)
if value is not None:
state_parts.append(str(value))
for event in doc_updates:
for key in ("title", "language", "content", "doc_id", "version"):
value = event.get(key)
if value is not None:
state_parts.append(str(value))
state_text = "\n".join(state_parts)
for expected_substring in required_state_substrings:
if str(expected_substring) not in state_text:
exact_arg_failures.append(
{
"field": "__state_contains",
"expected": expected_substring,
"actual": state_text[:1200],
}
)
if isinstance(parsed_first, dict):
for key, expected_value in expected_exact_args.items():
actual_value = parsed_first.get(key)
if actual_value != expected_value:
exact_arg_failures.append(
{
"field": key,
"expected": expected_value,
"actual": actual_value,
}
)
elif expected_exact_args:
exact_arg_failures.append(
{
"field": "*",
"expected": expected_exact_args,
"actual": parsed_first,
}
)
return {
"message": message,
"expected_tools": sorted(expected),
"expected_first_action": expected_actions,
"max_tool_calls": max_tool_calls,
"expected_exact_args": expected_arg_spec,
"tools": [e.get("tool") for e in starts],
"tool_events": tool_events,
"approval_tool_events": approval_events,
"first_action": tool_events[0]["action"] if tool_events else "",
"native_call_ok": bool(tool_events) and any(tool_ok(e.get("tool"), expected) for e in tool_events),
"first_action_ok": first_action_ok,
"tool_count_ok": max_tool_calls is None or len(tool_events) <= max_tool_calls,
"exact_args_ok": not exact_arg_failures,
"exact_arg_failures": exact_arg_failures,
"approval_required": bool(approval_events),
"execution_ok": (
bool(outputs)
and not approval_events
and all(output_ok(e) for e in outputs)
and not missing_required_actions
),
"duplicate_textual_call": duplicate,
"stream_errors": errors,
"tool_outputs": [
{
"tool": e.get("tool"),
"exit_code": e.get("exit_code"),
"output": str(e.get("output", ""))[:500],
**(
{
"doc_id": e.get("doc_id"),
"document_action": e.get("document_action"),
"document_title": e.get("document_title"),
"document_language": e.get("document_language"),
"document_version": e.get("document_version"),
"document_content": str(e.get("document_content", ""))[:1200],
}
if e.get("doc_id") or e.get("document_content")
else {}
),
**({"ask_user_action": e.get("ask_user", {}).get("action")} if isinstance(e.get("ask_user"), dict) else {}),
}
for e in outputs
],
"doc_updates": [
{
"doc_id": e.get("doc_id"),
"title": e.get("title"),
"language": e.get("language"),
"version": e.get("version"),
"content": str(e.get("content", ""))[:1200],
}
for e in doc_updates
],
"response": rendered.strip()[:1200],
"metrics_round_texts": metrics_round_texts[-6:],
"metrics_tool_calls": latest_metrics.get("tool_calls"),
"model_request_snapshots": model_request_snapshots,
"captured_event_types": event_types,
"elapsed_seconds": round(time.monotonic() - started, 3),
"approval_turns": approval_turns,
}
def workflow(name: str, tag: str) -> list[tuple[str, set[str], bool, str | tuple[str, ...] | None, int | None, dict[str, str]]]:
"""Return prompt, expected tools, cleanup marker, expected action, max calls, exact args."""
if name == "notes":
return [
(
f"Create a temporary normal note titled {tag} with content 'temporary fixture'.",
{"manage_notes"},
False,
"add",
1,
{"title": tag, "content": "temporary fixture"},
),
# Title-based mutations may resolve the title first; require the
# corresponding mutation to execute and allow that bounded pair.
(
f"Update the exact note titled {tag} so its content is 'updated fixture'.",
{"manage_notes"},
False,
"update",
1,
{"title": tag, "content": "updated fixture"},
),
(
f"Delete the exact temporary note titled {tag}. Use the title directly; do not search first.",
{"manage_notes"},
True,
"delete",
1,
{"title": tag},
),
(
f"Verify that the note titled {tag} no longer exists. Search for the exact title; do not create anything.",
{"manage_notes"},
True,
"search",
1,
{"title": tag},
),
]
if name == "calendar":
return [
(
f"Create one temporary calendar event titled {tag} on 2030-01-02 from 10:00 to 11:00, description 'temporary fixture'.",
{"manage_calendar"},
False,
"create_event",
None,
{"summary": tag, "description": "temporary fixture"},
),
(
f"Update the exact calendar event titled {tag}; change its location to 'Updated fixture location'. Use the exact title as the identifier.",
{"manage_calendar"},
False,
"update_event",
1,
{"summary": tag, "location": "Updated fixture location"},
),
(
f"Delete only the temporary calendar event titled {tag}. Use the exact title as the identifier.",
{"manage_calendar"},
True,
"delete_event",
1,
{"summary": tag},
),
(
f"Verify that calendar event {tag} is absent. Search the 2030-01-02 range; do not create anything.",
{"manage_calendar"},
True,
"list_events",
1,
{"start": "2030-01-02", "end": "2030-01-03", "query": tag},
),
]
if name == "memory":
return [
(
f"Add one temporary saved memory with exact marker {tag} and text 'temporary fixture'; category fact.",
{"manage_memory"},
False,
"add",
1,
{"__command_contains": [tag, "temporary fixture", "fact"]},
),
(
f"Search saved memory for the exact marker {tag}.",
{"manage_memory"},
False,
"search",
1,
{"__command_contains": tag},
),
(
f"Delete only the temporary memory containing exact marker {tag}. Search first and use its memory_id.",
{"manage_memory"},
True,
None,
None,
{"__command_contains": tag, "__actions_include": "delete"},
),
(
f"Verify that no saved memory containing exact marker {tag} remains. Search only; do not add anything.",
{"manage_memory"},
True,
"search",
1,
{"__command_contains": tag},
),
]
if name == "documents":
return [
(
f"Create a temporary editor document titled {tag} with exactly this short content: temporary fixture.",
{"create_document"},
False,
None,
1,
{"__state_contains": [tag, "temporary fixture"]},
),
(
f"Edit the active document {tag}: replace 'temporary fixture' with 'updated fixture'. Use the document edit tool.",
{"edit_document", "update_document"},
False,
None,
1,
{"__state_contains": ["updated fixture"]},
),
(
f"Delete only the editor document titled {tag}. Find its document id if needed, then use the document management delete action.",
{"manage_documents"},
True,
("list", "delete"),
None,
{"__state_contains": tag, "__actions_include": "delete"},
),
(
f"Verify that editor document {tag} no longer exists by searching documents. Do not create anything.",
{"manage_documents"},
True,
"list",
1,
{"__command_contains": tag},
),
]
if name == "contacts":
return [
(
f"Add one temporary fake contact named {tag}, email {tag.lower()}@invalid.example, phone +1-202-555-0199.",
{"manage_contact"},
False,
"add",
1,
{
"name": tag,
"email": f"{tag.lower()}@invalid.example",
"__command_contains": "+1-202-555-0199",
},
),
(
f"Update the exact contact named {tag}; change the phone to +1-202-555-0188.",
{"manage_contact"},
False,
"update",
None,
{"__command_contains": [tag, "+1-202-555-0188"]},
),
(
f"Delete only the fake contact named {tag}. List/search first to get its UID, then delete it.",
{"manage_contact"},
True,
None,
None,
{"__command_contains": tag, "__actions_include": "delete"},
),
(
f"Verify that contact {tag} is absent. Search contacts for the exact name; do not change any other contact.",
{"manage_contact"},
True,
"search",
1,
{"__command_contains": tag},
),
]
if name == "tasks":
return [
(
f"Create one disposable scheduled task named {tag} that runs daily at 23:59 UTC and prompts exactly 'temporary fixture'. Use task_type llm and output_target session.",
{"manage_tasks"},
False,
"create",
1,
{
"action": "create",
"name": tag,
"prompt": "temporary fixture",
"task_type": "llm",
"schedule": "daily",
"scheduled_time": "23:59",
"output_target": "session",
},
),
(
f"Pause only the disposable scheduled task named {tag}. List/search first if needed to get its task_id.",
{"manage_tasks"},
False,
None,
None,
{"__command_contains": tag, "__actions_include": "pause"},
),
(
f"Resume only the disposable scheduled task named {tag}. List/search first if needed to get its task_id.",
{"manage_tasks"},
False,
None,
None,
{"__command_contains": tag, "__actions_include": "resume"},
),
(
f"Delete only the disposable scheduled task named {tag}. List/search first if needed to get its task_id.",
{"manage_tasks"},
True,
None,
None,
{"__command_contains": tag, "__actions_include": "delete"},
),
(
f"Verify that scheduled task {tag} is absent. List/search tasks for the exact name; do not create anything.",
{"manage_tasks"},
True,
"list",
1,
{"__command_contains": tag},
),
]
if name == "skills":
return [
(
f"Add one disposable draft skill named {tag.lower()} with description 'temporary fixture', procedure ['do nothing'], verification ['confirm fixture'], status draft.",
{"manage_skills"},
False,
"add",
1,
{
"name": tag.lower(),
"description": "temporary fixture",
"__command_contains": ["do nothing", "confirm fixture", "draft"],
},
),
(
f"View the disposable draft skill named {tag.lower()} and confirm it exists.",
{"manage_skills"},
False,
"view",
1,
{"__command_contains": tag.lower()},
),
(
f"Delete only the disposable draft skill named {tag.lower()}.",
{"manage_skills"},
True,
"delete",
1,
{"__command_contains": tag.lower()},
),
(
f"Verify that disposable skill {tag.lower()} is absent by searching/listing skills. Do not create anything.",
{"manage_skills"},
True,
("list", "search"),
1,
{"__command_contains": tag.lower()},
),
]
raise ValueError(name)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--workflow", action="append", choices=["notes", "calendar", "memory", "documents", "contacts", "skills", "tasks"])
parser.add_argument("--base-url", default="http://127.0.0.1:7011")
parser.add_argument("--endpoint", default="http://192.168.1.21:8065/v1/chat/completions")
parser.add_argument("--endpoint-id", default="82e5463e")
parser.add_argument("--model", default="/Users/pewds/models/qwen36-27b-mlx-8bit")
parser.add_argument("--selected-endpoint-url", default="")
parser.add_argument("--selected-model", default="")
parser.add_argument("--cookie-file", default="data/sessions.json")
parser.add_argument("--output", required=True)
parser.add_argument("--prompt-mode", default="auto")
parser.add_argument("--timeout", type=float, default=240)
parser.add_argument("--hard-turn-timeout", type=float, default=0)
parser.add_argument(
"--no-auto-approve",
dest="auto_approve",
action="store_false",
help="Stop at the first exact approval instead of continuing it.",
)
parser.add_argument(
"--independent-turns",
action="store_true",
help="Create a fresh session for each turn. Useful for no-approve proposal-accuracy checks where prior unexecuted approvals would contaminate history.",
)
args = parser.parse_args()
workflows = args.workflow or ["notes", "calendar", "memory", "documents", "contacts", "skills"]
tag = "ODY-EVAL-CRUD-" + time.strftime("%Y%m%d-%H%M%S") + "-" + uuid.uuid4().hex[:8]
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
client = httpx.Client(cookies={"odysseus_session": cookie(Path(args.cookie_file))}, follow_redirects=False)
records = []
try:
for name in workflows:
workflow_records = []
previous = None
session_id = None
try:
if not args.independent_turns:
try:
create = client.post(
args.base_url.rstrip("/") + "/api/session",
data={
"name": f"[eval-crud] {name} {tag}",
"endpoint_url": args.endpoint,
"model": args.model,
"skip_validation": "true",
"rag": "false",
**({"endpoint_id": args.endpoint_id} if args.endpoint_id else {}),
},
timeout=30,
)
create.raise_for_status()
session_id = create.json()["id"]
except Exception as exc:
record = infra_record(
f"Create session for workflow {name}",
set(),
exc,
)
workflow_records.append(record)
print(json.dumps({"workflow": name, **record}, ensure_ascii=True), flush=True)
continue
for turn_index, (prompt, expected, cleanup, expected_action, max_calls, exact_args) in enumerate(workflow(name, tag), start=1):
if args.independent_turns:
try:
create = client.post(
args.base_url.rstrip("/") + "/api/session",
data={
"name": f"[eval-crud] {name} {tag} turn {turn_index}",
"endpoint_url": args.endpoint,
"model": args.model,
"skip_validation": "true",
"rag": "false",
**({"endpoint_id": args.endpoint_id} if args.endpoint_id else {}),
},
timeout=30,
)
create.raise_for_status()
session_id = create.json()["id"]
except Exception as exc:
record = infra_record(prompt, expected, exc, cleanup)
workflow_records.append(record)
print(json.dumps({"workflow": name, **record}, ensure_ascii=True), flush=True)
break
# A fuzzy memory search must never authorize deletion of an
# unrelated record. Require the unique marker to appear in
# the search result before allowing the delete turn.
if (
name == "memory"
and "Delete only the temporary memory" in prompt
and previous is not None
and tag not in " ".join(
item.get("output", "") for item in previous.get("tool_outputs", [])
)
):
record = {
"message": prompt,
"expected_tools": sorted(expected),
"tools": [],
"native_call_ok": False,
"execution_ok": False,
"duplicate_textual_call": False,
"stream_errors": [],
"tool_outputs": [],
"response": "BLOCKED: preceding memory search did not return the unique fixture marker",
"elapsed_seconds": 0,
"cleanup_or_verify_turn": cleanup,
"blocked_by_safety_guard": True,
}
workflow_records.append(record)
print(json.dumps({"workflow": name, **record}, ensure_ascii=True), flush=True)
break
record = turn(
client,
args,
session_id,
prompt,
expected,
expected_action,
max_calls,
exact_args,
)
record["cleanup_or_verify_turn"] = cleanup
record["independent_turn"] = bool(args.independent_turns)
workflow_records.append(record)
previous = record
print(json.dumps({"workflow": name, **record}, ensure_ascii=True), flush=True)
if args.independent_turns and session_id:
try:
client.delete(args.base_url.rstrip("/") + f"/api/session/{session_id}", timeout=15)
except Exception:
pass
session_id = None
finally:
if session_id:
try:
client.delete(args.base_url.rstrip("/") + f"/api/session/{session_id}", timeout=15)
except Exception:
pass
records.append({"workflow": name, "tag": tag, "turns": workflow_records})
write_checkpoint(output, records, args.model, tag)
finally:
client.close()
summary = build_summary(records, args.model, tag)
write_checkpoint(output, records, args.model, tag)
print("SUMMARY", json.dumps({k: summary[k] for k in summary if k != "records"}))
if __name__ == "__main__":
main()