#!/usr/bin/env python3 """Multi-turn Odysseus tool-use eval for contextual follow-up behavior.""" from __future__ import annotations import argparse import contextlib import json import os import re import time import uuid from pathlib import Path from typing import Any import httpx try: from scripts.eval_odysseus_tool_use import ( _cookie, _raise_for_status_with_body, _sse_events, _tool_approval_from_event, _visible_event_text, ) except ModuleNotFoundError: from eval_odysseus_tool_use import ( _cookie, _raise_for_status_with_body, _sse_events, _tool_approval_from_event, _visible_event_text, ) SCENARIOS: list[dict[str, Any]] = [ { "scenario": "public_domain_art_links_followup", "turns": [ { "message": "What are some good sites for public domain art?", "expected_tool": "no_tool", "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"], }, { "message": "send links", "expected_tool": "web_search", "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"], }, ], }, { "scenario": "notes_then_identity_boundary", "turns": [ { "message": "what are my notes?", "expected_tool": "manage_notes", "expected_action": "list", "required_any": ["note", "[", "test scenario"], }, { "message": "who are you?", "expected_tool": "no_tool", "required_any": ["odysseus", "assistant"], }, ], }, { "scenario": "email_followup_search", "turns": [ { "message": "what is my latest email?", "expected_tool": "mcp__email__list_emails", "required_any": ["email", "from", "subject", "latest"], }, { "message": "find emails from Runpod instead", "expected_tool": "mcp__email__search_emails", "required_any": ["runpod", "email", "no emails"], }, ], }, { "scenario": "calendar_then_general_fact", "turns": [ { "message": "what is on my calendar?", "expected_tool": "manage_calendar", "expected_action": "list_events", "required_any": ["event", "calendar", "found", "no events"], }, { "message": "what does VAT stand for?", "expected_tool": "no_tool", "required_any": ["value-added tax", "value added tax"], }, ], }, { "scenario": "public_domain_art_typo_links_followup", "turns": [ { "message": "What are some good sites for public domain art?", "expected_tool": "no_tool", "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"], }, { "message": "sned links for those", "expected_tool": "web_search", "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"], }, ], }, { "scenario": "notes_then_calendar_switch", "turns": [ { "message": "what are my notes?", "expected_tool": "manage_notes", "expected_action": "list", "required_any": ["note", "[", "test scenario"], }, { "message": "what is on my calendar next week?", "expected_tool": "manage_calendar", "expected_action": "list_events", "required_any": ["event", "calendar", "found", "no events"], }, ], }, { "scenario": "email_then_ambiguous_links_clarify", "turns": [ { "message": "what is my latest email?", "expected_tool": "mcp__email__list_emails", "required_any": ["email", "from", "subject", "latest"], }, { "message": "send links", "expected_tool": "no_tool", "required_any": ["which links", "what links", "clarify", "what topic", "which topic"], }, ], }, { "scenario": "web_answer_then_calendar_boundary", "turns": [ { "message": "What are some good sites for public domain art?", "expected_tool": "no_tool", "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"], }, { "message": "what is on my calendar?", "expected_tool": "manage_calendar", "expected_action": "list_events", "required_any": ["event", "calendar", "found", "no events"], }, ], }, { "scenario": "public_domain_art_sites_tail_followup", "turns": [ { "message": "What are some good sites for public domain art?", "expected_tool": "no_tool", "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"], }, { "message": "send links for the sites", "expected_tool": "web_search", "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"], }, ], }, { "scenario": "public_domain_art_typo_bare_links_followup", "turns": [ { "message": "What are some good sites for public domain art?", "expected_tool": "no_tool", "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"], }, { "message": "sned links", "expected_tool": "web_search", "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"], }, ], }, { "scenario": "public_domain_art_bare_websites_followup", "turns": [ { "message": "What are some good sites for public domain art?", "expected_tool": "no_tool", "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"], }, { "message": "for the websites", "expected_tool": "web_search", "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"], }, ], }, { "scenario": "email_then_typo_links_tail_clarify", "turns": [ { "message": "what is my latest email?", "expected_tool": "mcp__email__list_emails", "required_any": ["email", "from", "subject", "latest"], }, { "message": "sned links for those", "expected_tool": "no_tool", "required_any": ["which links", "what links", "clarify", "what topic", "which topic", "topic"], }, ], }, { "scenario": "email_then_bare_websites_clarify", "turns": [ { "message": "what is my latest email?", "expected_tool": "mcp__email__list_emails", "required_any": ["email", "from", "subject", "latest"], }, { "message": "for the websites", "expected_tool": "no_tool", "required_any": ["which links", "what links", "clarify", "what topic", "which topic", "topic", "website"], }, ], }, { "scenario": "notes_then_typo_links_tail_clarify", "turns": [ { "message": "what are my notes?", "expected_tool": "manage_notes", "expected_action": "list", "required_any": ["note", "[", "test scenario"], }, { "message": "sned links for those", "expected_tool": "no_tool", "required_any": ["which links", "what links", "clarify", "what topic", "which topic", "topic"], }, ], }, { "scenario": "notes_crud_followthrough", "fixture_prefix": "ODY-EVAL-CRUD-NOTES-", "turns": [ { "message": "Create a note titled ODY-EVAL-CRUD-NOTES-FLOW with content alpha checkpoint.", "expected_tool": "manage_notes", "expected_actions": ["add", "create"], "required_all": ["created", "ody-eval-crud-notes-flow"], "max_tool_count": 1, }, { "message": "Update that note so its content says beta checkpoint.", "expected_tool": "manage_notes", "expected_action": "update", "required_all": ["updated", "note"], "max_tool_count": 1, }, { "message": "Delete that note.", "expected_tool": "manage_notes", "expected_action": "delete", "required_all": ["deleted", "note"], "max_tool_count": 1, }, ], }, { "scenario": "calendar_crud_followthrough", "fixture_prefix": "ODY-EVAL-CRUD-CALENDAR-", "turns": [ { "message": ( "Create a calendar event titled ODY-EVAL-CRUD-CALENDAR-FLOW " "on 2026-08-25 from 10:00 to 10:30 at Test Lab." ), "expected_tool": "manage_calendar", "expected_actions": ["create_event", "create"], "required_all": ["created", "event", "ody-eval-crud-calendar-flow"], "max_tool_count": 1, }, { "message": "Update that calendar event location to Blue Room.", "expected_tool": "manage_calendar", "expected_actions": ["update_event", "update"], "required_all": ["updated", "event"], "max_tool_count": 1, }, { "message": "Delete that calendar event.", "expected_tool": "manage_calendar", "expected_actions": ["delete_event", "delete"], "required_all": ["deleted", "event"], "max_tool_count": 1, }, ], }, { "scenario": "memory_crud_followthrough", "fixture_prefix": "ODY-EVAL-CRUD-MEMORY-", "turns": [ { "message": "Remember this temporary eval fact: ODY-EVAL-CRUD-MEMORY-FLOW alpha checkpoint.", "expected_tool": "manage_memory", "expected_action": "add", "required_all": ["memory", "added"], "max_tool_count": 1, }, { "message": "Update that memory to say ODY-EVAL-CRUD-MEMORY-FLOW beta checkpoint.", "expected_tool": "manage_memory", "expected_action": "edit", "required_all": ["memory", "updated"], "max_tool_count": 1, }, { "message": "Delete that memory.", "expected_tool": "manage_memory", "expected_action": "delete", "required_all": ["memory", "deleted"], "max_tool_count": 1, }, ], }, { "scenario": "memory_add_one_call_efficiency", "fixture_prefix": "ODY-EVAL-CRUD-MEMORY-", "turns": [ { "message": "Remember this temporary eval fact: ODY-EVAL-CRUD-MEMORY-ONECALL alpha checkpoint.", "expected_tool": "manage_memory", "expected_action": "add", "required_all": ["memory", "added"], "max_tool_count": 1, }, { "message": "Delete that memory.", "expected_tool": "manage_memory", "expected_action": "delete", "required_all": ["memory", "deleted"], "max_tool_count": 1, }, ], }, { "scenario": "memory_add_wording_variants_efficiency", "fixture_prefix": "ODY-EVAL-CRUD-MEMORY-", "turns": [ { "message": "Save this as a memory: ODY-EVAL-CRUD-MEMORY-VAR-A alpha checkpoint.", "expected_tool": "manage_memory", "expected_action": "add", "required_all": ["memory", "added"], "max_tool_count": 1, }, { "message": "Delete that memory.", "expected_tool": "manage_memory", "expected_action": "delete", "required_all": ["memory", "deleted"], "max_tool_count": 1, }, { "message": "Add to memory that ODY-EVAL-CRUD-MEMORY-VAR-B beta checkpoint is temporary.", "expected_tool": "manage_memory", "expected_action": "add", "required_all": ["memory", "added"], "max_tool_count": 1, }, { "message": "Remove that memory.", "expected_tool": "manage_memory", "expected_action": "delete", "required_all": ["memory", "deleted"], "max_tool_count": 1, }, { "message": "Please remember: ODY-EVAL-CRUD-MEMORY-VAR-C gamma checkpoint.", "expected_tool": "manage_memory", "expected_action": "add", "required_all": ["memory", "added"], "max_tool_count": 1, }, { "message": "Forget that memory.", "expected_tool": "manage_memory", "expected_action": "delete", "required_all": ["memory", "deleted"], "max_tool_count": 1, }, ], }, { "scenario": "memory_no_tool_boundary", "turns": [ { "message": "do you remember what VAT stands for?", "expected_tool": "no_tool", "required_any": ["value-added tax", "value added tax"], }, { "message": "what should I remember before buying public domain art?", "expected_tool": "no_tool", "required_any": ["license", "copyright", "public domain", "source"], }, { "message": "remind me what Sweden is bordered by", "expected_tool": "no_tool", "required_any": ["norway", "finland"], }, { "message": "what does it mean to remember something in a computer?", "expected_tool": "no_tool", "required_any": ["store", "storage", "memory", "data", "information"], }, ], }, { "scenario": "tasks_crud_followthrough", "fixture_prefix": "ODY-EVAL-CRUD-TASKS-", "turns": [ { "message": ( "Create a scheduled task named ODY-EVAL-CRUD-TASKS-FLOW that runs daily at 09:00 UTC " "and has prompt alpha checkpoint." ), "expected_tool": "manage_tasks", "expected_action": "create", "required_all": ["created", "task", "ody-eval-crud-tasks-flow"], "max_tool_count": 1, }, { "message": "Update that task prompt to beta checkpoint.", "expected_tool": "manage_tasks", "expected_action": "edit", "required_all": ["updated", "task"], "max_tool_count": 1, }, { "message": "Delete that task.", "expected_tool": "manage_tasks", "expected_action": "delete", "required_all": ["deleted", "task"], "max_tool_count": 1, }, ], }, { "scenario": "documents_create_delete_followthrough", "fixture_prefix": "ODY-EVAL-CRUD-DOCUMENTS-", "turns": [ { "message": ( "Create an editor document titled ODY-EVAL-CRUD-DOCUMENTS-FLOW " "with markdown content alpha checkpoint." ), "expected_tool": "create_document", "required_all": ["document", "ody-eval-crud-documents-flow"], "max_tool_count": 1, }, { "message": "Delete that document.", "expected_tool": "manage_documents", "expected_action": "delete", "required_all": ["deleted", "document"], "max_tool_count": 1, }, ], }, ] TOOL_ALIASES = { "mcp_email_list_emails": "mcp__email__list_emails", "mcp_email_search_emails": "mcp__email__search_emails", "list_emails": "mcp__email__list_emails", "search_emails": "mcp__email__search_emails", } def malformed_text_surface(response_text: str) -> bool: value = (response_text or "").lower() if any( marker in value for marker in ( " str | None: if not tool: return tool return TOOL_ALIASES.get(tool, tool) def parse_action(command: str | None) -> str: if not command: return "" try: parsed = json.loads(command) except json.JSONDecodeError: parsed = command if isinstance(parsed, dict): return str(parsed.get("action") or "") if isinstance(parsed, str): return parsed.strip().splitlines()[0] if parsed.strip() else "" return "" def _fixture_owner() -> str: return os.environ.get("ODY_EVAL_OWNER", "pewds") def _cleanup_crud_fixtures() -> None: """Remove only eval-owned CRUD artifacts created by this script.""" owner = _fixture_owner() try: from core.database import ( CalendarCal, CalendarEvent, Document, DocumentVersion, Note, ScheduledTask, SessionLocal, ) except Exception as exc: print(json.dumps({"cleanup_warning": f"database import failed: {exc!r}"}), flush=True) else: db = SessionLocal() try: notes_q = db.query(Note).filter(Note.title.like("ODY-EVAL-CRUD-%")) if owner: notes_q = notes_q.filter(Note.owner == owner) for note in notes_q.all(): db.delete(note) events_q = db.query(CalendarEvent).filter(CalendarEvent.summary.like("ODY-EVAL-CRUD-%")) if owner: events_q = events_q.join(CalendarCal, CalendarEvent.calendar_id == CalendarCal.id).filter( CalendarCal.owner == owner ) for event in events_q.all(): db.delete(event) cals_q = db.query(CalendarCal).filter(CalendarCal.name.like("ODY-EVAL-CRUD-%")) if owner: cals_q = cals_q.filter(CalendarCal.owner == owner) for calendar in cals_q.all(): db.delete(calendar) docs_q = db.query(Document).filter(Document.title.like("ODY-EVAL-CRUD-%")) if owner: docs_q = docs_q.filter(Document.owner == owner) for doc in docs_q.all(): db.query(DocumentVersion).filter(DocumentVersion.document_id == doc.id).delete() db.delete(doc) tasks_q = db.query(ScheduledTask).filter(ScheduledTask.name.like("ODY-EVAL-CRUD-%")) if owner: tasks_q = tasks_q.filter(ScheduledTask.owner == owner) tasks_q.delete(synchronize_session=False) db.commit() except Exception as exc: db.rollback() print(json.dumps({"cleanup_warning": repr(exc)}), flush=True) finally: db.close() try: from src.constants import MEMORY_FILE memory_path = Path(MEMORY_FILE) if memory_path.exists(): entries = json.loads(memory_path.read_text(encoding="utf-8")) if isinstance(entries, list): filtered = [ entry for entry in entries if not ( isinstance(entry, dict) and "ODY-EVAL-CRUD-MEMORY-" in str(entry.get("text") or "") and (not owner or entry.get("owner") == owner) ) ] if len(filtered) != len(entries): memory_path.write_text(json.dumps(filtered, indent=2, ensure_ascii=True) + "\n", encoding="utf-8") except Exception as exc: print(json.dumps({"cleanup_warning": f"memory cleanup failed: {exc!r}"}), flush=True) @contextlib.contextmanager def _crud_fixture_cleanup(enabled: bool): if enabled: _cleanup_crud_fixtures() try: yield finally: if enabled: _cleanup_crud_fixtures() def output_ok(event: dict[str, Any]) -> bool: if event.get("exit_code") not in (0, None): return False text = str(event.get("output") or "") return not text.lstrip().lower().startswith("error") def event_action(event: dict[str, Any]) -> str: return parse_action(str(event.get("command") or "")) def create_session(client: httpx.Client, args, name: str) -> str: response = client.post( args.base_url.rstrip("/") + "/api/session", data={ "name": name, "endpoint_url": args.selected_endpoint_url or args.endpoint, "model": args.selected_model or args.model, "skip_validation": "true", "rag": "false", **({"endpoint_id": args.endpoint_id} if args.endpoint_id else {}), }, timeout=30, ) _raise_for_status_with_body(response) return response.json()["id"] def run_turn(client: httpx.Client, args, session_id: str, spec: dict[str, Any]) -> dict[str, Any]: started = time.monotonic() events: list[dict[str, Any]] = [] text: list[str] = [] errors: list[dict[str, Any]] = [] approval_turns = 0 turn_data = { "message": spec["message"], "session": session_id, "mode": "agent", "agent_prompt_mode": args.prompt_mode, **({"selected_endpoint_id": args.endpoint_id} if args.endpoint_id else {}), **({"selected_endpoint_url": args.selected_endpoint_url} if args.selected_endpoint_url else {}), **({"selected_model": args.selected_model} if args.selected_model else {}), } try: while True: approval = None with client.stream( "POST", args.base_url.rstrip("/") + "/api/chat_stream", data=turn_data, headers={"Accept": "text/event-stream"}, timeout=args.timeout, ) as response: _raise_for_status_with_body(response) for event in _sse_events(response): events.append(event) if event.get("type") == "error": errors.append(event) visible = _visible_event_text(event) if visible: if event.get("type") == "final_response": text[:] = [visible] else: text.append(visible) approval = approval or _tool_approval_from_event(event) if not args.auto_approve or not approval or approval_turns >= 3: break approval_turns += 1 turn_data = { **turn_data, "tool_approval_id": approval["approval_id"], "tool_approval_decision": "approve", } except Exception as exc: errors.append({"type": "client_exception", "error": repr(exc)}) starts = [event for event in events if event.get("type") == "tool_start"] outputs = [event for event in events if event.get("type") == "tool_output"] metrics = [ event.get("data") for event in events if event.get("type") == "metrics" and isinstance(event.get("data"), dict) ] snapshots = [ { key: event.get(key) for key in ( "round", "model", "messages", "tools", "temperature", "max_tokens", "agent_prompt_mode", ) } for event in events if event.get("type") == "model_request_snapshot" ] metric_tool_events = [ tool_event for metric in metrics for tool_event in (metric.get("tool_events") or []) if isinstance(tool_event, dict) ] summarized_tool_events = [ { "tool": canonical_tool(str(event.get("tool") or "")), "command": str(event.get("command") or ""), "exit_code": event.get("exit_code"), "output_preview": str(event.get("output") or "")[:500], } for event in [*outputs, *metric_tool_events] if isinstance(event, dict) ] observed_events = metric_tool_events or outputs or starts first = observed_events[0] if observed_events else {} first_tool = canonical_tool(first.get("tool")) first_action = parse_action(first.get("command")) response_text = "".join(text).strip() if not response_text and metrics: round_texts = metrics[-1].get("round_texts") or [] response_text = next((str(item).strip() for item in reversed(round_texts) if str(item).strip()), "") expected_tool = spec["expected_tool"] expected_action = spec.get("expected_action") or "" expected_actions = [str(item) for item in (spec.get("expected_actions") or [])] max_tool_count = spec.get("max_tool_count") if expected_action and not expected_actions: expected_actions = [expected_action] if expected_tool == "no_tool": tool_ok = not observed_events execution_ok = bool(response_text) and not errors else: tool_ok = first_tool == expected_tool executed = [ event for event in [*outputs, *metric_tool_events] if canonical_tool(str(event.get("tool") or "")) == expected_tool and (not expected_actions or event_action(event) in expected_actions) and output_ok(event) ] execution_ok = bool(executed) and not errors action_ok = not expected_actions or first_action in expected_actions lower_response = response_text.lower() required_any = [str(item).lower() for item in spec.get("required_any") or []] required_all = [str(item).lower() for item in spec.get("required_all") or []] response_quality_ok = bool(response_text) and ( not required_any or any(item in lower_response for item in required_any) ) and all(item in lower_response for item in required_all) if malformed_text_surface(response_text): response_quality_ok = False tool_efficiency_ok = True if isinstance(max_tool_count, int): tool_efficiency_ok = len(observed_events) <= max_tool_count latest_metrics = metrics[-1] if metrics else {} usage_buckets = latest_metrics.get("usage_buckets") if isinstance(latest_metrics, dict) else None return { "message": spec["message"], "expected_tool": expected_tool, "expected_action": expected_action, "expected_actions": expected_actions, "first_tool": first_tool, "first_action": first_action, "tool_count": len(observed_events), "tool_ok": bool(tool_ok), "action_ok": bool(action_ok), "execution_ok": bool(execution_ok), "response_quality_ok": bool(response_quality_ok), "tool_efficiency_ok": bool(tool_efficiency_ok), "max_tool_count": max_tool_count, "stream_errors": errors, "response": response_text[:2000], "input_tokens": latest_metrics.get("input_tokens"), "output_tokens": latest_metrics.get("output_tokens"), "tokens_per_second": latest_metrics.get("tokens_per_second"), "request_context_tokens": latest_metrics.get("request_context_tokens"), "usage_buckets": usage_buckets if isinstance(usage_buckets, list) else [], "tool_events": summarized_tool_events, "elapsed_seconds": round(time.monotonic() - started, 3), "approval_turns": approval_turns, "model_request_snapshots": snapshots, } def write_output(path: Path, records: list[dict[str, Any]], model: str) -> None: turns = [turn for record in records for turn in record["turns"]] summary = { "model": model, "scenarios": len(records), "turns": len(turns), "tool_success": sum(turn["tool_ok"] for turn in turns), "action_success": sum(turn["action_ok"] for turn in turns), "execution_success": sum(turn["execution_ok"] for turn in turns), "response_quality_success": sum(turn["response_quality_ok"] for turn in turns), "tool_efficiency_success": sum(turn.get("tool_efficiency_ok", True) for turn in turns), "stream_errors": sum(bool(turn["stream_errors"]) for turn in turns), "records": records, } tmp = path.with_name(path.name + ".tmp") tmp.write_text(json.dumps(summary, indent=2, ensure_ascii=True) + "\n") tmp.replace(path) def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--base-url", default="http://127.0.0.1:7011") parser.add_argument("--endpoint", default="http://host.docker.internal:18052/v1") parser.add_argument("--endpoint-id", default="v8c1000") parser.add_argument("--model", default="qwen35-9b-tool-router-v15-regular-chat-boundary-final") parser.add_argument("--selected-endpoint-url", default="") parser.add_argument("--selected-model", default="") parser.add_argument("--cookie-file", default="data/sessions.json") parser.add_argument("--output", required=True) parser.add_argument("--prompt-mode", default="compact") parser.add_argument("--timeout", type=float, default=180.0) parser.add_argument("--cases", default="") parser.add_argument("--no-auto-approve", dest="auto_approve", action="store_false") parser.add_argument("--keep-sessions", action="store_true") args = parser.parse_args() selected = {item.strip() for item in args.cases.split(",") if item.strip()} scenarios = [case for case in SCENARIOS if not selected or case["scenario"] in selected] unknown = selected - {case["scenario"] for case in SCENARIOS} if unknown: raise SystemExit(f"Unknown scenario(s): {', '.join(sorted(unknown))}") output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) client = httpx.Client( cookies={"odysseus_session": _cookie(Path(args.cookie_file))}, follow_redirects=False, ) records: list[dict[str, Any]] = [] try: needs_crud_cleanup = any(str(case.get("fixture_prefix") or "").startswith("ODY-EVAL-CRUD-") for case in scenarios) with _crud_fixture_cleanup(needs_crud_cleanup): for scenario in scenarios: session_id = create_session( client, args, "[eval-context] " + scenario["scenario"] + " " + time.strftime("%Y%m%d-%H%M%S") + "-" + uuid.uuid4().hex[:6], ) turns = [] try: for spec in scenario["turns"]: turn = run_turn(client, args, session_id, spec) turns.append(turn) print( json.dumps( { "scenario": scenario["scenario"], **{ key: turn.get(key) for key in ( "message", "expected_tool", "first_tool", "expected_action", "expected_actions", "first_action", "tool_ok", "action_ok", "execution_ok", "response_quality_ok", "tool_efficiency_ok", "max_tool_count", "tool_count", "input_tokens", "output_tokens", "elapsed_seconds", "stream_errors", ) }, }, ensure_ascii=True, ), flush=True, ) finally: if args.keep_sessions: print(json.dumps({"kept_session": session_id, "scenario": scenario["scenario"]}), flush=True) else: try: client.delete(args.base_url.rstrip("/") + f"/api/session/{session_id}", timeout=15) except Exception: pass records.append({"scenario": scenario["scenario"], "turns": turns}) write_output(output, records, args.selected_model or args.model) finally: client.close() write_output(output, records, args.selected_model or args.model) summary = json.loads(output.read_text()) print("SUMMARY", json.dumps({k: v for k, v in summary.items() if k != "records"})) if __name__ == "__main__": main()