#!/usr/bin/env python3 """Generate non-duplicate Odysseus flow variants from behavioral seed flows.""" from __future__ import annotations import argparse import concurrent.futures import json import re import time import urllib.request from pathlib import Path from typing import Any from odysseus_related_flow_audit import Flow, flow_matrix from repair_sft_corpus_with_kimi import endpoint, parse_json ROOT = Path(__file__).resolve().parents[1] def normalize_prompt(value: str) -> str: value = value.lower().replace("{marker}", " marker ") value = re.sub(r"\b\d{8}_\d{6}(?:-[a-f0-9]+)?\b", " marker ", value) value = re.sub(r"\b[a-f0-9]{8,}\b", " marker ", value) return re.sub(r"[^a-z0-9]+", " ", value).strip() def existing_prompts(path: Path) -> set[str]: if not path.exists(): return set() prompts = set() for line in path.read_text(encoding="utf-8").splitlines(): if not line.strip(): continue row = json.loads(line) value = row.get("user") if isinstance(value, str) and value.strip(): prompts.add(normalize_prompt(value)) return prompts def seed_payload(flow: Flow) -> dict[str, Any]: return { "id": flow.id, "domain": flow.domain, "title": flow.title, "turns": [ { "id": turn.id, "prompt": turn.prompt, "tools": list(turn.tools), "dry_run": turn.dry_run, } for turn in flow.turns ], } def generate(ep: dict[str, str], flow: Flow, count: int, timeout: float, retries: int) -> list[dict[str, Any]]: system = """You create realistic multi-turn user workflows for testing an assistant UI. Return strict JSON only: {"flows":[...]}. Each flow must contain id, domain, title, and turns. Treat the supplied flow as a behavioral seed, never as text to paraphrase mechanically. - Produce the requested number of substantially different scenarios. - Preserve the exact turn count, turn IDs, expected tools, dry_run values, and tool order. - Each conversation must remain coherent: follow-ups refer naturally to prior results or objects. - Change entities, goals, wording, and realistic task details across variants. - Keep {marker} exactly where a temporary unique name is required. - Never mention tests, audits, fixtures, harnesses, SFT, synthetic data, schemas, or training. - Do not use private real-world personal data. Invent ordinary benign names and content. - Do not add unsupported IDs or claim results before a tool has produced them. - Dry-run turns must explicitly avoid state changes; mutation turns should request the action clearly. - User prompts should sound casual and varied, including occasional concise follow-ups. """ body = { "model": ep["model"], "messages": [ {"role": "system", "content": system}, {"role": "user", "content": json.dumps({"count": count, "seed": seed_payload(flow)}, ensure_ascii=False)}, ], "temperature": 0.85, "max_tokens": 9000, "response_format": {"type": "json_object"}, } request = urllib.request.Request( ep["base_url"].rstrip("/") + "/chat/completions", data=json.dumps(body).encode(), headers={"Content-Type": "application/json", "Authorization": f"Bearer {ep['api_key']}"}, method="POST", ) last_error: Exception | None = None for attempt in range(retries + 1): try: with urllib.request.urlopen(request, timeout=timeout) as response: payload = json.loads(response.read().decode()) break except Exception as exc: last_error = exc if attempt == retries: raise time.sleep(2 * (attempt + 1)) else: raise RuntimeError("Kimi generation failed") from last_error message = payload["choices"][0]["message"] parsed = parse_json(str(message.get("content") or message.get("reasoning_content") or "")) flows = parsed.get("flows") if not isinstance(flows, list): raise ValueError(f"Kimi returned no flows for {flow.id}") return flows def validate_variant(seed: Flow, raw: dict[str, Any], index: int) -> dict[str, Any]: turns = raw.get("turns") if not isinstance(turns, list) or len(turns) != len(seed.turns): raise ValueError(f"{seed.id} variant {index}: wrong turn count") clean_turns = [] for expected, actual in zip(seed.turns, turns): if not isinstance(actual, dict): raise ValueError(f"{seed.id} variant {index}: invalid turn") tools = actual.get("tools") if tools != list(expected.tools) or bool(actual.get("dry_run", False)) != expected.dry_run: raise ValueError(f"{seed.id} variant {index}: tool contract changed") prompt = str(actual.get("prompt") or "").strip() if not prompt: raise ValueError(f"{seed.id} variant {index}: empty prompt") clean_turns.append({ "id": expected.id, "prompt": prompt, "tools": list(expected.tools), "dry_run": expected.dry_run, }) return { "id": f"{seed.id}_v{index:02d}", "domain": seed.domain, "title": str(raw.get("title") or f"{seed.title} variant {index}"), "turns": clean_turns, } def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--seeds", required=True, help="Comma-separated built-in flow IDs") parser.add_argument("--variants-per-seed", type=int, default=3) parser.add_argument("--trace", type=Path, default=ROOT / "data/sft_traces/sft_alex_creator.jsonl") parser.add_argument("--out", type=Path, required=True) parser.add_argument("--endpoint-id", default="f3904562") parser.add_argument("--model", default="moonshotai/kimi-k3") parser.add_argument("--workers", type=int, default=4) parser.add_argument("--timeout", type=float, default=90) parser.add_argument("--retries", type=int, default=1) args = parser.parse_args() matrix = {flow.id: flow for flow in flow_matrix()} seed_ids = [value.strip() for value in args.seeds.split(",") if value.strip()] missing = [value for value in seed_ids if value not in matrix] if missing: parser.error(f"unknown seeds: {', '.join(missing)}") seen = existing_prompts(args.trace) ep = endpoint(args.endpoint_id, args.model) output = [] rejected = [] generated: dict[str, list[dict[str, Any]]] = {} with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool: futures = { pool.submit(generate, ep, matrix[seed_id], args.variants_per_seed + 2, args.timeout, args.retries): seed_id for seed_id in seed_ids } for future in concurrent.futures.as_completed(futures): seed_id = futures[future] try: generated[seed_id] = future.result() print(f"generated {seed_id}", flush=True) except Exception as exc: rejected.append({"seed": seed_id, "reason": f"provider failure: {exc!r}"}) print(f"failed {seed_id}: {exc!r}", flush=True) for seed_id in seed_ids: seed = matrix[seed_id] candidates = generated.get(seed_id, []) accepted_for_seed = 0 for candidate in candidates: if accepted_for_seed >= args.variants_per_seed: break try: clean = validate_variant(seed, candidate, accepted_for_seed + 1) except (KeyError, TypeError, ValueError) as exc: rejected.append({"seed": seed_id, "reason": str(exc)}) continue normalized = [normalize_prompt(turn["prompt"]) for turn in clean["turns"]] if len(set(normalized)) != len(normalized) or any(prompt in seen for prompt in normalized): rejected.append({"seed": seed_id, "reason": "duplicate prompt"}) continue if any(re.search(r"\b(?:sft|fixture|harness|synthetic|audit)\b", turn["prompt"], re.I) for turn in clean["turns"]): rejected.append({"seed": seed_id, "reason": "training-meta language"}) continue output.append(clean) seen.update(normalized) accepted_for_seed += 1 args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(json.dumps({"flows": output, "rejected": rejected}, ensure_ascii=False, indent=2), encoding="utf-8") if accepted_for_seed < args.variants_per_seed: rejected.append({"seed": seed_id, "reason": f"only accepted {accepted_for_seed} variants"}) args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(json.dumps({"flows": output, "rejected": rejected}, ensure_ascii=False, indent=2), encoding="utf-8") print(json.dumps({"output": str(args.out), "flows": len(output), "turns": sum(len(row["turns"]) for row in output), "rejected": len(rejected)}, indent=2)) if __name__ == "__main__": main()