audit search variety with model provenance and per-turn latency

scripts/verify_clean_v3_search_quality.mjs

@@ -11,6 +11,7 @@ const owner = 'sft_alex_creator';
 const endpointId = process.env.ENDPOINT_ID || '1d1022ef';
 const endpointUrl = process.env.ENDPOINT_URL || (() => { throw new Error("ENDPOINT_URL is required"); })();
 const model = process.env.MODEL || 'odysseus-qwen3.5-tools-pre-heretic';
+const varietyOnly = process.env.VARIETY_ONLY === '1';
 const run = new Date().toISOString().replace(/[:.]/g, '-');
 const reportPath = path.resolve(process.env.REPORT_PATH || path.join(root, `reports/clean-v3-search-quality-${run}.json`));
 if (!reportPath.startsWith(path.join(root, 'reports') + path.sep) || fs.existsSync(reportPath)) throw Error('Report path must be new and under reports/');
@@ -19,7 +20,7 @@ const token = Object.entries(sessions).find(([, value]) => value?.username === o
 if (!token) throw Error(`No active ${owner} session`);
 
 const marker = `ody-search-${crypto.randomUUID()}`;
-const report = { run, owner, marker, status: 'running', scenarios: [], privacy: 'Public synthetic queries only; fetched bodies and private data are not retained.' };
+const report = { run, owner, marker, model, endpointUrl, status: 'running', scenarios: [], turns: [], privacy: 'Public synthetic queries only; fetched bodies and private data are not retained.' };
 const save = () => fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n');
 fs.mkdirSync(path.dirname(reportPath), { recursive: true }); save();
 const canonical = value => String(value || '').replace(/^mcp__email__/, '');
@@ -51,6 +52,7 @@ async function preparePage(context, id) {
 }
 
 async function send(page, prompt) {
+  const started = performance.now();
   const responsePromise = page.waitForResponse(r => new URL(r.url()).pathname === '/api/chat_stream' && r.request().method() === 'POST', { timeout: 120000 });
   await page.locator('textarea#message:visible').fill(prompt);
   await page.locator('textarea#message:visible').press('Enter');
@@ -60,7 +62,16 @@ async function send(page, prompt) {
   const starts = events.filter(event => event.type === 'tool_start').map(event => ({ tool: canonical(event.tool), args: event.command || '' }));
   const outputs = events.filter(event => event.type === 'tool_output').map(event => ({ tool: canonical(event.tool), exit_code: event.exit_code ?? null, error: Boolean(event.error) }));
   const final = events.filter(event => event.type === 'final_response').map(event => event.content || '').join('') || events.filter(event => typeof event.delta === 'string').map(event => event.delta).join('');
-  return { http_ok: response.ok(), contract, starts, outputs, final };
+  const metrics = events.find(event => event.type === 'metrics')?.data;
+  const observation = {
+    prompt, seconds: (performance.now() - started) / 1000,
+    rounds: metrics?.agent_rounds ?? null,
+    actual_model: metrics?.model ?? null,
+    tools: starts, outputs, final,
+    runtime_error: events.some(event => event.type === 'error') || /v3 test encountered an error/i.test(final),
+  };
+  report.turns.push(observation); save();
+  return { http_ok: response.ok(), contract, starts, outputs, final, ...observation };
 }
 
 let browser;
@@ -69,6 +80,51 @@ try {
   const context = await browser.newContext({ serviceWorkers: 'block', extraHTTPHeaders: { 'Accept-Encoding': 'identity' } });
   await context.addCookies([{ name: 'odysseus_session', value: token, url: base }]);
 
+  if (varietyOnly) {
+    const cases = [
+      ['news-typo', ['latset ai neews?', 'more about the second story, with sources'], true],
+      ['country-casual', ['whats new in japan rn'], true],
+      ['software-typo', ['latest pythno verison? official source pls'], true],
+      ['manual', ['find official english manual for Sony WH-1000XM5'], true],
+      ['comparison', ['compare current firefox and chrome privacy features with sources'], true],
+      ['research', ['How do sodium ion batteries compare with lithium ion for home storage? Find evidence and explain tradeoffs.'], true],
+      ['no-web-typo', ['whats 12 tims 7'], false],
+      ['no-web-greeting', ['helo'], false],
+    ];
+    async function runCase([name, prompts, needsWeb]) {
+      const scenario = { name, status: 'running', turns: [] };
+      report.scenarios.push(scenario); save();
+      let id, page;
+      try {
+        id = await createSession(context, `[search-variety] ${name} ${marker}`);
+        page = await preparePage(context, id);
+        for (const prompt of prompts) {
+          const turn = await send(page, prompt);
+          const tools = turn.starts.map(x => x.tool);
+          const checks = {
+            no_runtime_error: !turn.runtime_error,
+            model_matches: turn.actual_model === model,
+            nonempty_answer: turn.final.trim().length > 0,
+            no_reasoning_leak: noLeak(turn.final),
+            expected_web_use: needsWeb ? tools.some(x => ['web_search', 'web_fetch', 'private_browser'].includes(x)) : tools.length === 0,
+          };
+          scenario.turns.push({ prompt, checks, seconds: turn.seconds, tools,
+            status: Object.values(checks).every(Boolean) ? 'mechanics_passed' : 'failed',
+            quality_review: 'pending_manual_evidence_review' });
+          save();
+        }
+        scenario.status = scenario.turns.every(x => x.status === 'mechanics_passed') ? 'needs_quality_review' : 'failed';
+      } catch (error) { scenario.status = 'failed'; scenario.error = String(error).slice(0, 300); }
+      finally {
+        if (id) scenario.cleanup = { session_removed: (await context.request.delete(`${base}/api/session/${id}`)).ok() };
+        if (page) await page.close(); save();
+      }
+    }
+    // Two simultaneous conversations keep endpoint contention bounded.
+    const queue = [...cases];
+    await Promise.all([0, 1].map(async () => { while (queue.length) await runCase(queue.shift()); }));
+  } else {
+
   // One conversation proves discovery, evidence reuse, then explicit page inspection.
   {
     const scenario = { name: 'official-search-summary-fetch', status: 'running', turns: [] };
@@ -88,6 +144,8 @@ try {
         const expected = index === 0 ? 'web_search' : index === 2 ? 'web_fetch' : null;
         const checks = {
           http_ok: turn.http_ok,
+          no_runtime_error: !turn.runtime_error,
+          model_matches: turn.actual_model === model,
           clean_route: turn.contract?.selection_mode === 'clean_compact_v3_preview',
           expected_tool: expected ? tools.includes(expected) : tools.length === 0,
           successful_tools: turn.outputs.length === 0 || (() => {
@@ -95,7 +153,12 @@ try {
             return !last.error && (last.exit_code == null || last.exit_code === 0);
           })(),
           no_reasoning_leak: noLeak(turn.final),
-          grounded_answer: index === 0 ? /python|pypa|packag/i.test(turn.final) : index === 2 ? /pyproject|build|sdist|wheel|pip|twine/i.test(turn.final) : turn.final.trim().length > 15,
+          answer_contains_requested_information: index === 0
+            ? /https:\/\/packaging\.python\.org\b/.test(turn.final)
+            : index === 2
+              ? /pyproject\.toml/i.test(turn.final) && /\bwheel\b/i.test(turn.final)
+                && /\b(?:sdist|source distribution)\b/i.test(turn.final)
+              : /Python Packaging User Guide|PyPA/i.test(turn.final),
         };
         scenario.turns.push({ index, tools, output_statuses: turn.outputs, final: turn.final.slice(0, 500), final_chars: turn.final.length, checks, status: Object.values(checks).every(Boolean) ? 'passed' : 'failed' }); save();
       }
@@ -121,6 +184,8 @@ try {
       const checks = {
         http_ok: turn.http_ok, clean_route: turn.contract?.selection_mode === 'clean_compact_v3_preview',
         searched: searches.length >= 1, corrected_query: /market/i.test(query) && !/mraket/i.test(query),
+        no_runtime_error: !turn.runtime_error, model_matches: turn.actual_model === model,
+        covers_requested_indexes: /S&P\s*500/i.test(turn.final) && /Dow/i.test(turn.final) && /Nasdaq/i.test(turn.final),
         successful_tools: turn.outputs.every(x => !x.error && (x.exit_code == null || x.exit_code === 0)),
         no_reasoning_leak: noLeak(turn.final), no_irrelevant_misspelling_results: !/telegram|marketing|mraket/i.test(turn.final),
       };
@@ -146,6 +211,7 @@ try {
       const checks = {
         http_ok: turn.http_ok, clean_route: turn.contract?.selection_mode === 'clean_compact_v3_preview',
         bounded_search: searches.length >= 1 && searches.length <= 2,
+        no_runtime_error: !turn.runtime_error, model_matches: turn.actual_model === model,
         successful_tools: turn.outputs.every(x => !x.error && (x.exit_code == null || x.exit_code === 0)),
         no_reasoning_leak: noLeak(turn.final), honest_gap: /couldn.t find|cannot find|no (?:current )?(?:reliable|supporting|public|matching)|not (?:available|found|listed)|fictional|insufficient/i.test(turn.final),
       };
@@ -157,9 +223,10 @@ try {
       if (page) await page.close(); save();
     }
   }
+  }
 } finally { if (browser) await browser.close(); }
 
-report.status = report.scenarios.length === 3 && report.scenarios.every(x => x.status === 'passed' && x.cleanup?.session_removed) ? 'passed' : 'failed';
+report.status = varietyOnly ? 'needs_quality_review' : report.scenarios.length === 3 && report.scenarios.every(x => x.status === 'passed' && x.cleanup?.session_removed) ? 'passed' : 'failed';
 report.summary = { passed: report.scenarios.filter(x => x.status === 'passed').length, total: report.scenarios.length };
 save();
 console.log(JSON.stringify({ report: path.relative(root, reportPath), status: report.status, summary: report.summary }));