Squash Odysseus development history

scripts/verify_search_chats_followup.mjs

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+/** Real 7011 historical-chat search followed by a refined search. */
+import fs from 'node:fs';
+import path from 'node:path';
+import { chromium } from 'playwright';
+
+const root = path.resolve(new URL('..', import.meta.url).pathname);
+const base = process.env.BASE_URL || 'http://127.0.0.1:7011';
+const endpointId = process.env.ENDPOINT_ID || '1d1022ef';
+const endpointUrl = process.env.ENDPOINT_URL || 'http://100.67.207.85:19184/v1/chat/completions';
+const model = process.env.MODEL || 'odysseus-qwen3.5-tools-pre-heretic';
+const owner = 'sft_alex_creator';
+const reportPath = path.resolve(process.env.REPORT_PATH || path.join(root, `reports/search-chats-followup-${new Date().toISOString().replace(/[:.]/g, '-')}.json`));
+if (!reportPath.startsWith(path.join(root, 'reports') + path.sep) || fs.existsSync(reportPath)) throw Error('Report path must be new and under reports/');
+const prompts = [
+  'Search my past chats for the phrase tool grounding. Return at most three clickable chat titles. Read only.',
+  'Search those past chats again, but narrow the query to tool evidence. Return at most three clickable chat titles. Read only.',
+];
+const auth = JSON.parse(fs.readFileSync('/home/pewds/odysseus-cookbook-fresh/data/sessions.json', 'utf8'));
+const token = Object.entries(auth).find(([, value]) => value?.username === owner)?.[0];
+if (!token) throw Error(`No active ${owner} session`);
+const report = { model, status: 'running', turns: [], cleanup: false, privacy: 'No chat titles, transcript matches, result text, or answer text retained.' };
+const save = () => { fs.mkdirSync(path.dirname(reportPath), { recursive: true }); fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n'); };
+const parseSSE = body => body.replace(/\r\n/g, '\n').split('\n\n').flatMap(frame => {
+  const raw = frame.split('\n').filter(line => line.startsWith('data:')).map(line => line.slice(5).trimStart()).join('\n');
+  if (!raw || raw === '[DONE]') return [];
+  try { return [JSON.parse(raw)]; } catch { return [{ type: 'invalid_sse' }]; }
+});
+const parseArgs = event => { try { return JSON.parse(event?.command || '{}'); } catch { return {}; } };
+
+let browser, context, page, session = '';
+try {
+  browser = await chromium.launch({ headless: true, args: ['--no-proxy-server'] });
+  context = await browser.newContext({ serviceWorkers: 'block', extraHTTPHeaders: {
+    'Accept-Encoding': 'identity', 'x-odysseus-routing-experiment': 'recent_model_choice',
+  } });
+  await context.addCookies([{ name: 'odysseus_session', value: token, url: base }]);
+  const created = await context.request.post(`${base}/api/session`, { multipart: {
+    name: '[search-chats-followup] refined-query', model, endpoint_id: endpointId,
+    endpoint_url: endpointUrl, skip_validation: 'true', rag: 'false',
+  }});
+  if (!created.ok()) throw Error(`Session create HTTP ${created.status()}`);
+  session = (await created.json()).id;
+  page = await context.newPage();
+  await page.goto(`${base}/#${session}`, { waitUntil: 'domcontentloaded', timeout: 30000 });
+  await page.waitForFunction(id => window.__odysseusSessionReadyId === id, session, { timeout: 30000 });
+  const agent = page.locator('#mode-agent-btn');
+  if (await agent.getAttribute('aria-pressed') !== 'true') await agent.click();
+  for (let index = 0; index < prompts.length; index++) {
+    const waiting = page.waitForResponse(r => new URL(r.url()).pathname === '/api/chat_stream' && r.request().method() === 'POST', { timeout: 120000 });
+    await page.locator('textarea#message:visible').fill(prompts[index]);
+    await page.locator('textarea#message:visible').press('Enter');
+    const response = await waiting;
+    const events = parseSSE(await response.text());
+    await page.waitForFunction(() => !document.querySelector('#chat-history .msg-ai.streaming'), null, { timeout: 15000 }).catch(() => {});
+    const contract = events.find(event => event.type === 'turn_contract') || {};
+    const starts = events.filter(event => event.type === 'tool_start');
+    const outputs = events.filter(event => event.type === 'tool_output' && event.tool === 'search_chats');
+    const expected = starts.filter(event => event.tool === 'search_chats');
+    const args = parseArgs(expected[0]);
+    const checks = {
+      http_ok: response.ok(),
+      clean_route: contract.selection_mode === 'clean_compact_v3_preview',
+      model_choice_route: contract.routing_experiment === 'recent_model_choice',
+      memory_capability: (contract.active_capabilities || []).includes('memory'),
+      expected_offered: (contract.offered || []).includes('search_chats'),
+      exactly_one_expected_call: starts.length === 1 && expected.length === 1,
+      argument_contract: typeof args.query === 'string' && (index === 0 ? /grounding/i.test(args.query) : /evidence/i.test(args.query)),
+      exactly_one_successful_output: outputs.length === 1 && !outputs[0]?.error && (outputs[0]?.exit_code == null || outputs[0]?.exit_code === 0),
+      no_stream_error: !events.some(event => ['error', 'invalid_sse'].includes(event.type)),
+    };
+    report.turns.push({ index, tools: starts.map(event => event.tool), argument_keys: Object.keys(args).sort(), checks, status: Object.values(checks).every(Boolean) ? 'passed' : 'failed' });
+    save();
+  }
+} catch (error) {
+  report.error = String(error).split('\n')[0].slice(0, 400);
+} finally {
+  if (page) await page.close();
+  if (context && session) report.cleanup = (await context.request.delete(`${base}/api/session/${encodeURIComponent(session)}`)).ok();
+  if (browser) await browser.close();
+}
+report.status = report.turns.length === prompts.length && report.turns.every(turn => turn.status === 'passed') && report.cleanup ? 'passed' : 'failed';
+save();
+console.log(JSON.stringify({ report: path.relative(root, reportPath), status: report.status, turns: report.turns, cleanup: report.cleanup }));
+if (report.status !== 'passed') process.exitCode = 1;