mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-09-17 05:32:20 +02:00
144 lines
9.6 KiB
JavaScript
144 lines
9.6 KiB
JavaScript
#!/usr/bin/env node
|
|
/** Real Agent UI: missing record -> corrected identity -> evidence-only recall. */
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import crypto from 'node:crypto';
|
|
import { chromium } from 'playwright';
|
|
|
|
const root = path.resolve(new URL('..', import.meta.url).pathname);
|
|
const base = 'http://127.0.0.1:7011';
|
|
const owner = 'sft_alex_creator';
|
|
const model = 'odysseus-qwen3.5-tools-pre-heretic';
|
|
const selected = new Set((process.env.FAMILIES || 'notes,documents').split(','));
|
|
const specs = [
|
|
{family: 'notes', noun: 'note', tool: 'manage_notes', api: '/api/notes', bodyKey: 'content', actions: ['view', 'read', 'get']},
|
|
{family: 'documents', noun: 'document', tool: 'manage_documents', api: '/api/document', bodyKey: 'current_content', actions: ['read', 'view', 'open', 'get']},
|
|
].filter(s => selected.has(s.family));
|
|
if (!specs.length) throw Error('No matching families');
|
|
const auth = JSON.parse(fs.readFileSync('/home/pewds/odysseus-cookbook-fresh/data/sessions.json', 'utf8'));
|
|
const token = Object.entries(auth).find(([, v]) => v?.username === owner)?.[0];
|
|
if (!token) throw Error('No SFT session');
|
|
const reportPath = path.join(root, `reports/record-read-recovery-${new Date().toISOString().replace(/[:.]/g, '-')}.json`);
|
|
const report = {status: 'running', model, routing: 'recent_model_choice', cases: [],
|
|
privacy: 'Disposable SFT records only. No raw account data, answers, IDs, or authentication retained.'};
|
|
const save = () => fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n');
|
|
const parseSSE = body => body.replace(/\r\n/g, '\n').split('\n\n').flatMap(frame => {
|
|
const data = frame.split('\n').filter(l => l.startsWith('data:')).map(l => l.slice(5).trimStart()).join('\n');
|
|
if (!data || data === '[DONE]') return [];
|
|
try { return [JSON.parse(data)]; } catch { return [{type: 'invalid_sse'}]; }
|
|
});
|
|
const argsOf = e => { try { return JSON.parse(e.command || '{}'); } catch { return {}; } };
|
|
let browser, context;
|
|
try {
|
|
browser = await chromium.launch({headless: true, args: ['--no-proxy-server']});
|
|
context = await browser.newContext({serviceWorkers: 'block', extraHTTPHeaders: {
|
|
'Accept-Encoding': 'identity', 'x-odysseus-routing-experiment': 'recent_model_choice',
|
|
}});
|
|
await context.addCookies([{name: 'odysseus_session', value: token, url: base}]);
|
|
const session = async label => {
|
|
const response = await context.request.post(`${base}/api/session`, {multipart: {
|
|
name: `[record-read-recovery] ${label}`, model, endpoint_id: '1d1022ef',
|
|
endpoint_url: 'http://100.67.207.85:19184/v1/chat/completions', skip_validation: 'true', rag: 'false',
|
|
}});
|
|
if (!response.ok()) throw Error(`Session create HTTP ${response.status()}`);
|
|
return (await response.json()).id;
|
|
};
|
|
for (const spec of specs) {
|
|
const result = {family: spec.family, status: 'running', turns: [], cleanup: {record: false, sessions: false}};
|
|
report.cases.push(result); save();
|
|
const title = `read-fixture-${crypto.randomUUID()}`;
|
|
const code = `amber-${crypto.randomUUID().slice(0, 8)}`;
|
|
const delay = crypto.randomInt(17, 48);
|
|
const body = `Recovery code: ${code}\nRetry delay: ${delay} seconds.\nMaximum attempts: 6.`;
|
|
const missing = crypto.randomUUID();
|
|
let id, page, seededSession, chat;
|
|
let sourceVerified = false;
|
|
try {
|
|
seededSession = await session('fixture storage');
|
|
const created = await context.request.post(`${base}${spec.api}`, {data: {
|
|
title, content: body, session_id: seededSession,
|
|
...(spec.family === 'documents' ? {language: 'markdown'} : {source: 'eval'}),
|
|
}});
|
|
if (!created.ok()) throw Error(`Fixture create HTTP ${created.status()}`);
|
|
id = (await created.json()).id;
|
|
const original = await (await context.request.get(`${base}${spec.api}/${id}`)).json();
|
|
if (original.title !== title || original[spec.bodyKey] !== body) throw Error('Fixture mismatch');
|
|
if ((await context.request.get(`${base}${spec.api}/${missing}`)).status() !== 404) throw Error('Missing-ID precondition failed');
|
|
chat = await session('read and recover');
|
|
page = await context.newPage();
|
|
await page.goto(`${base}/#${chat}`, {waitUntil: 'domcontentloaded'});
|
|
await page.waitForFunction(id => window.__odysseusSessionReadyId === id, chat, {timeout: 30000});
|
|
if (await page.locator('#mode-agent-btn').getAttribute('aria-pressed') !== 'true') await page.locator('#mode-agent-btn').click();
|
|
const prompts = [
|
|
`Read my ${spec.noun} with ID ${missing}. Tell me whether it exists. Do not create anything or substitute another record.`,
|
|
`Sorry, I meant ID ${id}. Read that one and tell me its recovery code and retry delay.`,
|
|
'How many seconds was the delay? Answer from what you just read; do not change or rerun anything.',
|
|
];
|
|
for (let index = 0; index < prompts.length; index++) {
|
|
const waiting = page.waitForResponse(r => new URL(r.url()).pathname === '/api/chat_stream' && r.request().method() === 'POST', {timeout: 120000});
|
|
await page.locator('textarea#message:visible').fill(prompts[index]);
|
|
await page.locator('textarea#message:visible').press('Enter');
|
|
const response = await waiting;
|
|
const submitted = response.request().postData() || '';
|
|
const events = parseSSE(await response.text());
|
|
await page.waitForFunction(() => !document.querySelector('#chat-history .streaming'), null, {timeout: 15000}).catch(() => {});
|
|
const contract = events.find(e => e.type === 'turn_contract') || {};
|
|
const starts = events.filter(e => e.type === 'tool_start');
|
|
const outputs = events.filter(e => e.type === 'tool_output');
|
|
const args = argsOf(starts[0] || {});
|
|
const target = args.id || args.document_id || args.uid || args.note_id;
|
|
const final = events.filter(e => e.type === 'final_response').map(e => e.content || '').join('')
|
|
|| events.filter(e => typeof e.delta === 'string').map(e => e.delta).join('');
|
|
const displayed = await page.locator('#chat-history .msg-ai .stream-content').last().innerText({timeout: 5000}).catch(() => '');
|
|
const matches = text => index === 0 ? /not found|does(?:n.t| not) exist|could(?:n.t| not) find|no .*found|unable to find/i.test(text)
|
|
: index === 1 ? text.includes(code) && new RegExp(`\\b${delay}\\b`).test(text)
|
|
: new RegExp(`\\b${delay}\\s*(?:seconds|s\\b)`, 'i').test(text);
|
|
if (index === 1) sourceVerified = outputs.some(e => e.tool === spec.tool && !e.error && String(e.output).includes(code) && String(e.output).includes(String(delay)));
|
|
const state = await (await context.request.get(`${base}${spec.api}/${id}`)).json();
|
|
const checks = {
|
|
http_ok: response.ok(), clean_route: contract.selection_mode === 'clean_compact_v3_preview',
|
|
model_choice: contract.routing_experiment === 'recent_model_choice',
|
|
no_injected_fixture_answer: !submitted.includes(code),
|
|
offered: index === 2 || (contract.offered || []).includes(spec.tool),
|
|
exact_call: index === 2 ? starts.length === 0 : starts.length === 1 && starts[0].tool === spec.tool
|
|
&& spec.actions.includes(args.action) && target === (index === 0 ? missing : id),
|
|
execution_outcome: index === 2 ? outputs.length === 0 : outputs.length === 1
|
|
&& (index === 0 ? outputs[0].error === true && outputs[0].execution_attempted === true && outputs[0].blocked === false
|
|
: !outputs[0].error && outputs[0].exit_code === 0),
|
|
grounded_source: index === 0 || sourceVerified,
|
|
final_evidence: matches(final), rendered_evidence: matches(displayed),
|
|
unchanged_record: state.title === title && state[spec.bodyKey] === body,
|
|
no_stream_error: !events.some(e => ['error', 'invalid_sse'].includes(e.type)),
|
|
};
|
|
result.turns.push({index, tools: starts.map(e => e.tool), argument_keys: Object.keys(args), checks,
|
|
status: Object.values(checks).every(Boolean) ? 'passed' : 'failed'}); save();
|
|
}
|
|
result.status = result.turns.every(t => t.status === 'passed') ? 'passed' : 'failed';
|
|
} catch (error) { result.status = 'failed'; result.error = String(error).split('\n')[0].slice(0, 300); }
|
|
finally {
|
|
if (page) await page.close();
|
|
try {
|
|
if (id) {
|
|
const row = await (await context.request.get(`${base}${spec.api}/${id}`)).json();
|
|
if (row.title !== title) throw Error('Refuse non-fixture cleanup');
|
|
const deleted = await context.request.delete(`${base}${spec.api}/${id}`);
|
|
if (!deleted.ok()) throw Error('Fixture cleanup failed');
|
|
const checked = await context.request.get(`${base}${spec.api}/${id}`);
|
|
result.cleanup.record = checked.status() === 404 || (spec.family === 'documents' && (await checked.json()).is_active === false);
|
|
}
|
|
result.cleanup.sessions = true;
|
|
for (const sid of [chat, seededSession].filter(Boolean)) {
|
|
if (!(await context.request.delete(`${base}/api/session/${sid}`)).ok()) result.cleanup.sessions = false;
|
|
}
|
|
} catch { result.cleanup.error = true; }
|
|
if (!result.cleanup.record || !result.cleanup.sessions) result.status = 'failed';
|
|
save();
|
|
}
|
|
}
|
|
} catch (error) { report.error = String(error).split('\n')[0].slice(0, 300); }
|
|
finally { if (browser) await browser.close(); }
|
|
report.status = report.cases.length === specs.length && report.cases.every(c => c.status === 'passed') ? 'passed' : 'failed';
|
|
save();
|
|
console.log(JSON.stringify({report: path.relative(root, reportPath), status: report.status, cases: report.cases}));
|
|
if (report.status !== 'passed') process.exitCode = 1;
|