mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-12 01:42:20 +02:00
Squash Odysseus development history
This commit is contained in:
pewdiepie-archdaemon
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env node
|
||||
/** Real Agent UI: missing record -> corrected identity -> evidence-only recall. */
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import { chromium } from 'playwright';
|
||||
|
||||
const root = path.resolve(new URL('..', import.meta.url).pathname);
|
||||
const base = 'http://127.0.0.1:7011';
|
||||
const owner = 'sft_alex_creator';
|
||||
const model = 'odysseus-qwen3.5-tools-pre-heretic';
|
||||
const selected = new Set((process.env.FAMILIES || 'notes,documents').split(','));
|
||||
const specs = [
|
||||
{family: 'notes', noun: 'note', tool: 'manage_notes', api: '/api/notes', bodyKey: 'content', actions: ['view', 'read', 'get']},
|
||||
{family: 'documents', noun: 'document', tool: 'manage_documents', api: '/api/document', bodyKey: 'current_content', actions: ['read', 'view', 'open', 'get']},
|
||||
].filter(s => selected.has(s.family));
|
||||
if (!specs.length) throw Error('No matching families');
|
||||
const auth = JSON.parse(fs.readFileSync('/home/pewds/odysseus-cookbook-fresh/data/sessions.json', 'utf8'));
|
||||
const token = Object.entries(auth).find(([, v]) => v?.username === owner)?.[0];
|
||||
if (!token) throw Error('No SFT session');
|
||||
const reportPath = path.join(root, `reports/record-read-recovery-${new Date().toISOString().replace(/[:.]/g, '-')}.json`);
|
||||
const report = {status: 'running', model, routing: 'recent_model_choice', cases: [],
|
||||
privacy: 'Disposable SFT records only. No raw account data, answers, IDs, or authentication retained.'};
|
||||
const save = () => fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n');
|
||||
const parseSSE = body => body.replace(/\r\n/g, '\n').split('\n\n').flatMap(frame => {
|
||||
const data = frame.split('\n').filter(l => l.startsWith('data:')).map(l => l.slice(5).trimStart()).join('\n');
|
||||
if (!data || data === '[DONE]') return [];
|
||||
try { return [JSON.parse(data)]; } catch { return [{type: 'invalid_sse'}]; }
|
||||
});
|
||||
const argsOf = e => { try { return JSON.parse(e.command || '{}'); } catch { return {}; } };
|
||||
let browser, context;
|
||||
try {
|
||||
browser = await chromium.launch({headless: true, args: ['--no-proxy-server']});
|
||||
context = await browser.newContext({serviceWorkers: 'block', extraHTTPHeaders: {
|
||||
'Accept-Encoding': 'identity', 'x-odysseus-routing-experiment': 'recent_model_choice',
|
||||
}});
|
||||
await context.addCookies([{name: 'odysseus_session', value: token, url: base}]);
|
||||
const session = async label => {
|
||||
const response = await context.request.post(`${base}/api/session`, {multipart: {
|
||||
name: `[record-read-recovery] ${label}`, model, endpoint_id: '1d1022ef',
|
||||
endpoint_url: 'http://100.67.207.85:19184/v1/chat/completions', skip_validation: 'true', rag: 'false',
|
||||
}});
|
||||
if (!response.ok()) throw Error(`Session create HTTP ${response.status()}`);
|
||||
return (await response.json()).id;
|
||||
};
|
||||
for (const spec of specs) {
|
||||
const result = {family: spec.family, status: 'running', turns: [], cleanup: {record: false, sessions: false}};
|
||||
report.cases.push(result); save();
|
||||
const title = `read-fixture-${crypto.randomUUID()}`;
|
||||
const code = `amber-${crypto.randomUUID().slice(0, 8)}`;
|
||||
const delay = crypto.randomInt(17, 48);
|
||||
const body = `Recovery code: ${code}\nRetry delay: ${delay} seconds.\nMaximum attempts: 6.`;
|
||||
const missing = crypto.randomUUID();
|
||||
let id, page, seededSession, chat;
|
||||
let sourceVerified = false;
|
||||
try {
|
||||
seededSession = await session('fixture storage');
|
||||
const created = await context.request.post(`${base}${spec.api}`, {data: {
|
||||
title, content: body, session_id: seededSession,
|
||||
...(spec.family === 'documents' ? {language: 'markdown'} : {source: 'eval'}),
|
||||
}});
|
||||
if (!created.ok()) throw Error(`Fixture create HTTP ${created.status()}`);
|
||||
id = (await created.json()).id;
|
||||
const original = await (await context.request.get(`${base}${spec.api}/${id}`)).json();
|
||||
if (original.title !== title || original[spec.bodyKey] !== body) throw Error('Fixture mismatch');
|
||||
if ((await context.request.get(`${base}${spec.api}/${missing}`)).status() !== 404) throw Error('Missing-ID precondition failed');
|
||||
chat = await session('read and recover');
|
||||
page = await context.newPage();
|
||||
await page.goto(`${base}/#${chat}`, {waitUntil: 'domcontentloaded'});
|
||||
await page.waitForFunction(id => window.__odysseusSessionReadyId === id, chat, {timeout: 30000});
|
||||
if (await page.locator('#mode-agent-btn').getAttribute('aria-pressed') !== 'true') await page.locator('#mode-agent-btn').click();
|
||||
const prompts = [
|
||||
`Read my ${spec.noun} with ID ${missing}. Tell me whether it exists. Do not create anything or substitute another record.`,
|
||||
`Sorry, I meant ID ${id}. Read that one and tell me its recovery code and retry delay.`,
|
||||
'How many seconds was the delay? Answer from what you just read; do not change or rerun anything.',
|
||||
];
|
||||
for (let index = 0; index < prompts.length; index++) {
|
||||
const waiting = page.waitForResponse(r => new URL(r.url()).pathname === '/api/chat_stream' && r.request().method() === 'POST', {timeout: 120000});
|
||||
await page.locator('textarea#message:visible').fill(prompts[index]);
|
||||
await page.locator('textarea#message:visible').press('Enter');
|
||||
const response = await waiting;
|
||||
const submitted = response.request().postData() || '';
|
||||
const events = parseSSE(await response.text());
|
||||
await page.waitForFunction(() => !document.querySelector('#chat-history .streaming'), null, {timeout: 15000}).catch(() => {});
|
||||
const contract = events.find(e => e.type === 'turn_contract') || {};
|
||||
const starts = events.filter(e => e.type === 'tool_start');
|
||||
const outputs = events.filter(e => e.type === 'tool_output');
|
||||
const args = argsOf(starts[0] || {});
|
||||
const target = args.id || args.document_id || args.uid || args.note_id;
|
||||
const final = events.filter(e => e.type === 'final_response').map(e => e.content || '').join('')
|
||||
|| events.filter(e => typeof e.delta === 'string').map(e => e.delta).join('');
|
||||
const displayed = await page.locator('#chat-history .msg-ai .stream-content').last().innerText({timeout: 5000}).catch(() => '');
|
||||
const matches = text => index === 0 ? /not found|does(?:n.t| not) exist|could(?:n.t| not) find|no .*found|unable to find/i.test(text)
|
||||
: index === 1 ? text.includes(code) && new RegExp(`\\b${delay}\\b`).test(text)
|
||||
: new RegExp(`\\b${delay}\\s*(?:seconds|s\\b)`, 'i').test(text);
|
||||
if (index === 1) sourceVerified = outputs.some(e => e.tool === spec.tool && !e.error && String(e.output).includes(code) && String(e.output).includes(String(delay)));
|
||||
const state = await (await context.request.get(`${base}${spec.api}/${id}`)).json();
|
||||
const checks = {
|
||||
http_ok: response.ok(), clean_route: contract.selection_mode === 'clean_compact_v3_preview',
|
||||
model_choice: contract.routing_experiment === 'recent_model_choice',
|
||||
no_injected_fixture_answer: !submitted.includes(code),
|
||||
offered: index === 2 || (contract.offered || []).includes(spec.tool),
|
||||
exact_call: index === 2 ? starts.length === 0 : starts.length === 1 && starts[0].tool === spec.tool
|
||||
&& spec.actions.includes(args.action) && target === (index === 0 ? missing : id),
|
||||
execution_outcome: index === 2 ? outputs.length === 0 : outputs.length === 1
|
||||
&& (index === 0 ? outputs[0].error === true && outputs[0].execution_attempted === true && outputs[0].blocked === false
|
||||
: !outputs[0].error && outputs[0].exit_code === 0),
|
||||
grounded_source: index === 0 || sourceVerified,
|
||||
final_evidence: matches(final), rendered_evidence: matches(displayed),
|
||||
unchanged_record: state.title === title && state[spec.bodyKey] === body,
|
||||
no_stream_error: !events.some(e => ['error', 'invalid_sse'].includes(e.type)),
|
||||
};
|
||||
result.turns.push({index, tools: starts.map(e => e.tool), argument_keys: Object.keys(args), checks,
|
||||
status: Object.values(checks).every(Boolean) ? 'passed' : 'failed'}); save();
|
||||
}
|
||||
result.status = result.turns.every(t => t.status === 'passed') ? 'passed' : 'failed';
|
||||
} catch (error) { result.status = 'failed'; result.error = String(error).split('\n')[0].slice(0, 300); }
|
||||
finally {
|
||||
if (page) await page.close();
|
||||
try {
|
||||
if (id) {
|
||||
const row = await (await context.request.get(`${base}${spec.api}/${id}`)).json();
|
||||
if (row.title !== title) throw Error('Refuse non-fixture cleanup');
|
||||
const deleted = await context.request.delete(`${base}${spec.api}/${id}`);
|
||||
if (!deleted.ok()) throw Error('Fixture cleanup failed');
|
||||
const checked = await context.request.get(`${base}${spec.api}/${id}`);
|
||||
result.cleanup.record = checked.status() === 404 || (spec.family === 'documents' && (await checked.json()).is_active === false);
|
||||
}
|
||||
result.cleanup.sessions = true;
|
||||
for (const sid of [chat, seededSession].filter(Boolean)) {
|
||||
if (!(await context.request.delete(`${base}/api/session/${sid}`)).ok()) result.cleanup.sessions = false;
|
||||
}
|
||||
} catch { result.cleanup.error = true; }
|
||||
if (!result.cleanup.record || !result.cleanup.sessions) result.status = 'failed';
|
||||
save();
|
||||
}
|
||||
}
|
||||
} catch (error) { report.error = String(error).split('\n')[0].slice(0, 300); }
|
||||
finally { if (browser) await browser.close(); }
|
||||
report.status = report.cases.length === specs.length && report.cases.every(c => c.status === 'passed') ? 'passed' : 'failed';
|
||||
save();
|
||||
console.log(JSON.stringify({report: path.relative(root, reportPath), status: report.status, cases: report.cases}));
|
||||
if (report.status !== 'passed') process.exitCode = 1;
|
||||
Reference in New Issue
Block a user