jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-10 00:42:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Squash Odysseus development history

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-09-11 06:04:19 +00:00
parent e5c99a5eee
commit 6ee6502010
2050 changed files with 538359 additions and 57745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/verify_clean_v3_vl.mjs
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env node
/** Real 7011 image attachment -> answer -> reload -> image follow-up check. */
import fs from 'node:fs';
import path from 'node:path';
import { chromium } from 'playwright';
const root = path.resolve(new URL('..', import.meta.url).pathname);
const base = 'http://127.0.0.1:7011';
const owner = 'sft_alex_creator';
const routingMode = 'recent_model_choice';
const endpointId = process.env.ENDPOINT_ID || '1d1022ef';
const endpointUrl = process.env.ENDPOINT_URL || 'http://100.67.207.85:19184/v1/chat/completions';
const fixture = path.resolve(process.env.FIXTURE_PATH || path.join(root, 'tests/fixtures/vl/basic-shapes.png'));
const reportPath = process.env.REPORT_PATH
? path.resolve(process.env.REPORT_PATH)
: path.join(root, `reports/clean-v3-vl-live-${new Date().toISOString().replace(/[:.]/g, '-')}.json`);
if (!reportPath.startsWith(path.join(root, 'reports') + path.sep)) throw Error('Report must be under reports/');
if (fs.existsSync(reportPath)) throw Error('Report exists; refuse overwrite');
const authSessions = JSON.parse(fs.readFileSync('/home/pewds/odysseus-cookbook-fresh/data/sessions.json'));
const token = Object.entries(authSessions).find(([, value]) => value?.username === owner)?.[0];
if (!token) throw Error('Dedicated SFT account has no active auth session');
const report = { status: 'running', owner, fixture: path.relative(root, fixture), turns: [], checks: {}, cleanup: null };
const save = () => fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n');
let browser, context, session;
const parseEvents = async response => (await response.text())
.split(/\r?\n\r?\n/)
.filter(line => line.startsWith('data: ') && line.slice(6) !== '[DONE]')
.map(line => JSON.parse(line.slice(6)));
try {
browser = await chromium.launch({ headless: true, args: ['--no-proxy-server'] });
context = await browser.newContext({ serviceWorkers: 'block',
...(process.env.MOBILE === 'true' ? { viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true } : {}),
extraHTTPHeaders: { 'x-odysseus-routing-experiment': routingMode } });
await context.addCookies([{ name: 'odysseus_session', value: token, url: base }]);
const created = await context.request.post(`${base}/api/session`, { multipart: {
name: '[clean-v3-vl] basic shapes',
model: 'odysseus-qwen3.5-tools-pre-heretic',
endpoint_id: endpointId,
endpoint_url: endpointUrl,
skip_validation: 'true', rag: 'false',
}});
if (!created.ok()) throw Error(`session create ${created.status()}`);
session = (await created.json()).id;
report.session = session; save();
const page = await context.newPage();
await page.goto(`${base}/#${session}`, { waitUntil: 'domcontentloaded' });
await page.waitForFunction(id => window.__odysseusSessionReadyId === id, session);
const agent = page.locator('#mode-agent-btn');
if (await agent.getAttribute('aria-pressed') !== 'true') await agent.click();
const send = async prompt => {
const responsePromise = page.waitForResponse(r => new URL(r.url()).pathname === '/api/chat_stream' && r.request().method() === 'POST', { timeout: 90000 });
await page.locator('textarea#message:visible').fill(prompt);
await page.locator('textarea#message:visible').press('Enter');
const response = await responsePromise;
const events = await parseEvents(response);
const text = events.filter(x => typeof x.delta === 'string').map(x => x.delta).join('');
const final = events.filter(x => x.type === 'final_response').map(x => x.content || '').join('') || text;
const contract = events.find(x => x.type === 'turn_contract');
if (contract?.routing_experiment !== routingMode) throw Error('Wrong model-specific runtime');
const turn = {
prompt, http: response.status(), selection_mode: contract?.selection_mode,
image_context_count: contract?.multimodal_image_count,
image_rehydration: contract?.image_rehydration,
final, tools: events.filter(x => x.type === 'tool_output').map(x => ({ tool: x.tool, exit_code: x.exit_code, error: x.error })),
};
report.turns.push(turn); save();
return turn;
};
await page.locator('#file-input').setInputFiles(fixture);
if (process.env.MOBILE === 'true') {
// Mobile intentionally asks the user to crop or keep the original first.
await page.locator('.attach-crop-overlay [data-action="original"]').click();
}
await page.locator('#attach-strip .thumb').waitFor({ state: 'visible' }).catch(async error => {
report.attachment_diagnostics = await page.evaluate(() => ({
strip_count: document.querySelectorAll('#attach-strip').length,
thumb_count: document.querySelectorAll('#attach-strip .thumb').length,
strip_display: document.querySelector('#attach-strip') && getComputedStyle(document.querySelector('#attach-strip')).display,
body_classes: document.body.className,
}));
await page.screenshot({ path: reportPath.replace(/\.json$/, '.png') });
throw error;
});
const first = await send('Read the image. State the exact heading and describe the left and right shapes with their colors.');
const firstText = first.final.toLowerCase();
if (first.selection_mode !== 'clean_compact_v3_preview') throw Error('First turn did not use clean v3');
report.checks.first_turn_route = true;
report.checks.ocr = firstText.includes('odysseus 42');
report.checks.visual_objects = ['red', 'circle', 'blue', 'square'].every(required => firstText.includes(required));
if (!report.checks.visual_objects) throw Error('First answer missed one or more visual objects');
await page.reload({ waitUntil: 'domcontentloaded' });
await page.waitForFunction(id => window.__odysseusSessionReadyId === id, session);
await page.waitForFunction(() => document.querySelectorAll('#chat-history .msg').length >= 2);
const second = await send('What color was the shape on the right?');
if (second.selection_mode !== 'clean_compact_v3_preview') throw Error('Follow-up did not use clean v3');
report.checks.reload_followup_route = true;
report.checks.reload_followup_grounding = /\bblue\b/i.test(second.final);
if (!report.checks.reload_followup_grounding) throw Error('Image follow-up was not grounded in the prior image');
const third = await send('Look at the original image again very carefully. What exact letters and number are in the heading?');
if (third.selection_mode !== 'clean_compact_v3_preview') throw Error('OCR retry did not use clean v3');
report.checks.ocr_retry_route = true;
report.checks.ocr_retry_grounding = /odysseus\s*42/i.test(third.final);
const fourth = await send('Use the OCR tool to extract the heading text from the attached image, not its filename.');
report.checks.explicit_ocr_called = fourth.tools.some(tool => tool.tool === 'extract_text' && tool.exit_code === 0);
report.checks.explicit_ocr_grounding = /odysseus\s*42/i.test(fourth.final);
const fifth = await send('Run OCR on that same image again, but return only the number this time.');
report.checks.numeric_ocr_called = fifth.tools.some(tool => tool.tool === 'extract_text' && tool.exit_code === 0);
report.checks.numeric_ocr_grounding = /\b42\b/.test(fifth.final);
report.checks.no_tool_errors = report.turns.every(turn => turn.tools.every(tool => !tool.error && tool.exit_code === 0));
report.status = Object.values(report.checks).every(Boolean) ? 'passed' : 'partial';
} catch (error) {
report.status = 'failed';
report.error = `${error.name}: ${error.message}`;
} finally {
if (session && context) {
const removed = await context.request.delete(`${base}/api/session/${encodeURIComponent(session)}`);
report.cleanup = { session, status: removed.status(), removed: removed.ok() };
if (!report.cleanup.removed) report.status = 'failed';
}
save();
if (browser) await browser.close();
}
console.log(JSON.stringify({ status: report.status, turns: report.turns.map(t => ({ mode: t.selection_mode, final: t.final })), cleanup: report.cleanup, error: report.error }));
if (report.status !== 'passed') process.exitCode = 1;