jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-17 12:22:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add matched native-trace recovery-history synthesis probes

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-09-17 21:26:23 +00:00
parent 7eb4eb6346
commit 71784cd16f
1 changed files with 15 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/probe_search_synthesis.mjs
+15 -2
View File
@@ -29,8 +29,20 @@ print(eval(compile(ast.Expression(assignment.value), '<canonical-system-expressi
const results = []; const results = [];
const placements = (process.env.PLACEMENTS || 'user_evidence,tool_evidence,harness_system').split(','); const placements = (process.env.PLACEMENTS || 'user_evidence,tool_evidence,harness_system').split(',');
for (const placement of placements) { for (const placement of placements) {
const messages = [placement === 'harness_system' ? {role:'system',content:harnessSystem} : system, {role:'user', content:turn.prompt}]; const tracePlacement = ['trace_full', 'trace_no_controls'].includes(placement);
if (placement === 'user_evidence') { const messages = [placement === 'harness_system' || tracePlacement ? {role:'system',content:harnessSystem} : system, {role:'user', content:turn.prompt}];
if (tracePlacement) {
const trace = structuredClone(turn.runtime_trace || []);
if (!trace.length) throw Error('Native runtime trace required');
// Remove only the recorded final answer. Both variants keep identical
// successful/failed tool observations and earlier assistant messages.
if (trace.at(-1)?.role === 'assistant' && !trace.at(-1)?.tool_calls?.length) trace.pop();
for (const message of trace) {
if (placement === 'trace_no_controls' && message._harness_control) continue;
const {role, content, tool_calls, tool_call_id} = message;
messages.push({role, content, ...(tool_calls ? {tool_calls} : {}), ...(tool_call_id ? {tool_call_id} : {})});
}
} else if (placement === 'user_evidence') {
messages[1].content += '\n\nSEARCH EVIDENCE:\n' + evidence.map(x => x.output).join('\n\n'); messages[1].content += '\n\nSEARCH EVIDENCE:\n' + evidence.map(x => x.output).join('\n\n');
} else { } else {
for (const [i, item] of evidence.entries()) { for (const [i, item] of evidence.entries()) {
@@ -47,6 +59,7 @@ for (const placement of placements) {
if (!response.ok) throw Error(`Endpoint HTTP ${response.status}`); if (!response.ok) throw Error(`Endpoint HTTP ${response.status}`);
const data = await response.json(); const data = await response.json();
const result = {placement,seconds:(performance.now()-started)/1000, const result = {placement,seconds:(performance.now()-started)/1000,
message_count:messages.length,
answer:data.choices[0].message.content,finish_reason:data.choices[0].finish_reason,usage:data.usage}; answer:data.choices[0].message.content,finish_reason:data.choices[0].finish_reason,usage:data.usage};
results.push(result); console.log(JSON.stringify(result)); results.push(result); console.log(JSON.stringify(result));
} }