jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-07 23:42:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

Validate clarification and correct context-test setup; record rejected prompt experiment

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-09-17 21:49:00 +00:00
parent cb305ca45e
commit a312e918e7
3 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/probe_search_synthesis.mjs
+5 -1
View File
@@ -30,7 +30,11 @@ const results = [];
const placements = (process.env.PLACEMENTS || 'user_evidence,tool_evidence,harness_system').split(',');
for (const placement of placements) {
const tracePlacement = ['trace_full', 'trace_no_controls'].includes(placement);
const messages = [placement === 'harness_system' || tracePlacement ? {role:'system',content:harnessSystem} : system, {role:'user', content:turn.prompt}];
const completenessSystem = harnessSystem.replace(
'Answer concisely, with useful source/note links when returned.',
'Answer every requested part using the available evidence. For research and comparisons, explain concrete findings, tradeoffs, and uncertainty with supporting source URLs. Distinguish source claims from your inferences and state unresolved conflicts. Do not fill evidence gaps with plausible details. Keep simple questions brief.'
);
const messages = [placement === 'harness_complete' ? {role:'system',content:completenessSystem} : placement === 'harness_system' || tracePlacement ? {role:'system',content:harnessSystem} : system, {role:'user', content:turn.prompt}];
if (tracePlacement) {
const trace = structuredClone(turn.runtime_trace || []);
if (!trace.length) throw Error('Native runtime trace required');
scripts/verify_clean_v3_search_quality.mjs
+1 -1
View File
@@ -136,7 +136,7 @@ try {
['no-web-ambiguous', ['can u look it up'], false],
['no-web-missing-object', ['please find that'], false],
['no-web-missing-price', ['what about its price?'], false],
['grounded-lookup-followup', ['Remember this topic for my next question: the Python release schedule. Do not search yet.', 'can u look it up'], [false, true]],
['grounded-lookup-followup', ['My next question is about the Python release schedule. For now, just acknowledge; do not search or save anything.', 'can u look it up'], [false, true]],
];
async function runCase([name, prompts, needsWeb]) {
const scenario = { name, status: 'running', turns: [] };