jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-06 15:02:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Record controlled missing-referent and schema-omission diagnostics

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-09-17 21:44:46 +00:00
parent e066b5d4dc
commit f8fc4ec4f7
2 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/probe_search_tool_choice.mjs
+5 -4
View File
@@ -6,7 +6,7 @@ const tools = JSON.parse(execFileSync((process.env.PYTHON || "python3"), ['-c',
'import json; from src.clean_agent_preview import compact_schemas; from src.tool_schemas import FUNCTION_TOOL_SCHEMAS; print(json.dumps(compact_schemas([s for s in FUNCTION_TOOL_SCHEMAS if s["function"]["name"] == "web_search"])))'], {encoding:'utf8'}));
const model = process.env.MODEL || 'model-f';
const endpoint = process.env.ENDPOINT_URL || (() => { throw new Error("ENDPOINT_URL is required"); })();
const prompts = ['Catch me up on the biggest AI developments this week. Explain why they matter and link your sources.', 'serch latest ai news pls'];
const prompts = process.env.PROBE_PROMPTS ? JSON.parse(process.env.PROBE_PROMPTS) : ['Catch me up on the biggest AI developments this week. Explain why they matter and link your sources.', 'serch latest ai news pls'];
const results = [];
let system = 'You are Odysseus. Use web_search to find current information relevant to the user request.';
if (process.env.CANONICAL_SYSTEM === '1') {
@@ -24,16 +24,17 @@ shell_clause = 'Shell commands are disabled. '
print(eval(compile(ast.Expression(assignment.value), '<canonical-system-expression>', 'eval')))
`], {input:fs.readFileSync('src/clean_agent_preview.py','utf8'),encoding:'utf8'}).trim();
}
system += process.env.SYSTEM_APPEND || '';
for (const prompt of prompts) {
for (const choice of ['auto', 'required', {type:'function', function:{name:'web_search'}}]) {
for (const choice of (process.env.PROBE_CHOICES ? JSON.parse(process.env.PROBE_CHOICES) : ['auto', 'required', {type:'function', function:{name:'web_search'}}])) {
const started = performance.now();
const response = await fetch(endpoint, {method:'POST', headers:{'Content-Type':'application/json'}, signal:AbortSignal.timeout(90000),
body:JSON.stringify({model, messages:[{role:'system',content:system},{role:'user',content:prompt}], tools, tool_choice:choice, temperature:0, max_tokens:256, stream:false, chat_template_kwargs:{enable_thinking:false}})});
body:JSON.stringify({model, messages:[{role:'system',content:system},{role:'user',content:prompt}], ...(process.env.OMIT_TOOLS === '1' ? {} : {tools, tool_choice:choice}), temperature:0, max_tokens:256, stream:false, chat_template_kwargs:{enable_thinking:false}})});
const data = await response.json();
const result = {prompt,choice,status:response.status,seconds:(performance.now()-started)/1000,message:data.choices?.[0]?.message,error:data.error};
results.push(result); console.log(JSON.stringify(result));
}
}
const target = `reports/search-tool-choice-probe-${Date.now()}.json`;
fs.writeFileSync(target, JSON.stringify({model,system,tools,results},null,2)+'\n');
fs.writeFileSync(target, JSON.stringify({model,system,tools_omitted:process.env.OMIT_TOOLS === '1',tools,results},null,2)+'\n');
console.log(target);