diff --git a/scripts/verify_clean_v3_search_quality.mjs b/scripts/verify_clean_v3_search_quality.mjs index 5e69033e0..05010fb76 100644 --- a/scripts/verify_clean_v3_search_quality.mjs +++ b/scripts/verify_clean_v3_search_quality.mjs @@ -20,7 +20,7 @@ const token = Object.entries(sessions).find(([, value]) => value?.username === o if (!token) throw Error(`No active ${owner} session`); const marker = `ody-search-${crypto.randomUUID()}`; -const report = { run, owner, marker, model, endpointUrl, status: 'running', scenarios: [], turns: [], privacy: 'Public synthetic queries only; fetched bodies and private data are not retained.' }; +const report = { run, owner, marker, model, endpointUrl, status: 'running', scenarios: [], turns: [], privacy: 'Public test queries and bounded public tool evidence; no private account data.' }; const save = () => fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n'); fs.mkdirSync(path.dirname(reportPath), { recursive: true }); save(); const canonical = value => String(value || '').replace(/^mcp__email__/, ''); @@ -68,6 +68,10 @@ async function send(page, prompt) { rounds: metrics?.agent_rounds ?? null, actual_model: metrics?.model ?? null, tools: starts, outputs, final, + evidence: events.filter(event => event.type === 'tool_output').map(event => ({ + tool: canonical(event.tool), arguments: event.command, + output: String(event.output || '').slice(0, 10000), error: Boolean(event.error), + })), runtime_error: events.some(event => event.type === 'error') || /v3 test encountered an error/i.test(final), }; report.turns.push(observation); save(); @@ -90,6 +94,14 @@ try { ['research', ['How do sodium ion batteries compare with lithium ion for home storage? Find evidence and explain tradeoffs.'], true], ['no-web-typo', ['whats 12 tims 7'], false], ['no-web-greeting', ['helo'], false], + ['news-short', ['ai news today'], true], + ['news-natural', ['Catch me up on the biggest AI developments this week. Explain why they matter and link your sources.'], true], + ['research-typo', ['reserch sodium ion vs lithium batterys for home stroage. whats the tradeof? sources pls'], true], + ['official-domain', ['latest Python stable release? use only python.org sources'], true], + ['context-refinement', ['Find current Firefox privacy documentation from Mozilla.', 'How does that compare with Chrome? Find official sources for that too.'], true], + ['evidence-reuse', ['Find the official Python release page.', 'Explain what you found in plain English. Do not search again.'], [true, false]], + ['no-web-rewrite', ['fix spelling: i recieved the calender invte'], false], + ['no-web-compound', ['helo can u explain what a web browser is? no search needed'], false], ]; async function runCase([name, prompts, needsWeb]) { const scenario = { name, status: 'running', turns: [] }; @@ -98,7 +110,8 @@ try { try { id = await createSession(context, `[search-variety] ${name} ${marker}`); page = await preparePage(context, id); - for (const prompt of prompts) { + for (const [turnIndex, prompt] of prompts.entries()) { + const turnNeedsWeb = Array.isArray(needsWeb) ? needsWeb[turnIndex] : needsWeb; const turn = await send(page, prompt); const tools = turn.starts.map(x => x.tool); const checks = { @@ -106,7 +119,7 @@ try { model_matches: turn.actual_model === model, nonempty_answer: turn.final.trim().length > 0, no_reasoning_leak: noLeak(turn.final), - expected_web_use: needsWeb ? tools.some(x => ['web_search', 'web_fetch', 'private_browser'].includes(x)) : tools.length === 0, + expected_web_use: turnNeedsWeb ? tools.some(x => ['web_search', 'web_fetch', 'private_browser'].includes(x)) : tools.length === 0, }; scenario.turns.push({ prompt, checks, seconds: turn.seconds, tools, status: Object.values(checks).every(Boolean) ? 'mechanics_passed' : 'failed', @@ -121,7 +134,8 @@ try { } } // Two simultaneous conversations keep endpoint contention bounded. - const queue = [...cases]; + const selected = new Set((process.env.CASES || '').split(',').filter(Boolean)); + const queue = cases.filter(([name]) => !selected.size || selected.has(name)); await Promise.all([0, 1].map(async () => { while (queue.length) await runCase(queue.shift()); })); } else { diff --git a/services/search/core.py b/services/search/core.py index 62dea902d..cafe3a7a2 100644 --- a/services/search/core.py +++ b/services/search/core.py @@ -173,6 +173,10 @@ _EMPTY_RESULT_RELAXATION_TERMS = { def _relaxed_query_after_empty(query: str) -> str: """Remove request scaffolding once an exact provider query returns nothing.""" + # Token-based relaxation cannot preserve search operators, quoted phrases, + # or exclusions. Do not silently broaden an explicit source constraint. + if re.search(r'\b\w+:|["\u201c\u201d]|(?:^|\s)-\S', str(query or "")): + return "" tokens = re.findall(r"[A-Za-z0-9][A-Za-z0-9_.+-]*", str(query or "")) retained = [ token for token in tokens @@ -281,12 +285,40 @@ def _result_has_query_overlap(query: str, result: dict) -> bool: def _filter_low_relevance_results(query: str, results: list[dict]) -> list[dict]: if not results: return [] - relevant = [result for result in results if _result_has_query_overlap(query, result)] + relevant = [result for result in results + if _result_matches_site_scope(query, result) + and _result_has_query_overlap(query, result)] # Only reject a provider when it returned a fully off-topic page set. Mixed # result pages are common; ranking can handle those. return relevant if relevant else [] +def _result_matches_site_scope(query: str, result: dict) -> bool: + """Enforce explicit site constraints even when a provider ignores them.""" + scopes = re.findall(r'(?