mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-07 15:32:21 +02:00
Alexandre Teixeira
39b8563213
Use Map-backed cost ledgers so externally derived session and run identifiers never cross ordinary object prototype semantics. Preserve the existing JSON storage format and extend browser and isolated ledger regressions for replay, overflow, legacy data, and reserved keys.
252 lines
12 KiB
Python
252 lines
12 KiB
Python
"""Saved Agent rounds must render and bill with actual per-round provenance."""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
|
|
import pytest
|
|
|
|
|
|
_SOURCE = (
|
|
Path(__file__).resolve().parents[1] / "static" / "js" / "chatRenderer.js"
|
|
).read_text(encoding="utf-8")
|
|
_CHAT_SOURCE = (
|
|
Path(__file__).resolve().parents[1] / "static" / "js" / "chat.js"
|
|
).read_text(encoding="utf-8")
|
|
_SLASH_SOURCE = (
|
|
Path(__file__).resolve().parents[1] / "static" / "js" / "slashCommands.js"
|
|
).read_text(encoding="utf-8")
|
|
_HAS_NODE = shutil.which("node") is not None
|
|
|
|
|
|
def _function_source(name):
|
|
match = re.search(
|
|
rf"^(?:export )?function {name}\(.*?^\}}",
|
|
_SOURCE,
|
|
re.MULTILINE | re.DOTALL,
|
|
)
|
|
assert match, f"{name} not found"
|
|
return match.group(0).replace("export function", "function", 1)
|
|
|
|
|
|
def _run_node(source):
|
|
proc = subprocess.run(
|
|
["node", "--input-type=module"],
|
|
input=source,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
assert proc.returncode == 0, proc.stderr
|
|
return json.loads(proc.stdout.strip())
|
|
|
|
|
|
def test_saved_agent_rounds_prefer_round_model_provenance():
|
|
assert "const roundModels = metadata.round_models || [];" in _SOURCE
|
|
assert "const contModel = roundModels[r] || pair.actualModel || pair.requestedModel;" in _SOURCE
|
|
assert "Array.isArray(metadata.round_texts) && metadata.round_texts.length > 1" in _SOURCE
|
|
assert "const roundEndpointIds = metadata.round_endpoint_ids || [];" in _SOURCE
|
|
assert "const roundEndpointLabels = metadata.round_endpoint_labels || [];" in _SOURCE
|
|
assert "r < roundEndpointIds.length" in _SOURCE
|
|
assert "r < roundEndpointLabels.length" in _SOURCE
|
|
assert "roundEndpointIds[r] || pair.actualEndpointId" not in _SOURCE
|
|
|
|
|
|
def test_metrics_cost_uses_actual_fallback_endpoint_classification():
|
|
assert "metrics.endpoint_cost_tracked" in _SOURCE
|
|
assert "endpointCostTracked === false" in _SOURCE
|
|
assert "endpointCostTracked !== true && !isCostTrackedEndpoint(selectedUrl)" in _SOURCE
|
|
assert "Array.isArray(metrics.usage_buckets)" in _SOURCE
|
|
|
|
|
|
@pytest.mark.skipif(not _HAS_NODE, reason="node binary not on PATH")
|
|
def test_agent_usage_buckets_sum_only_billable_answering_routes():
|
|
source = "\n".join([
|
|
"let currentUrl = '';",
|
|
"function _currentEndpointUrl() { return currentUrl; }",
|
|
"function isCostTrackedEndpoint(url) { return url === 'paid'; }",
|
|
"function getModelCost(_model, inputTokens, outputTokens) { return (inputTokens + outputTokens) / 1000; }",
|
|
_function_source("_billableCost"),
|
|
_function_source("_metricsBillableCost"),
|
|
"const paidSelected = {usage_buckets: [",
|
|
" {model: 'selected', input_tokens: 100, output_tokens: 10, endpoint_cost_tracked: true},",
|
|
" {model: 'local-fallback', input_tokens: 200, output_tokens: 20, endpoint_cost_tracked: false},",
|
|
"]};",
|
|
"const localSelected = {usage_buckets: [",
|
|
" {model: 'selected', input_tokens: 100, output_tokens: 10, endpoint_cost_tracked: false},",
|
|
" {model: 'paid-fallback', input_tokens: 200, output_tokens: 20, endpoint_cost_tracked: true},",
|
|
"]};",
|
|
"currentUrl = 'local';",
|
|
"const paidToLocal = _metricsBillableCost(paidSelected, 'final', 300, 30);",
|
|
"currentUrl = 'paid';",
|
|
"const localToPaid = _metricsBillableCost(localSelected, 'final', 300, 30);",
|
|
"console.log(JSON.stringify({paidToLocal, localToPaid}));",
|
|
])
|
|
|
|
assert _run_node(source) == {"paidToLocal": 0.11, "localToPaid": 0.22}
|
|
|
|
|
|
@pytest.mark.skipif(not _HAS_NODE, reason="node binary not on PATH")
|
|
def test_force_answer_synthesis_segment_is_included_in_fallback_cost():
|
|
source = "\n".join([
|
|
"function _currentEndpointUrl() { return 'local-selected'; }",
|
|
"function isCostTrackedEndpoint() { return false; }",
|
|
"function getModelCost(_model, inputTokens, outputTokens) { return (inputTokens + outputTokens) / 1000; }",
|
|
_function_source("_billableCost"),
|
|
_function_source("_metricsBillableCost"),
|
|
"const metrics = {usage_buckets: [",
|
|
" {round: 6, model: 'paid-fallback', input_tokens: 100, output_tokens: 0, endpoint_cost_tracked: true},",
|
|
" {round: 6, model: 'paid-fallback', input_tokens: 80, output_tokens: 20, endpoint_cost_tracked: true},",
|
|
"]};",
|
|
"console.log(JSON.stringify({cost: _metricsBillableCost(metrics, 'paid-fallback', 180, 20)}));",
|
|
])
|
|
|
|
assert _run_node(source) == {"cost": 0.2}
|
|
|
|
|
|
@pytest.mark.skipif(not _HAS_NODE, reason="node binary not on PATH")
|
|
def test_repeated_live_metrics_render_records_session_cost_once():
|
|
source = "\n".join([
|
|
"const _COST_KEY = 'ody-session-cost';",
|
|
"const state = {};",
|
|
"const localStorage = {",
|
|
" getItem(key) { return state[key] || null; },",
|
|
" setItem(key, value) { state[key] = value; },",
|
|
"};",
|
|
"const window = {sessionModule: {getCurrentSessionId() { return 'session'; }}};",
|
|
"function updateSessionCostUI() {}",
|
|
"function _currentEndpointUrl() { return 'local'; }",
|
|
"function isCostTrackedEndpoint(url) { return url === 'paid'; }",
|
|
"function getModelCost(_model, inputTokens, outputTokens) { return (inputTokens + outputTokens) / 1000; }",
|
|
_function_source("_billableCost"),
|
|
_function_source("_metricsBillableCost"),
|
|
_function_source("_readCostLedger"),
|
|
_function_source("_writeCostLedger"),
|
|
_function_source("recordSessionMetricsCost"),
|
|
"const metrics = {model: 'paid-model', input_tokens: 100, output_tokens: 10, endpoint_cost_tracked: true};",
|
|
"recordSessionMetricsCost(metrics);",
|
|
"recordSessionMetricsCost(metrics);",
|
|
"console.log(JSON.stringify({cost: JSON.parse(state[_COST_KEY]).session, recorded: metrics._costRecorded}));",
|
|
])
|
|
|
|
assert _run_node(source) == {"cost": 0.11, "recorded": True}
|
|
|
|
|
|
@pytest.mark.skipif(not _HAS_NODE, reason="node binary not on PATH")
|
|
def test_replayed_metrics_use_run_identity_for_durable_cost_deduplication():
|
|
source = "\n".join([
|
|
"const _COST_KEY = 'ody-session-cost';",
|
|
"const _COST_RUNS_KEY = 'ody-session-cost-runs';",
|
|
"const _MAX_COST_RUNS_PER_SESSION = 256;",
|
|
"const state = {};",
|
|
"const localStorage = {",
|
|
" getItem(key) { return state[key] || null; },",
|
|
" setItem(key, value) { state[key] = value; },",
|
|
"};",
|
|
"const window = {sessionModule: {getCurrentSessionId() { return 'session'; }}};",
|
|
"function updateSessionCostUI() {}",
|
|
"function _currentEndpointUrl() { return 'local'; }",
|
|
"function isCostTrackedEndpoint(url) { return url === 'paid'; }",
|
|
"function getModelCost(_model, inputTokens, outputTokens) { return (inputTokens + outputTokens) / 1000; }",
|
|
_function_source("_billableCost"),
|
|
_function_source("_metricsBillableCost"),
|
|
_function_source("_readCostLedger"),
|
|
_function_source("_writeCostLedger"),
|
|
_function_source("_readCostRunLedger"),
|
|
_function_source("_writeCostRunLedger"),
|
|
_function_source("recordSessionMetricsCost"),
|
|
_function_source("getSessionCost"),
|
|
"const firstObject = {model: 'paid-model', input_tokens: 100, output_tokens: 10, endpoint_cost_tracked: true, _costRecordId: 'run-1'};",
|
|
"const replayedObject = {...firstObject};",
|
|
"recordSessionMetricsCost(firstObject);",
|
|
"recordSessionMetricsCost(replayedObject);",
|
|
"console.log(JSON.stringify({cost: getSessionCost('session'), runs: JSON.parse(state[_COST_RUNS_KEY]).session}));",
|
|
])
|
|
|
|
assert _run_node(source) == {"cost": 0.11, "runs": {"run-1": 0.11}}
|
|
|
|
|
|
@pytest.mark.skipif(not _HAS_NODE, reason="node binary not on PATH")
|
|
def test_run_cost_ledger_sums_segments_and_updates_repeated_segment_metrics():
|
|
source = "\n".join([
|
|
"const _COST_KEY = 'ody-session-cost';",
|
|
"const _COST_RUNS_KEY = 'ody-session-cost-runs';",
|
|
"const _MAX_COST_RUNS_PER_SESSION = 256;",
|
|
"const state = {};",
|
|
"const localStorage = {",
|
|
" getItem(key) { return state[key] || null; },",
|
|
" setItem(key, value) { state[key] = value; },",
|
|
"};",
|
|
"const window = {sessionModule: {getCurrentSessionId() { return 'session'; }}};",
|
|
"function updateSessionCostUI() {}",
|
|
"function _currentEndpointUrl() { return 'paid'; }",
|
|
"function isCostTrackedEndpoint() { return true; }",
|
|
"function getModelCost(_model, inputTokens, outputTokens) { return (inputTokens + outputTokens) / 1000; }",
|
|
_function_source("_billableCost"),
|
|
_function_source("_metricsBillableCost"),
|
|
_function_source("_readCostLedger"),
|
|
_function_source("_writeCostLedger"),
|
|
_function_source("_readCostRunLedger"),
|
|
_function_source("_writeCostRunLedger"),
|
|
_function_source("recordSessionMetricsCost"),
|
|
_function_source("getSessionCost"),
|
|
"recordSessionMetricsCost({model: 'student', input_tokens: 100, output_tokens: 10, _costRecordId: 'run:primary'});",
|
|
"recordSessionMetricsCost({model: 'student', input_tokens: 120, output_tokens: 20, _costRecordId: 'run:primary'});",
|
|
"recordSessionMetricsCost({model: 'teacher', input_tokens: 200, output_tokens: 30, _costRecordId: 'run:teacher'});",
|
|
"console.log(JSON.stringify({cost: getSessionCost('session'), runs: JSON.parse(state[_COST_RUNS_KEY]).session}));",
|
|
])
|
|
|
|
assert _run_node(source) == {
|
|
"cost": 0.37,
|
|
"runs": {"run:primary": 0.14, "run:teacher": 0.23},
|
|
}
|
|
|
|
|
|
@pytest.mark.skipif(not _HAS_NODE, reason="node binary not on PATH")
|
|
def test_local_selected_endpoint_does_not_erase_paid_fallback_ledger():
|
|
source = "\n".join([
|
|
"const _COST_KEY = 'ody-session-cost';",
|
|
"const _COST_RUNS_KEY = 'ody-session-cost-runs';",
|
|
"const state = {'ody-session-cost': JSON.stringify({session: 0.125})};",
|
|
"const localStorage = {",
|
|
" getItem(key) { return state[key] || null; },",
|
|
" setItem(key, value) { state[key] = value; },",
|
|
"};",
|
|
"const badge = {style: {}, textContent: ''};",
|
|
"const document = {getElementById() { return badge; }};",
|
|
"const window = {sessionModule: {getCurrentSessionId() { return 'session'; }, getCurrentEndpointUrl() { return 'local'; }}};",
|
|
_function_source("_readCostLedger"),
|
|
_function_source("_writeCostLedger"),
|
|
_function_source("_readCostRunLedger"),
|
|
_function_source("_writeCostRunLedger"),
|
|
_function_source("getSessionCost"),
|
|
_function_source("updateSessionCostUI"),
|
|
"updateSessionCostUI();",
|
|
"console.log(JSON.stringify({stored: JSON.parse(state[_COST_KEY]).session, display: badge.style.display, text: badge.textContent}));",
|
|
])
|
|
|
|
assert _run_node(source) == {
|
|
"stored": 0.125,
|
|
"display": "",
|
|
"text": "$0.125",
|
|
}
|
|
|
|
|
|
def test_live_and_resumed_terminal_events_apply_usage_metrics_before_reload():
|
|
assert "metrics = json.data || metrics;" in _CHAT_SOURCE
|
|
assert "displayMetrics(terminalMetricsTarget, metrics);" in _CHAT_SOURCE
|
|
assert "metricsData = json.data || metricsData;" in _CHAT_SOURCE
|
|
assert "displayMetrics(holder, metricsData);" in _CHAT_SOURCE
|
|
assert "json.type === 'agent_terminal' || json.type === 'chat_terminal'" in _CHAT_SOURCE
|
|
assert "chatRenderer.recordSessionMetricsCost(metrics, streamSessionId);" in _CHAT_SOURCE
|
|
assert "chatRenderer.recordSessionMetricsCost(metricsData, sessionId);" in _CHAT_SOURCE
|
|
assert "metricsData._costRecordId = _metricsCostRecordId(resumeRunId, json);" in _CHAT_SOURCE
|
|
assert "bgTerminal.status = 'completed';" in _CHAT_SOURCE
|
|
|
|
|
|
def test_usage_command_does_not_hide_existing_fallback_cost_for_local_selection():
|
|
assert "const cost = chatRenderer.getSessionCost" in _SLASH_SOURCE
|
|
assert "const cost = costTracked && chatRenderer.getSessionCost" not in _SLASH_SOURCE
|