mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-11-03 11:42:23 +01:00
refactor(skills): extract evaluation and audit orchestration (#6711)
* refactor(skills): extract evaluation and audit orchestration * chore(security): scope OpenAPI digest Gitleaks exception * refactor(skills): establish domain-oriented service boundaries * refactor(skills): address architecture review feedback * fix(skills): restore prompt seams and harden signature contracts * test(skills): restore ReDoS coverage and simplify API baseline
This commit is contained in:
Alexandre Teixeira
@@ -0,0 +1,439 @@
|
||||
"""Sequential skill audit coordination and transient owner-scoped jobs."""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from services.memory.skills import SkillsManager
|
||||
from .dependencies import AuditDependencies
|
||||
|
||||
from .agent_runner import SkillAuditUnavailable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_skill_audit_jobs: dict = {}
|
||||
|
||||
|
||||
async def audit_one_skill(skills_manager, skill, url, model, headers,
|
||||
teacher, owner, log, workload: str = "foreground", *, deps: AuditDependencies) -> dict:
|
||||
"""Test → judge → self-edit+retry → (teacher edit+retry) → flag. Never deletes;
|
||||
a skill the teacher still can't fix is demoted to draft for manual review.
|
||||
`teacher` is (url, model, headers) or None. `log(msg)` records progress."""
|
||||
name = skill.get("name")
|
||||
|
||||
# Reflect the audit outcome in the skill's confidence so the main list
|
||||
# updates: a clean pass earns high confidence; a pass that needed fixing
|
||||
# earns a bit less; a skill that still fails is marked low.
|
||||
def _set_conf(c):
|
||||
try:
|
||||
skills_manager.update_skill(name, {"confidence": c}, owner=owner)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
md = skills_manager.read_skill_md(name, owner=owner)
|
||||
if not md:
|
||||
log(f"{name}: no source — skipped")
|
||||
return {"skill": name, "result": "skipped"}
|
||||
|
||||
# Cheap deterministic cleanup first. If this is an obvious lower-priority
|
||||
# duplicate, do not spend LLM turns on necessity, retrieval precision,
|
||||
# skill-vs-baseline testing, self-edit, or teacher escalation.
|
||||
duplicate_of = deps.duplicate_blocker(skills_manager, name, owner)
|
||||
if duplicate_of:
|
||||
reason = f"Lower-priority duplicate of {duplicate_of}"
|
||||
try:
|
||||
skills_manager.update_skill(name, {"status": "draft", "confidence": 0.35}, owner=owner)
|
||||
skills_manager.set_audit(name, "skipped", by_teacher=False, worker_model=model, owner=owner)
|
||||
skills_manager.set_necessity(name, False, [duplicate_of], reason, owner=owner)
|
||||
except Exception:
|
||||
pass
|
||||
log(f"{name}: draft — skipped audit ({reason[:100]})")
|
||||
return {"skill": name, "result": "skipped_duplicate", "reason": reason, "confidence": 0.35, "status": "draft"}
|
||||
|
||||
# Advisory necessity/redundancy check — runs once, independent of the test
|
||||
# outcome, and only records a flag the UI surfaces (never deletes/demotes).
|
||||
others = []
|
||||
nec = None
|
||||
try:
|
||||
# Only compare against skills the SAME owner can see, so the necessity
|
||||
# judge never sees (or flags "redundant_with") another user's skills.
|
||||
sk_owner = skill.get("owner")
|
||||
others = [
|
||||
{"name": s.get("name"), "description": s.get("description", "")}
|
||||
for s in skills_manager.load(owner=owner)
|
||||
if s.get("name") and s.get("name") != name
|
||||
and (not sk_owner or not s.get("owner") or s.get("owner") == sk_owner)
|
||||
]
|
||||
nec = await deps.evaluate_necessity(
|
||||
md, others, url, model, headers, workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
if nec is not None:
|
||||
skills_manager.set_necessity(name, nec.get("necessary", True),
|
||||
nec.get("redundant_with"), nec.get("reason"),
|
||||
owner=owner)
|
||||
if not nec.get("necessary", True):
|
||||
log(f"{name}: possibly unnecessary — {nec.get('reason', '')[:80]}")
|
||||
except Exception as e:
|
||||
log(f"{name}: necessity check skipped — {e}")
|
||||
|
||||
# Retrieval precision check: if broad tags/trigger text would make this
|
||||
# narrow skill over-inject, fix only metadata before the functional test.
|
||||
try:
|
||||
if deps.should_check_retrieval_precision(skill):
|
||||
rp = await deps.evaluate_retrieval_precision(
|
||||
md, others, url, model, headers, workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
if rp and not rp.get("ok"):
|
||||
issues = rp.get("issues") or ["metadata: retrieval: narrow tags and when_to_use to the intended trigger"]
|
||||
log(f"{name}: narrowing retrieval metadata — {(rp.get('summary') or issues[0])[:80]}")
|
||||
fixed = await deps.improve_markdown(md, {
|
||||
"verdict": "pass",
|
||||
"confidence": 1.0,
|
||||
"summary": rp.get("summary") or "Retrieval metadata is too broad.",
|
||||
"issues": issues,
|
||||
}, "Retrieval audit only: the procedure may work, but matching metadata is too broad.",
|
||||
url, model, headers, workload=workload, owner=owner)
|
||||
if fixed and fixed.strip() != md.strip() and deps.apply_markdown(skills_manager, name, fixed, owner):
|
||||
md = fixed
|
||||
refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None)
|
||||
if refreshed:
|
||||
skill = refreshed
|
||||
except Exception as e:
|
||||
log(f"{name}: retrieval precision check skipped — {e}")
|
||||
|
||||
task = deps.test_task(skill)
|
||||
log(f"{name}: testing…")
|
||||
skill_messages = deps.test_messages(md, task)
|
||||
transcript, skill_stats, approval_required = await deps.run_arm(
|
||||
skill_messages,
|
||||
url,
|
||||
model,
|
||||
headers,
|
||||
owner,
|
||||
workload=workload,
|
||||
)
|
||||
if approval_required is not None:
|
||||
try:
|
||||
from src.tool_approvals import tool_approval_store
|
||||
tool_approval_store.consume(
|
||||
approval_required.get("approval_id"),
|
||||
decision="deny",
|
||||
owner=owner,
|
||||
session_id=None,
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Could not retire unattended skill approval", exc_info=True)
|
||||
verdict = {
|
||||
"verdict": "inconclusive",
|
||||
"confidence": 1.0,
|
||||
"summary": (
|
||||
"This automated audit reached an exact action that requires "
|
||||
"a human approval; no action was executed."
|
||||
),
|
||||
"issues": [
|
||||
"Run this skill's manual test and review the sealed action."
|
||||
],
|
||||
"approval_required": True,
|
||||
}
|
||||
else:
|
||||
baseline_task = task
|
||||
log(f"{name}: running no-skill baseline…")
|
||||
baseline_transcript, baseline_stats, baseline_approval = await deps.run_arm(
|
||||
deps.baseline_messages(baseline_task),
|
||||
url,
|
||||
model,
|
||||
headers,
|
||||
owner,
|
||||
workload=workload,
|
||||
)
|
||||
if baseline_approval is not None:
|
||||
try:
|
||||
from src.tool_approvals import tool_approval_store
|
||||
tool_approval_store.consume(
|
||||
baseline_approval.get("approval_id"),
|
||||
decision="deny",
|
||||
owner=owner,
|
||||
session_id=None,
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Could not retire unattended baseline approval", exc_info=True)
|
||||
verdict = await deps.evaluate_run(
|
||||
md,
|
||||
task,
|
||||
transcript,
|
||||
url,
|
||||
model,
|
||||
headers,
|
||||
baseline_transcript=baseline_transcript,
|
||||
skill_stats=skill_stats,
|
||||
baseline_stats=baseline_stats,
|
||||
workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
v = verdict.get("verdict")
|
||||
log(f"{name}: verdict = {v} ({verdict.get('summary', '')[:80]})")
|
||||
if verdict.get("approval_required"):
|
||||
# An unattended audit is not authority for an action influenced by the
|
||||
# skill under test. Preserve the skill's current publication/confidence
|
||||
# state and route the exact action to the manual test UI instead of
|
||||
# letting a safety pause demote, rewrite, or auto-publish the skill.
|
||||
skills_manager.set_audit(
|
||||
name,
|
||||
"inconclusive",
|
||||
by_teacher=False,
|
||||
worker_model=model,
|
||||
owner=owner,
|
||||
audit_summary=verdict.get("summary") or "The test requires approval for an external action.",
|
||||
)
|
||||
status = skill.get("status") or "draft"
|
||||
log(f"{name}: {status} unchanged — exact action needs manual approval")
|
||||
return {
|
||||
"skill": name,
|
||||
"result": "approval_required",
|
||||
"verdict": verdict,
|
||||
"status": status,
|
||||
}
|
||||
if v == "pass":
|
||||
# Procedure works. If the reviewer still flagged metadata (tags/category/
|
||||
# when_to_use/description), do ONE fixer pass to correct the frontmatter
|
||||
# without re-testing — a metadata-only fix can't break a passing run.
|
||||
meta_issues = [i for i in (verdict.get("issues") or []) if str(i).lower().lstrip().startswith("metadata:")]
|
||||
if meta_issues:
|
||||
log(f"{name}: pass, but fixing {len(meta_issues)} metadata issue(s)…")
|
||||
fixed = await deps.improve_markdown(
|
||||
md, verdict, transcript, url, model, headers, workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
if fixed and fixed.strip() != md.strip():
|
||||
deps.apply_markdown(skills_manager, name, fixed, owner)
|
||||
_set_conf(0.95)
|
||||
skills_manager.set_audit(
|
||||
name,
|
||||
"pass",
|
||||
by_teacher=False,
|
||||
worker_model=model,
|
||||
owner=owner,
|
||||
**deps.verdict_efficiency(verdict),
|
||||
)
|
||||
refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None)
|
||||
status = deps.finalize_status(skills_manager, name, owner, "pass", 0.95, (refreshed or {}).get("necessity"), verdict)
|
||||
log(f"{name}: {status} — confidence 95%")
|
||||
return {"skill": name, "result": "pass", "verdict": verdict, "confidence": 0.95, "status": status}
|
||||
if v in ("unknown", "inconclusive"):
|
||||
skills_manager.set_audit(
|
||||
name,
|
||||
"inconclusive",
|
||||
by_teacher=False,
|
||||
worker_model=model,
|
||||
owner=owner,
|
||||
**deps.verdict_efficiency(verdict),
|
||||
)
|
||||
status = deps.finalize_status(skills_manager, name, owner, "inconclusive", skill.get("confidence") or 0.0, skill.get("necessity"))
|
||||
log(f"{name}: {status} — inconclusive")
|
||||
return {"skill": name, "result": "inconclusive", "verdict": verdict, "status": status}
|
||||
|
||||
# Self-edit + retry.
|
||||
log(f"{name}: self-editing to fix issues…")
|
||||
new_md = await deps.improve_markdown(
|
||||
md, verdict, transcript, url, model, headers, workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
if new_md and new_md.strip() != md.strip() and deps.apply_markdown(skills_manager, name, new_md, owner):
|
||||
md = new_md
|
||||
transcript, verdict = await deps.run_test_once(
|
||||
md, task, url, model, headers, owner, workload=workload,
|
||||
)
|
||||
v = verdict.get("verdict")
|
||||
log(f"{name}: retry (self) = {v}")
|
||||
if v == "pass":
|
||||
_set_conf(0.85)
|
||||
skills_manager.set_audit(
|
||||
name,
|
||||
"pass",
|
||||
by_teacher=False,
|
||||
worker_model=model,
|
||||
owner=owner,
|
||||
**deps.verdict_efficiency(verdict),
|
||||
)
|
||||
refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None)
|
||||
status = deps.finalize_status(skills_manager, name, owner, "pass", 0.85, (refreshed or {}).get("necessity"), verdict)
|
||||
log(f"{name}: {status} — confidence 85% after self-edit")
|
||||
return {"skill": name, "result": "pass_after_self_edit", "verdict": verdict, "confidence": 0.85, "status": status}
|
||||
|
||||
# Teacher escalation (if a distinct teacher model is configured). The
|
||||
# teacher only REWRITES the skill — it does NOT run the test. The point is
|
||||
# to verify the regular (student) model can now succeed with the teacher's
|
||||
# improved procedure, so the retry runs on the worker model, not the teacher.
|
||||
teacher_ran = False
|
||||
if teacher and teacher[0] and teacher[1] and (teacher[1] != model or teacher[0] != url):
|
||||
teacher_ran = True
|
||||
t_url, t_model, t_headers = teacher
|
||||
log(f"{name}: teacher {t_model} rewriting the skill…")
|
||||
t_md = await deps.improve_markdown(
|
||||
md, verdict, transcript, t_url, t_model, t_headers, workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
if t_md and t_md.strip() != md.strip() and deps.apply_markdown(skills_manager, name, t_md, owner):
|
||||
md = t_md
|
||||
# Re-test with the STUDENT model (the model the skill runs under in use).
|
||||
transcript, verdict = await deps.run_test_once(
|
||||
md, task, url, model, headers, owner, workload=workload,
|
||||
)
|
||||
v = verdict.get("verdict")
|
||||
log(f"{name}: retry on student after teacher rewrite = {v}")
|
||||
if v == "pass":
|
||||
_set_conf(0.8)
|
||||
skills_manager.set_audit(
|
||||
name,
|
||||
"pass",
|
||||
by_teacher=True,
|
||||
worker_model=model,
|
||||
teacher_model=t_model,
|
||||
owner=owner,
|
||||
**deps.verdict_efficiency(verdict),
|
||||
)
|
||||
refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None)
|
||||
status = deps.finalize_status(skills_manager, name, owner, "pass", 0.8, (refreshed or {}).get("necessity"), verdict)
|
||||
log(f"{name}: {status} — confidence 80% after teacher rewrite")
|
||||
return {"skill": name, "result": "pass_after_teacher", "verdict": verdict, "confidence": 0.8, "status": status}
|
||||
|
||||
# Still failing → demote to draft + low confidence + flag (do NOT delete).
|
||||
try:
|
||||
skills_manager.update_skill(name, {"status": "draft", "confidence": 0.35}, owner=owner)
|
||||
except Exception:
|
||||
pass
|
||||
skills_manager.set_audit(
|
||||
name, v or "fail", by_teacher=teacher_ran,
|
||||
worker_model=model,
|
||||
teacher_model=(teacher[1] if teacher_ran and teacher else ""),
|
||||
owner=owner,
|
||||
**deps.verdict_efficiency(verdict),
|
||||
)
|
||||
log(f"{name}: flagged — confidence lowered, kept as draft for manual review")
|
||||
return {"skill": name, "result": "flagged", "verdict": verdict, "confidence": 0.35}
|
||||
|
||||
|
||||
async def run_audit_batch(key, skills_manager, names, url, model, headers, teacher, owner,
|
||||
workload: str = "foreground", *, deps: AuditDependencies):
|
||||
"""Background: audit each named skill in sequence, recording progress."""
|
||||
import asyncio as _asyncio
|
||||
import time as _time
|
||||
|
||||
job = deps.audit_jobs.get(key)
|
||||
if job is None:
|
||||
return
|
||||
|
||||
def log(msg):
|
||||
job["log"].append(msg)
|
||||
if len(job["log"]) > 1000:
|
||||
del job["log"][0:len(job["log"]) - 1000]
|
||||
|
||||
cancelled = False
|
||||
try:
|
||||
for nm in names:
|
||||
if job.get("cancel"):
|
||||
cancelled = True
|
||||
log("(cancelled)")
|
||||
break
|
||||
job["current"] = nm
|
||||
skills = skills_manager.load(owner=owner)
|
||||
sk = next((s for s in skills if s.get("name") == nm), None)
|
||||
if not sk:
|
||||
continue
|
||||
try:
|
||||
res = await deps.audit_one(
|
||||
skills_manager, sk, url, model, headers, teacher, owner, log,
|
||||
workload=workload,
|
||||
)
|
||||
except _asyncio.CancelledError:
|
||||
cancelled = True
|
||||
job["cancel"] = True
|
||||
log("(cancelled)")
|
||||
raise
|
||||
except SkillAuditUnavailable as e:
|
||||
job["unavailable"] = str(e)
|
||||
log(f"Audit paused: {e}. Skill verdicts unchanged; retry when the model is available.")
|
||||
break
|
||||
except Exception as e:
|
||||
log(f"{nm}: error — {e}")
|
||||
res = {"skill": nm, "result": "error"}
|
||||
skills_manager.set_audit(nm, "inconclusive", worker_model=model, owner=owner)
|
||||
try:
|
||||
refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == nm), None)
|
||||
if refreshed:
|
||||
res["skill_state"] = {
|
||||
"name": refreshed.get("name"),
|
||||
"status": refreshed.get("status"),
|
||||
"confidence": refreshed.get("confidence"),
|
||||
"audit_verdict": refreshed.get("audit_verdict"),
|
||||
"audit_by_teacher": refreshed.get("audit_by_teacher"),
|
||||
"audit_worker_model": refreshed.get("audit_worker_model"),
|
||||
"audit_teacher_model": refreshed.get("audit_teacher_model"),
|
||||
"audited_at": refreshed.get("audited_at"),
|
||||
"saved_turns": refreshed.get("saved_turns"),
|
||||
"saved_tool_calls": refreshed.get("saved_tool_calls"),
|
||||
"baseline_verdict": refreshed.get("baseline_verdict"),
|
||||
"usefulness": refreshed.get("usefulness"),
|
||||
"necessity": refreshed.get("necessity"),
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
job["results"].append(res)
|
||||
job["done"] = len(job["results"])
|
||||
except _asyncio.CancelledError:
|
||||
cancelled = True
|
||||
finally:
|
||||
if not cancelled and not job.get("cancel") and not job.get("unavailable"):
|
||||
try:
|
||||
deps.finalize_batch(skills_manager, job.get("results") or [], owner, log)
|
||||
except Exception:
|
||||
logger.warning("Could not finalize skills audit batch", exc_info=True)
|
||||
job["current"] = None
|
||||
job["status"] = "cancelled" if cancelled or job.get("cancel") else "error" if job.get("unavailable") else "done"
|
||||
job["finished"] = _time.time()
|
||||
job.pop("task", None)
|
||||
|
||||
|
||||
async def run_scheduled_skill_audit(skills_manager: SkillsManager,
|
||||
owner: Optional[str] = None,
|
||||
max_skills: int = 8, *, deps: AuditDependencies) -> dict:
|
||||
"""Nightly audit pass. Audits the LEAST-recently-audited skills first and
|
||||
caps the batch so it rotates through the library over successive nights
|
||||
instead of re-checking the same ones every run. Reuses the same job store
|
||||
the manual 'Audit all' button uses, so its progress shows in the UI too."""
|
||||
import time as _time
|
||||
|
||||
key = (owner or "",)
|
||||
existing = deps.audit_jobs.get(key)
|
||||
if existing and existing.get("status") == "running":
|
||||
logger.info("Scheduled skill audit skipped — a run is already active.")
|
||||
return {"status": "running", "skipped": True}
|
||||
|
||||
try:
|
||||
url, model, headers, teacher = deps.resolve_models(owner=owner)
|
||||
except ValueError as e:
|
||||
logger.info(f"Scheduled skill audit skipped — {e}")
|
||||
return {"status": "skipped", "reason": str(e)}
|
||||
|
||||
from services.memory.skill_lifecycle import automatic_audit_candidates
|
||||
skills = automatic_audit_candidates(skills_manager.load(owner=owner), limit=max_skills)
|
||||
names = [s["name"] for s in skills]
|
||||
if not names:
|
||||
return {"status": "done", "total": 0}
|
||||
|
||||
deps.audit_jobs[key] = {
|
||||
"status": "running", "scope": "scheduled", "model": model,
|
||||
"teacher": teacher[1] if teacher else None,
|
||||
"total": len(names), "done": 0, "current": None,
|
||||
"results": [], "log": [f"Nightly audit of {len(names)} least-recently-checked skill(s) with {model}"
|
||||
+ (f"; teacher {teacher[1]}" if teacher else "")],
|
||||
"started": _time.time(), "cancel": False,
|
||||
}
|
||||
logger.info(f"Scheduled skill audit starting: {len(names)} skill(s) (owner={owner or 'all'})")
|
||||
await deps.run_batch(
|
||||
key, skills_manager, names, url, model, headers, teacher, owner,
|
||||
workload="background",
|
||||
)
|
||||
job = deps.audit_jobs.get(key, {})
|
||||
return {"status": "done", "total": len(names), "results": job.get("results", [])}
|
||||
Reference in New Issue
Block a user