"""Sequential skill audit coordination and transient owner-scoped jobs.""" import logging from typing import Optional from services.memory.skills import SkillsManager from .dependencies import AuditDependencies from .agent_runner import SkillAuditUnavailable logger = logging.getLogger(__name__) _skill_audit_jobs: dict = {} async def audit_one_skill(skills_manager, skill, url, model, headers, teacher, owner, log, workload: str = "foreground", *, deps: AuditDependencies) -> dict: """Test → judge → self-edit+retry → (teacher edit+retry) → flag. Never deletes; a skill the teacher still can't fix is demoted to draft for manual review. `teacher` is (url, model, headers) or None. `log(msg)` records progress.""" name = skill.get("name") # Reflect the audit outcome in the skill's confidence so the main list # updates: a clean pass earns high confidence; a pass that needed fixing # earns a bit less; a skill that still fails is marked low. def _set_conf(c): try: skills_manager.update_skill(name, {"confidence": c}, owner=owner) except Exception: pass md = skills_manager.read_skill_md(name, owner=owner) if not md: log(f"{name}: no source — skipped") return {"skill": name, "result": "skipped"} # Cheap deterministic cleanup first. If this is an obvious lower-priority # duplicate, do not spend LLM turns on necessity, retrieval precision, # skill-vs-baseline testing, self-edit, or teacher escalation. duplicate_of = deps.duplicate_blocker(skills_manager, name, owner) if duplicate_of: reason = f"Lower-priority duplicate of {duplicate_of}" try: skills_manager.update_skill(name, {"status": "draft", "confidence": 0.35}, owner=owner) skills_manager.set_audit(name, "skipped", by_teacher=False, worker_model=model, owner=owner) skills_manager.set_necessity(name, False, [duplicate_of], reason, owner=owner) except Exception: pass log(f"{name}: draft — skipped audit ({reason[:100]})") return {"skill": name, "result": "skipped_duplicate", "reason": reason, "confidence": 0.35, "status": "draft"} # Advisory necessity/redundancy check — runs once, independent of the test # outcome, and only records a flag the UI surfaces (never deletes/demotes). others = [] nec = None try: # Only compare against skills the SAME owner can see, so the necessity # judge never sees (or flags "redundant_with") another user's skills. sk_owner = skill.get("owner") others = [ {"name": s.get("name"), "description": s.get("description", "")} for s in skills_manager.load(owner=owner) if s.get("name") and s.get("name") != name and (not sk_owner or not s.get("owner") or s.get("owner") == sk_owner) ] nec = await deps.evaluate_necessity( md, others, url, model, headers, workload=workload, owner=owner, ) if nec is not None: skills_manager.set_necessity(name, nec.get("necessary", True), nec.get("redundant_with"), nec.get("reason"), owner=owner) if not nec.get("necessary", True): log(f"{name}: possibly unnecessary — {nec.get('reason', '')[:80]}") except Exception as e: log(f"{name}: necessity check skipped — {e}") # Retrieval precision check: if broad tags/trigger text would make this # narrow skill over-inject, fix only metadata before the functional test. try: if deps.should_check_retrieval_precision(skill): rp = await deps.evaluate_retrieval_precision( md, others, url, model, headers, workload=workload, owner=owner, ) if rp and not rp.get("ok"): issues = rp.get("issues") or ["metadata: retrieval: narrow tags and when_to_use to the intended trigger"] log(f"{name}: narrowing retrieval metadata — {(rp.get('summary') or issues[0])[:80]}") fixed = await deps.improve_markdown(md, { "verdict": "pass", "confidence": 1.0, "summary": rp.get("summary") or "Retrieval metadata is too broad.", "issues": issues, }, "Retrieval audit only: the procedure may work, but matching metadata is too broad.", url, model, headers, workload=workload, owner=owner) if fixed and fixed.strip() != md.strip() and deps.apply_markdown(skills_manager, name, fixed, owner): md = fixed refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) if refreshed: skill = refreshed except Exception as e: log(f"{name}: retrieval precision check skipped — {e}") task = deps.test_task(skill) log(f"{name}: testing…") skill_messages = deps.test_messages(md, task) transcript, skill_stats, approval_required = await deps.run_arm( skill_messages, url, model, headers, owner, workload=workload, ) if approval_required is not None: try: from src.tool_approvals import tool_approval_store tool_approval_store.consume( approval_required.get("approval_id"), decision="deny", owner=owner, session_id=None, ) except Exception: logger.debug("Could not retire unattended skill approval", exc_info=True) verdict = { "verdict": "inconclusive", "confidence": 1.0, "summary": ( "This automated audit reached an exact action that requires " "a human approval; no action was executed." ), "issues": [ "Run this skill's manual test and review the sealed action." ], "approval_required": True, } else: baseline_task = task log(f"{name}: running no-skill baseline…") baseline_transcript, baseline_stats, baseline_approval = await deps.run_arm( deps.baseline_messages(baseline_task), url, model, headers, owner, workload=workload, ) if baseline_approval is not None: try: from src.tool_approvals import tool_approval_store tool_approval_store.consume( baseline_approval.get("approval_id"), decision="deny", owner=owner, session_id=None, ) except Exception: logger.debug("Could not retire unattended baseline approval", exc_info=True) verdict = await deps.evaluate_run( md, task, transcript, url, model, headers, baseline_transcript=baseline_transcript, skill_stats=skill_stats, baseline_stats=baseline_stats, workload=workload, owner=owner, ) v = verdict.get("verdict") log(f"{name}: verdict = {v} ({verdict.get('summary', '')[:80]})") if verdict.get("approval_required"): # An unattended audit is not authority for an action influenced by the # skill under test. Preserve the skill's current publication/confidence # state and route the exact action to the manual test UI instead of # letting a safety pause demote, rewrite, or auto-publish the skill. skills_manager.set_audit( name, "inconclusive", by_teacher=False, worker_model=model, owner=owner, audit_summary=verdict.get("summary") or "The test requires approval for an external action.", ) status = skill.get("status") or "draft" log(f"{name}: {status} unchanged — exact action needs manual approval") return { "skill": name, "result": "approval_required", "verdict": verdict, "status": status, } if v == "pass": # Procedure works. If the reviewer still flagged metadata (tags/category/ # when_to_use/description), do ONE fixer pass to correct the frontmatter # without re-testing — a metadata-only fix can't break a passing run. meta_issues = [i for i in (verdict.get("issues") or []) if str(i).lower().lstrip().startswith("metadata:")] if meta_issues: log(f"{name}: pass, but fixing {len(meta_issues)} metadata issue(s)…") fixed = await deps.improve_markdown( md, verdict, transcript, url, model, headers, workload=workload, owner=owner, ) if fixed and fixed.strip() != md.strip(): deps.apply_markdown(skills_manager, name, fixed, owner) _set_conf(0.95) skills_manager.set_audit( name, "pass", by_teacher=False, worker_model=model, owner=owner, **deps.verdict_efficiency(verdict), ) refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) status = deps.finalize_status(skills_manager, name, owner, "pass", 0.95, (refreshed or {}).get("necessity"), verdict) log(f"{name}: {status} — confidence 95%") return {"skill": name, "result": "pass", "verdict": verdict, "confidence": 0.95, "status": status} if v in ("unknown", "inconclusive"): skills_manager.set_audit( name, "inconclusive", by_teacher=False, worker_model=model, owner=owner, **deps.verdict_efficiency(verdict), ) status = deps.finalize_status(skills_manager, name, owner, "inconclusive", skill.get("confidence") or 0.0, skill.get("necessity")) log(f"{name}: {status} — inconclusive") return {"skill": name, "result": "inconclusive", "verdict": verdict, "status": status} # Self-edit + retry. log(f"{name}: self-editing to fix issues…") new_md = await deps.improve_markdown( md, verdict, transcript, url, model, headers, workload=workload, owner=owner, ) if new_md and new_md.strip() != md.strip() and deps.apply_markdown(skills_manager, name, new_md, owner): md = new_md transcript, verdict = await deps.run_test_once( md, task, url, model, headers, owner, workload=workload, ) v = verdict.get("verdict") log(f"{name}: retry (self) = {v}") if v == "pass": _set_conf(0.85) skills_manager.set_audit( name, "pass", by_teacher=False, worker_model=model, owner=owner, **deps.verdict_efficiency(verdict), ) refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) status = deps.finalize_status(skills_manager, name, owner, "pass", 0.85, (refreshed or {}).get("necessity"), verdict) log(f"{name}: {status} — confidence 85% after self-edit") return {"skill": name, "result": "pass_after_self_edit", "verdict": verdict, "confidence": 0.85, "status": status} # Teacher escalation (if a distinct teacher model is configured). The # teacher only REWRITES the skill — it does NOT run the test. The point is # to verify the regular (student) model can now succeed with the teacher's # improved procedure, so the retry runs on the worker model, not the teacher. teacher_ran = False if teacher and teacher[0] and teacher[1] and (teacher[1] != model or teacher[0] != url): teacher_ran = True t_url, t_model, t_headers = teacher log(f"{name}: teacher {t_model} rewriting the skill…") t_md = await deps.improve_markdown( md, verdict, transcript, t_url, t_model, t_headers, workload=workload, owner=owner, ) if t_md and t_md.strip() != md.strip() and deps.apply_markdown(skills_manager, name, t_md, owner): md = t_md # Re-test with the STUDENT model (the model the skill runs under in use). transcript, verdict = await deps.run_test_once( md, task, url, model, headers, owner, workload=workload, ) v = verdict.get("verdict") log(f"{name}: retry on student after teacher rewrite = {v}") if v == "pass": _set_conf(0.8) skills_manager.set_audit( name, "pass", by_teacher=True, worker_model=model, teacher_model=t_model, owner=owner, **deps.verdict_efficiency(verdict), ) refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) status = deps.finalize_status(skills_manager, name, owner, "pass", 0.8, (refreshed or {}).get("necessity"), verdict) log(f"{name}: {status} — confidence 80% after teacher rewrite") return {"skill": name, "result": "pass_after_teacher", "verdict": verdict, "confidence": 0.8, "status": status} # Still failing → demote to draft + low confidence + flag (do NOT delete). try: skills_manager.update_skill(name, {"status": "draft", "confidence": 0.35}, owner=owner) except Exception: pass skills_manager.set_audit( name, v or "fail", by_teacher=teacher_ran, worker_model=model, teacher_model=(teacher[1] if teacher_ran and teacher else ""), owner=owner, **deps.verdict_efficiency(verdict), ) log(f"{name}: flagged — confidence lowered, kept as draft for manual review") return {"skill": name, "result": "flagged", "verdict": verdict, "confidence": 0.35} async def run_audit_batch(key, skills_manager, names, url, model, headers, teacher, owner, workload: str = "foreground", *, deps: AuditDependencies): """Background: audit each named skill in sequence, recording progress.""" import asyncio as _asyncio import time as _time job = deps.audit_jobs.get(key) if job is None: return def log(msg): job["log"].append(msg) if len(job["log"]) > 1000: del job["log"][0:len(job["log"]) - 1000] cancelled = False try: for nm in names: if job.get("cancel"): cancelled = True log("(cancelled)") break job["current"] = nm skills = skills_manager.load(owner=owner) sk = next((s for s in skills if s.get("name") == nm), None) if not sk: continue try: res = await deps.audit_one( skills_manager, sk, url, model, headers, teacher, owner, log, workload=workload, ) except _asyncio.CancelledError: cancelled = True job["cancel"] = True log("(cancelled)") raise except SkillAuditUnavailable as e: job["unavailable"] = str(e) log(f"Audit paused: {e}. Skill verdicts unchanged; retry when the model is available.") break except Exception as e: log(f"{nm}: error — {e}") res = {"skill": nm, "result": "error"} skills_manager.set_audit(nm, "inconclusive", worker_model=model, owner=owner) try: refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == nm), None) if refreshed: res["skill_state"] = { "name": refreshed.get("name"), "status": refreshed.get("status"), "confidence": refreshed.get("confidence"), "audit_verdict": refreshed.get("audit_verdict"), "audit_by_teacher": refreshed.get("audit_by_teacher"), "audit_worker_model": refreshed.get("audit_worker_model"), "audit_teacher_model": refreshed.get("audit_teacher_model"), "audited_at": refreshed.get("audited_at"), "saved_turns": refreshed.get("saved_turns"), "saved_tool_calls": refreshed.get("saved_tool_calls"), "baseline_verdict": refreshed.get("baseline_verdict"), "usefulness": refreshed.get("usefulness"), "necessity": refreshed.get("necessity"), } except Exception: pass job["results"].append(res) job["done"] = len(job["results"]) except _asyncio.CancelledError: cancelled = True finally: if not cancelled and not job.get("cancel") and not job.get("unavailable"): try: deps.finalize_batch(skills_manager, job.get("results") or [], owner, log) except Exception: logger.warning("Could not finalize skills audit batch", exc_info=True) job["current"] = None job["status"] = "cancelled" if cancelled or job.get("cancel") else "error" if job.get("unavailable") else "done" job["finished"] = _time.time() job.pop("task", None) async def run_scheduled_skill_audit(skills_manager: SkillsManager, owner: Optional[str] = None, max_skills: int = 8, *, deps: AuditDependencies) -> dict: """Nightly audit pass. Audits the LEAST-recently-audited skills first and caps the batch so it rotates through the library over successive nights instead of re-checking the same ones every run. Reuses the same job store the manual 'Audit all' button uses, so its progress shows in the UI too.""" import time as _time key = (owner or "",) existing = deps.audit_jobs.get(key) if existing and existing.get("status") == "running": logger.info("Scheduled skill audit skipped — a run is already active.") return {"status": "running", "skipped": True} try: url, model, headers, teacher = deps.resolve_models(owner=owner) except ValueError as e: logger.info(f"Scheduled skill audit skipped — {e}") return {"status": "skipped", "reason": str(e)} from services.memory.skill_lifecycle import automatic_audit_candidates skills = automatic_audit_candidates(skills_manager.load(owner=owner), limit=max_skills) names = [s["name"] for s in skills] if not names: return {"status": "done", "total": 0} deps.audit_jobs[key] = { "status": "running", "scope": "scheduled", "model": model, "teacher": teacher[1] if teacher else None, "total": len(names), "done": 0, "current": None, "results": [], "log": [f"Nightly audit of {len(names)} least-recently-checked skill(s) with {model}" + (f"; teacher {teacher[1]}" if teacher else "")], "started": _time.time(), "cancel": False, } logger.info(f"Scheduled skill audit starting: {len(names)} skill(s) (owner={owner or 'all'})") await deps.run_batch( key, skills_manager, names, url, model, headers, teacher, owner, workload="background", ) job = deps.audit_jobs.get(key, {}) return {"status": "done", "total": len(names), "results": job.get("results", [])}