diff --git a/.gitleaks.toml b/.gitleaks.toml index 3a51ae4c1..0eec3f1ae 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -21,3 +21,17 @@ regexes = [ '''^inherits_2\.0\.1$''', '''^inherits_2\.0\.3$''', ] + + +[[allowlists]] +description = "Reviewed Skills OpenAPI baseline SHA-256 digest" +targetRules = ["generic-api-key"] +condition = "AND" +paths = [ + '''(?:^|/)tests/fixtures/skills_api_baseline\.json$''', + '''(?:^|/)docs/skills-audit-architecture\.md$''', +] +regexTarget = "secret" +regexes = [ + '''^7abc5b334a6c3c52dd3eba15d8bc516da728a67f14fb7f56d0bb5992d89a16ca$''', +] diff --git a/docs/skills-audit-architecture.md b/docs/skills-audit-architecture.md new file mode 100644 index 000000000..78227a763 --- /dev/null +++ b/docs/skills-audit-architecture.md @@ -0,0 +1,379 @@ +# Skills evaluation and audit architecture + +The Skills backend separates HTTP transport, model evaluation, agent execution, +audit orchestration and publication policy. `services/skills/` coordinates those +responsibilities; `services/memory/skills.py` remains the canonical persistence +and lifecycle owner. The extraction preserves the existing HTTP and Python +contracts. [PR #6711](https://github.com/odysseus-dev/odysseus/pull/6711) tracks +this boundary change under [issue #6710](https://github.com/odysseus-dev/odysseus/issues/6710). + +## Responsibilities and dependency boundaries + +| Module | Responsibility | +| --- | --- | +| `routes/skills_routes.py` | FastAPI request models, endpoint declarations, authentication, ownership, HTTP errors, exact approval validation and Python compatibility bindings | +| `services/skills/job_coordinator.py` | Transient job creation, continuation after approval validation, candidate selection, public polling projections and cancellation requests | +| `services/skills/agent_runner.py` | Manual execution, audit arms, agent loop integration, transcripts, authority propagation, approval pauses and same-task stream closure | +| `services/skills/audit_workflow.py` | Per-skill audit sequencing, self-edit retries, teacher escalation, sequential batches and scheduled execution | +| `services/skills/evaluation.py` | Model grading, verdict parsing, necessity and retrieval checks, efficiency fields and markdown improvement generation | +| `services/skills/publication.py` | Confidence and publishing policy, duplicate rules, batch finalization and markdown updates through SkillsManager | +| `services/skills/model_selection.py` | Owner-scoped worker/teacher resolution, utility fallback, endpoint selection and credential lookup | +| `services/skills/prompts.py` | Shared task construction and explicitly untrusted skill context | +| `services/skills/dependencies.py` | Frozen `AuditDependencies` containing callable bindings and transient registry references | +| `services/memory/skills.py` | Canonical skill identity, owner isolation, durable storage, indexing, retrieval and lifecycle state | + +The package initializer identifies the domain without adding a service facade. +Evaluation does not schedule jobs or persist skills. Publication computes policy +and applies updates through the supplied SkillsManager. Services do not import +`routes.skills_routes`. The existing preferences adapter in `publication.py` +still imports `routes.prefs_routes`; moving that adapter is a separate change. + +```mermaid +flowchart TD + HTTP[Authenticated HTTP requests] --> Routes[skills_routes] + Builtins[builtin_actions callers] --> Routes + Routes --> Jobs[job_coordinator] + Routes --> Models[model_selection] + Routes --> Evaluation[evaluation] + Routes --> Runner[agent_runner] + Routes --> Workflow[audit_workflow] + Jobs -->|injected calls| Runner + Jobs -->|injected calls| Workflow + Workflow -->|injected calls| Runner + Workflow -->|injected calls| Evaluation + Workflow -->|injected calls| Publication[publication] + Runner -->|injected calls| Evaluation + Runner --> Runtime[Agent runtime and authority enforcement] + Runner --> Prompts[prompts and untrusted context] + Jobs --> Prompts + Workflow --> Prompts + Routes --> Manager[Canonical SkillsManager] + Runner --> Manager + Workflow --> Manager + Publication --> Manager + Publication --> Preferences[Existing preferences adapter] + Models --> Resolver[Endpoint resolver and owner-scoped credentials] +``` + +Arrows include injected calls, rather than only Python imports. The workflow +imports `SkillAuditUnavailable` from the runner. Route wrappers construct +`AuditDependencies` on entry from the current route bindings, so existing +callers and monkeypatches reach the same collaborators without rebinding shared +service globals. + +The manual test registry belongs to `agent_runner.py`; the audit registry belongs +to `audit_workflow.py`. Routes expose aliases to those dictionaries. Registries +hold process-local job state and are not durable skill stores. All durable +metadata, confidence, markdown and publication changes use SkillsManager. +Published-only retrieval and local exclusion of `remote-only` skills stay in +the manager/index boundary. Service loggers use their module names; the route +module retains its own logger. Root handlers supply the existing log output. + +## Execution and publication lifecycle + +1. The HTTP handler authenticates the caller, verifies ownership and validates + the request. Manual model choices retain registered-endpoint validation and + owner-scoped credential lookup. +2. Job coordination creates the owner-keyed transient record and launches the + execution coroutine. Replacing a manual test retires its pending approval. + Polling exposes only the existing public fields, excluding credentials, + private transcripts, continuation state and asyncio tasks. +3. The runner uses the originating request authority or the consumed sealed + approval authority. It captures the transcript and closes the async stream + with `aclosing` inside the task that opened it, including early approval + pauses, cancellation and exceptions. +4. Manual approval pauses return to the HTTP validation boundary. Continuation + reconstructs context from the original untrusted skill and accumulated + untrusted transcript. Manual completion records audit metadata and adjusts + confidence (pass .95, needs_work .6, fail .4) without publishing. +5. An automated audit checks deterministic duplicates, advisory necessity and, + when indicated, retrieval precision. It runs skill and no-skill arms on the + same task, grades the result, then may self-edit and retry. An optional + distinct teacher rewrites the skill; the worker performs the retry. +6. Publication applies owner preferences, confidence thresholds, necessity, + utility and duplicate rules through SkillsManager. A normal sequential batch + finalizes its processed results. Cancellation or infrastructure unavailability + skips batch finalization and clears task/current state. + +A clean audit pass sets confidence to .95, a self-edit pass to .85 and a teacher +rewrite pass to .8. Continued failure becomes draft at .35. Unknown and +inconclusive verdicts preserve publication state. Automatic publishing requires +an enabled owner policy, a passing verdict at or above the configured threshold, +and no necessity, worse-baseline or duplicate blocker. The default threshold is +.85; owner values are clamped to [0, 1]. Markdown edits pin the existing skill name +so usage and audit sidecars remain attached to the same identity. + +Per-skill duplicate blocking and batch duplicate finalization have distinct +selection rules and must not be consolidated without a behavior proposal. +Batch finalization excludes builtins and binned records, changes only processed +skills, protects approval-required results, and bins duplicate losers. Duplicate +similarity uses the existing .38 threshold and name-suffix grouping. + +Batches run sequentially per owner, retain current/done/results and timestamps, +and cap progress logs at 1,000 entries. Ordinary exceptions record inconclusive +skill audits. Infrastructure failures stop the batch without inventing a skill +verdict. Scheduled execution skips an active owner job, selects existing lifecycle +candidates by oldest audit, caps the batch (default eight), and marks model phases +as background work. These services retain the existing scheduling and +cancellation semantics. + +## Security and approval invariants + +HTTP authentication and owner verification remain at the transport boundary. +Manager reads and writes receive the owner. Ownerless builtins retain their +existing visibility; an authenticated user cannot access an ownerless ordinary +skill. Existing ownerless deployment behavior is preserved. + +The `/test-approval` handler validates the pending job, approval ID, decision, +owner and empty session binding, then destructively consumes the sealed approval +with `allow_continuation=False` before calling the coordinator. Stale, mismatched +or reused approvals fail with 409. The consumed grant authorizes only its exact +action. Services do not manufacture approval authority or relax process identity, +workspace confinement or request-authority enforcement in the agent runtime. + +Skill markdown and resumed transcripts remain explicitly untrusted context, +never trusted system instructions. Audit arms suppress saved skill injection. +Unattended runs deny encountered approval records instead of leaving reusable +grants. An initial approval-required audit result preserves the skill's publication +and confidence and directs the user to manual approval. Retry handling has an +inherited gap: an approval pause after a rewrite can still lead to demotion. +Same-task stream closure preserves the originating ContextVar cleanup, but manual +cancellation can retain private continuation state. These are separate hardening +items, not guarantees supplied by the extraction. + +## Model selection and evaluation + +Audit resolution uses an explicit endpoint/model pair when supplied, looking up +credentials among enabled owner-visible endpoints. A model-only choice uses the +existing owner-scoped model resolver. Otherwise it resolves the utility model, +including the resolver's default behavior, then takes the first configured +utility fallback if the primary result is incomplete. A missing worker raises +`ValueError`. Served-model normalization keeps an exact match, then a basename +match, then the first served ID; discovery exceptions keep the resolved choice. + +An enabled teacher resolves separately with the same owner. Its credentials are +not borrowed from the worker. Teacher resolution failure logs the existing warning +and leaves the teacher absent. Manual tests retain their HTTP endpoint validation +and registered fallback credentials. + +Owner reasoning effort is resolved for each evaluation or rewrite call. Workload +flags, credentials, retries, prompts and permissive parsing remain unchanged: + +| Phase | Token budget | Timeout in seconds | +| --- | --- | --- | +| Run review, up to two attempts | 32768 | 180 | +| Necessity | 8192 | 120 | +| Retrieval precision | 4096 | 90 | +| Markdown improvement | 16384 | 180 | + +Review parsing retains think-block stripping, trailing-comma recovery, last +usable verdict selection, prose fallback, bounded fields and unknown outcomes. +These model outputs are advisory and do not grant execution authority. + +## Public HTTP contract + +All paths below have prefix `/api/skills`. Successful status defaults to 200. +Request models and endpoint declaration order remain fixed. A frozen +OpenAPI and declaration-order fixture records +the baseline contract independently of the extracted implementation. + +| Method and suffix | Successful response | Authorization and errors | +| --- | --- | --- | +| GET empty | `skills,count` | Owner-filtered load | +| GET `/index` | `index,count` | Owner-scoped eligible index | +| GET `/slash-catalog` | `skills,count` | Owner-scoped index; sorted slash entries | +| GET `/builtin` | `builtin,count` and optional `error` | Existing shared capability text | +| GET `/builtin/{name}` | `name,text,default,is_overridden` | 404 missing tool; 500 import failure | +| PUT `/builtin/{name}` | `ok,name,is_overridden` | Admin; 404 unknown, 400 missing text | +| DELETE `/builtin/{name}` | `ok,name,is_overridden` | Admin | +| POST `/import-from-url` | `ok,skill,files` | Admin and current owner; 400 import, 502 HTTP, 500 other failure | +| POST `/add` | `ok,deduped,skill` | Current owner passed to manager; unchanged old/new request fields | +| POST `/{skill_id}/invoke` | `ok,type,name,command,message` | Eligible owner index only; 404 unavailable/source missing | +| GET `/{skill_id}` | Full skill dictionary | Owner load; 404 missing | +| GET `/{skill_id}/markdown` | `name,markdown` | Owner verification; 404 missing/source missing | +| POST `/{skill_id}/test` | `ok,status,skill,model` | Owner verification, HTTP request authority, registered fallback credentials; 400 missing/invalid model | +| POST `/{skill_id}/test-approval` | `ok,status,decision` | User approval request, owner verification, pending ID/owner/empty session match; 400 invalid body/decision, 409 stale/mismatch/consumed | +| GET `/{skill_id}/test-status` | `status` or `status,task,model,log,verdict,approval` | Owner-keyed lookup; server continuation/credentials omitted | +| POST `/audit-all` | `ok,status,total` plus running `done,model` or empty `results,log` | Owner-keyed singleton; 400 model resolution | +| GET `/audit-all/status` | `status` or job progress/result/log/timestamps | Owner-keyed lookup; task and credentials omitted | +| POST `/audit-all/cancel` | `ok,status` | Owner-keyed lookup and cancellation | +| POST `/{skill_id}/markdown` | `ok,name` | Owner verification; 400 missing/parse, 404 missing, 500 update | +| PUT `/{skill_id}` | `ok` | Owner verification and scoped manager update; 404 missing | +| DELETE `/{skill_id}` | `ok` | Owner verification and scoped manager deletion; 404 missing | +| POST `/search` | `skills,query,count` | Owner-filtered input; 400 empty query | + +Strict owner verification allows an ownerless builtin but denies an ownerless +ordinary skill for an authenticated user. The existing ownerless deployment +behavior remains. Global auth middleware is unchanged. + +## Python compatibility and frozen contracts + +`app.py` imports `setup_skills_routes`. `src/builtin_actions.py` uses the existing +route helpers and audit registry. Route aliases and wrappers preserve the original +28 inspectable interfaces, `SkillAuditUnavailable`, registry identity and patchable +collaborators. Lazy model/agent imports and route-level effort injection keep +existing offline test seams effective. + +The task and message callbacks are bound through `AuditDependencies` at service +entry. Patches to route-level test tasks and test/baseline messages reach manual +execution, approval continuation, both audit arms, worker retries and scheduled +audits. The `_skill_baseline_task` compatibility alias delegates to the current +route test-task builder. Execution continues to reuse one task for both arms, +as in the original implementation; it does not independently derive a baseline +task through that alias. + +`tests/fixtures/skills_api_baseline.json` records declaration order and structural +signatures captured from pre-extraction revision +`86b886c611338282af6d4a717246ea4e256f58df`. Signature records retain parameter +names, order, kinds, defaults, default factories and annotations, including +Pydantic constraint metadata. They compare typing origins and arguments instead +of Python-version-dependent annotation representations. The expanded records also +pin field validation aliases, alias priority, discriminators, `validate_default`, +rebuilt field annotations and model configuration. Field order remains visible +through the constructor parameter sequence. Only the known builtin factories +`list`, `tuple`, `dict`, `set` and `frozenset` are supported. Custom factories, +including closures and factories taking validated data, fail closed without being +executed. Unsupported metadata/configuration values also raise rather than using +unstable representations. This is a structural contract for supported interfaces, +not proof of behavioral equivalence for arbitrary validators or Python code. + +The expanded fixture was derived again from the original revision above in a +detached scratch worktree, with the updated comparator copied in. Python 3.11 +and 3.12 produced byte-identical snapshots. The previous parameter/return records, +all 28 interface names, all 22 endpoint declarations and the original OpenAPI +digest were checked unchanged before accepting the new field records. + +`tests/fixtures/skills_openapi_baseline.json` is the same baseline's complete +Skills-router OpenAPI document, generated with a standalone FastAPI application. +Sorted keys, ASCII escaping, two-space indentation and a final newline make the +fixture deterministic. Tests require exact canonical equality and print a unified +diff on mismatch. The redundant digest field and assertions have been removed; +they provided no independent compatibility guarantee. The fixture pins model +schemas and constraints. CI reads committed fixtures and needs no Git history +for these tests. Baseline updates require an intentional compatibility decision, rather +than regenerating fixtures from a failing implementation. + +The narrow `.gitleaks.toml` exception remains necessary for the historical public +digest in earlier fixture and documentation revisions. It applies to the exact digest, +the `generic-api-key` rule and the two named paths. Removing a current copy cannot +remove findings from published history. Keep the exception unless the pinned +full-history scan proves it unnecessary; do not rewrite history to remove it. +The checksum-verified 8.30.1 scan without this exception still reports both +historical copies in commit `a547c4d8`; retaining the exception passes the full +PR/dev history scan. +Reconsider it after squash merge only after verifying the resulting reachable +history with the pinned full-history scan. + +The ReDoS source guard scans `services/skills/evaluation.py`, the canonical +evaluation parser module, as well as the retained route and memory-extractor +locations. Verdict regex runtime tests import the canonical module directly. +The source guard must reject reintroducing the lazy think-block expression; +the ordinary-input and adversarial-flood runtime tests remain in place. An isolated +source mutation reproduced the old guard's false green for the evaluation module. +The corrected guard rejected the vulnerable expression in each of the three +locations and passed again after restoring the temporary copies. The mutated +regex was never executed, and production parser behavior was not changed. + +## Reproducible validation + +From the repository root, use a virtual environment with `requirements-dev.txt` +installed and the repository's frontend dependencies available. The examples use +`.venv/bin/python`; repeat the contract tests with a Python 3.12 environment as +well as Python 3.11. Record interpreter/dependency versions with results. + +```sh +.venv/bin/python -m pytest -q \ + tests/test_skills_audit_characterization.py tests/test_skills_audit_services.py +.venv/bin/python -m compileall -q app.py core routes src services scripts tests +.venv/bin/python -c 'import routes.skills_routes; import services.skills.agent_runner; import services.skills.audit_workflow' +.venv/bin/python scripts/build_frontend.py --check +node scripts/check_frontend.mjs +node --test tests/skillsApproval.test.mjs tests/frontendArchitecture.test.mjs +git diff --check +``` + +The focused Skills, authority and approval selection is reproducible with: + +```sh +.venv/bin/python - <<'PYTHON' +from pathlib import Path +import subprocess +import sys + +files = set() +for path in Path("tests").rglob("test_*.py"): + text = path.read_text() + if ("skill" in path.name or "approval" in path.name + or "utility_reasoning" in path.name or "skills_routes" in text + or "review_endpoint_credentials" in path.name): + files.add(str(path)) +files.update([ + "tests/test_request_authority.py", + "tests/test_endpoint_registered_authority.py", + "tests/test_redos_verdict_continuation.py", + "tests/test_redos_residual_python.py", +]) +subprocess.run([sys.executable, "-m", "pytest", "-q", "-rs", + "--durations=15", *sorted(files)], check=True) +PYTHON +``` + +The original selection contained 729 cases. Additional characterization cases +increase that count without removing the original coverage. For the full suite, +remove a development `APP_PORT` and put the environment's tools on PATH: + +```sh +env -u APP_PORT PATH="$PWD/.venv/bin:$PATH" \ + PLAYWRIGHT_BROWSERS_PATH="$PWD/.cache/ms-playwright" \ + .venv/bin/python -m pytest -q -rs --durations=25 +``` + +Run the checksum-verified Gitleaks binary pinned in +`.github/workflows/secret-scan.yml` on full reachable history, including the PR +head and fetched official `dev`. The current workflow pins 8.30.1. Its CI command +is `gitleaks git --no-banner --redact --verbose .` with a full-history checkout. +A local checkout can restrict refs to the relevant histories with +`--log-opts='HEAD upstream/dev'`; do not restrict the commit range to just the +latest patch. Scan uncommitted fixtures too before committing. + +## Historical evidence and validation limits + +The initial extraction validation at `a547c4d8` reported 42 characterization and +service cases, 729 focused cases, 13 relevant Node cases and 17 isolated uvicorn +HTTP checks passing. The historical repository-wide run reported 13,959 passed, +60 skipped and three failures. These counts describe the extraction validation, +not the current revision. The HTTP checks used isolated state and authentication +and included second-owner denial, but exercised neither a configured live model +nor an external tool action. + +The two containment failures reproduced on unchanged baseline `86b886c6` with +extra `Failed to create stream fd: No such file or directory` stderr output: + +- `test_namespace_handshake_closes_model_stdin` +- `test_execution_environment_cannot_replace_probed_bwrap[False]` + +`test_legacy_cleanup_against_a_private_real_tmux_server` failed under the original +full-suite load and passed isolated reruns. Its cause was not fully diagnosed. +These historical failures must be distinguished from new regressions; successful +CI and maintainer review remain integration gates. Earlier AST comparisons and +the [maintainer review](https://github.com/odysseus-dev/odysseus/pull/6711#issuecomment-6170115749) +provide historical extraction evidence, not an independent review of later fixes. + +## Known limitations and follow-up boundaries + +[Issue #6635](https://github.com/odysseus-dev/odysseus/issues/6635) remains open: +manual tests without an appropriate workspace can still fail to execute their +proposed action. Stream cleanup and exact approvals are preserved, but do not +supply a missing workspace. + +Jobs remain process-local and do not survive restarts. Durable jobs, moving the +preferences adapter below transport, consolidating storage under the domain +package, or changing duplicate/publication policies need separate designs. +The canonical manager, skill format/lifecycle modules, provider implementations, +agent authority enforcement and frontend contracts remain their existing owners. + +The [second-review correction report](skills-audit-second-review.md) records the +restored prompt seams, independently expanded constructor baseline, actual +validation results and a separate prioritized plan for the six inherited P2 +failure paths. Model-controlled lifecycle fields and preference-load failures +require maintainer action before automatic rewrite/publication deployment. diff --git a/docs/skills-audit-second-review.md b/docs/skills-audit-second-review.md new file mode 100644 index 000000000..91fdaae43 --- /dev/null +++ b/docs/skills-audit-second-review.md @@ -0,0 +1,200 @@ +# Skills PR #6711 second-review corrections + +The correction scope is route prompt compatibility and structural constructor +contracts. PR #6711 remains draft. Durable skill ownership stays with +`SkillsManager`. No publication policy, authority mechanism, provider runtime, +frontend behavior or protected manager/index implementation was changed. + +The reviewed head was `a6a33674ed4facf11f53e7cdd41f882ad0fa8482`. The independent +original baseline is `86b886c611338282af6d4a717246ea4e256f58df`. Official `dev` +integration was tested against `c3819e3cbf56c4f899eacc66b21115a77d9c570d`. + +## Compatibility corrections + +Service-local imports captured prompt builders outside the route namespace. +Patching the public route symbols therefore stopped affecting execution after +extraction. `AuditDependencies` now binds the current test-task, test-message and +baseline-message builders. Manual execution, exact-approval continuation, initial +audits, baseline arms, self/teacher retries and scheduled batches use those +bindings. The route `_skill_baseline_task` wrapper again delegates to the current +route `_skill_test_task`. Both execution arms still use one shared task, matching +the original behavior. No service imports Skills routes. + +The signature checker treated a factory's module/qualified name as its identity +and omitted Pydantic settings outside the generated Python signature. Closures +with different captured values and default-validation changes were false +negatives. New tests additionally demonstrated hidden validation-alias path and +model-configuration changes. The checker now records public field validation +settings, rebuilt annotations, defaults and inherited model configuration. See +the [Pydantic FieldInfo reference](https://docs.pydantic.dev/latest/api/fields/) +for the distinction between factories, validation aliases and `validate_default`. + +Only known builtin factories are supported by identity. Custom factories, +including factories requiring validated field data, raise `TypeError` without +being invoked during comparison. Unsupported metadata/configuration values also +fail closed. No bytecode, arbitrary object representations or factory output is +serialized. The comparator does not promise equivalence of arbitrary Python +validators or factory implementations. + +The expanded fixture was generated from the original baseline in a new detached +worktree, with only the revised test comparator copied in. Python 3.11 and 3.12 +produced byte-identical files. Every previous parameter/return record remains +equal. All 28 interfaces and 22 endpoint declarations remain present. The +canonical OpenAPI fixture is unchanged, and its original digest is unchanged. +Independent native `inspect.signature` snapshots also match original versus +corrected implementations on each interpreter. + +## Regression evidence and validation + +The new tests were run before production/comparator edits: 12 failed and four +passed, demonstrating the six signature cases and six broken prompt cases. After +correction, all 16 added cases pass. The prompt probes capture actual +`stream_agent_loop` ingress rather than replacing the runners or coordination. +They cover patched and normal execution, guard escaping and taint metadata, +owner authority, exact-approval continuation, and worker retries after both +self and teacher rewrites. The same ten prompt cases pass on the original +implementation. The six signature cases reject custom factories without calling +them and detect default validation, alias paths and model validation settings. + +Environments: Python 3.11.17 and 3.12.13, FastAPI 0.143.1, Pydantic 2.14.1, +pytest 9.1.1. Python test commands set `FASTEMBED_MODEL=review-disabled` and +`HF_HUB_OFFLINE=1` to exercise unavailable optional embeddings without model +downloads. They do not validate live providers or embedding quality. + +| Executed scope | Result | +| --- | --- | +| 53-file Skills, approval, authority and lifecycle selection, Python 3.11 | 757 passed | +| Same selection, Python 3.12 | 757 passed | +| Both characterization/service files, included above | 70 passed | +| Original baseline, available focused selection plus revised characterization | 724 passed | +| Original baseline, revised characterization plus prompt probes, each interpreter | 47 passed | +| Current-dev combined tree, focused selection plus eight intervening dev test files, each interpreter | 900 passed | +| Repository-wide suite, Python 3.11 | 13,991 passed, 60 skipped, two inherited environment failures, six subtests passed | +| Frontend publisher and architecture checks | Passed, zero changed modules | +| Relevant Node tests | 13 passed | +| Compilation, both Python versions | Passed | +| Import graph and fresh service-first/route-first imports, both versions | Passed; acyclic; no reverse Skills route import | +| Original normalized moved-body comparison | 26 of 26 matched after explicit dependency substitutions | +| Pinned/checksum-verified Gitleaks 8.30.1, reachable HEAD and official dev histories | No leaks | +| Same Gitleaks, tracked working files | No leaks | +| Diff/whitespace and protected-source comparisons | Passed | + +The two repository-wide failures are +`tests/test_containment_enforcement.py::test_namespace_handshake_closes_model_stdin` +and `test_execution_environment_cannot_replace_probed_bwrap[False]`. Isolated +reruns on both original and corrected implementations reproduce the same extra +`Failed to create stream fd: No such file or directory` stderr. The independent +review traced it to login-profile `im-config` calls to `systemd-cat` inside +Bubblewrap without the journal stream socket. These environment failures remain; +they were not hidden by excluding tests. A full Python 3.12 suite was not run. + +The combined test tree used the official merge revision `5419ed37`, whose tree +exactly matches `git merge-tree --write-tree upstream/dev a6a33674`, with the +correction diff applied in a detached scratch worktree. Published history was +not rebased, amended, reset or rewritten. + +Local commands, independent fixture captures, baseline reproductions and full +logs are retained under `/tmp/skills-6711-corrections/`. The independent review +and adversarial probe source remain under `/tmp/skills-6711-second/`. These +temporary evidence paths are not portable repository fixtures. CI should read +the committed golden fixtures, not regenerate them from the implementation. + +## Separate inherited P2 hardening plan + +All six defects demonstrably exist on the official original baseline. The six +safety assertions in `/tmp/skills-6711-second/test_adversarial.py` fail on both +the original and corrected trees with the same outcomes. Logs are +`inherited-original.log` and `inherited-head.log` in the correction evidence +directory. The original/source body comparison also matches these paths after +dependency substitutions. There is no demonstrated material worsening from the +extraction. That finding does not make their security impact acceptable. + +Priorities below concern a separate hardening change. Items 1 and 2 are release +blockers requiring maintainer resolution before deploying automatic skill +rewrite/publication workflows. This recommendation is independent of whether +the architecture PR is integrated. No tool-authority bypass or cross-owner +credential disclosure was demonstrated, and no critical extraction regression +was found that requires expanding this compatibility correction. + +1. **Model-controlled lifecycle/provenance fields.** Current source: + [publication.py:328](../services/skills/publication.py#L328), especially line + 342; the bypassed retrieval condition is + [skills.py:703](../services/memory/skills.py#L703). Original source: + [routes/skills_routes.py:1039](https://github.com/odysseus-dev/odysseus/blob/86b886c611338282af6d4a717246ea4e256f58df/routes/skills_routes.py#L1039). + `test_rewrite_cannot_promote_model_text_to_builtin` uses the real manager and + saves model markdown with `source: builtin`, `status: published` and + `confidence: 1`. Identity/owner remain pinned, but the lifecycle fields persist + and the skill becomes retrieval-eligible without a passing audit. Corrective + change: allowlist procedure and approved descriptive/retrieval fields during + model rewrites; preserve existing owner, provenance, publication, confidence, + audit and teacher lifecycle state. Only verified policy may update those + fields. Metadata-only fixes must preserve the procedure or trigger a retest. + Add real-manager tests for self, teacher and metadata-only rewrites. + +2. **Preference loading fails open for automatic publication.** Current source: + [publication.py:13](../services/skills/publication.py#L13) and + [skills.py:59](../services/memory/skills.py#L59). Original source: + [routes/skills_routes.py:710](https://github.com/odysseus-dev/odysseus/blob/86b886c611338282af6d4a717246ea4e256f58df/routes/skills_routes.py#L710). + `test_preferences_read_failure_does_not_enable_auto_publication` makes the + loader raise `PermissionError`; automatic publication is enabled rather than + preserving the owner's possible opt-out. Corrective change: distinguish a + successfully loaded empty preference store from an exception, log the failure + and disable automatic publication on exceptions. Align the manager retrieval + preference adapter in the separate fix. Test unreadable/malformed existing + preference files and the ordinary empty-store default independently. + +3. **Manual cancellation retains private credentials and stale running state.** + Current source: + [agent_runner.py:125](../services/skills/agent_runner.py#L125) and + [job_coordinator.py:49](../services/skills/job_coordinator.py#L49). Original: + [routes/skills_routes.py:635](https://github.com/odysseus-dev/odysseus/blob/86b886c611338282af6d4a717246ea4e256f58df/routes/skills_routes.py#L635). + `test_manual_cancellation_cleans_sensitive_continuation` suspends and cancels + the stream. The stream closes in its originating task, but job status remains + running and `_run.headers.Authorization` remains resident. Polling still hides + these fields; this is retention, not demonstrated disclosure. Corrective + change: handle `CancelledError`, mark cancelled, clear private continuation + data, retire only the attached exact approval and re-raise. Preserve state on + a normal awaiting-approval return. Retain task handles for coordinated + replacement/cancellation and test owner/approval isolation. + +4. **Approval pauses during retries lead to demotion.** Current source: + [audit_workflow.py:244](../services/skills/audit_workflow.py#L244), teacher + retry at line 280 and demotion at line 303. Original: + [routes/skills_routes.py:1448](https://github.com/odysseus-dev/odysseus/blob/86b886c611338282af6d4a717246ea4e256f58df/routes/skills_routes.py#L1448) + and lines 1484/1507. `test_retry_approval_preserves_publication_and_confidence` + begins published at .9, fails the first grade, then pauses the rewritten test + for approval. The result is flagged/draft/.35. The lower-level approval is + correctly denied; state handling afterwards is wrong. Corrective change: + immediately recognize `approval_required` after every retry, stop escalation + and demotion, and preserve the intended pre-pause lifecycle state. Coordinate + with item 1 so model rewrites cannot replace that saved state. Test both + self/teacher retries, repeated pauses and batch protection. + +5. **Duplicate losers are binned after keeper publication fails.** Current: + [publication.py:217](../services/skills/publication.py#L217), including loser + writes at line 228. Original: + [routes/skills_routes.py:914](https://github.com/odysseus-dev/odysseus/blob/86b886c611338282af6d4a717246ea4e256f58df/routes/skills_routes.py#L914). + `test_duplicate_losers_retained_if_keeper_write_fails` raises disk-full during + keeper publication and still observes a loser binned. Corrective change: + require confirmed keeper persistence before binning losers when automatic + publication is enabled. Treat both exceptions and False returns as failure, + retain candidates and report recovery information. Preserve existing keeper + selection and duplicate scoring rules; add failure/retry tests. + +6. **Reported publication status can disagree with canonical storage.** Current: + [publication.py:320](../services/skills/publication.py#L320). Original: + [routes/skills_routes.py:1017](https://github.com/odysseus-dev/odysseus/blob/86b886c611338282af6d4a717246ea4e256f58df/routes/skills_routes.py#L1017). + `test_publication_does_not_report_unpersisted_success` returns False from the + manager update; the helper reports published while the stored skill remains + draft. Exceptions are also swallowed by the inspected code. Corrective change: + check persistence success, log failures and return reloaded canonical state or + an explicit persistence error. Test False/exception outcomes and the job/UI + projection. Coordinate this change with item 5 without changing publication + thresholds or scoring. + +Maestrum's read-only triage delegation produced no usable result within its +usefulness window and was cancelled. Cleanup completed and the session records +an interrupted result, not a successful review. The coordinator reclaimed the +remaining report work and verified the source references and reproduction logs +above. No worker claim was used as proof. No GitHub issue, new PR, maintainer +comment or PR-description edit was made. diff --git a/routes/skills_routes.py b/routes/skills_routes.py index 12f7cdd79..f469fc305 100644 --- a/routes/skills_routes.py +++ b/routes/skills_routes.py @@ -8,8 +8,6 @@ The on-disk format is SKILL.md (frontmatter + structured body) under """ import logging -import re -from contextlib import aclosing from typing import List, Optional import httpx @@ -19,44 +17,25 @@ from pydantic import BaseModel, Field from services.memory.skills import SkillsManager from src.auth_helpers import get_current_user -from src.prompt_security import untrusted_context_message -from src.text_helpers import strip_closed_think_blocks +from services.skills import evaluation as _evaluation +from services.skills import agent_runner as _agent_runner +from services.skills import audit_workflow as _audit_workflow +from services.skills import model_selection as _model_selection +from services.skills import prompts as _prompts +from services.skills import publication as _publication +from services.skills import job_coordinator as _job_coordinator +from services.skills.dependencies import AuditDependencies from src.utility_effort import effort_for_call from core.middleware import require_admin logger = logging.getLogger(__name__) -# Last-resort verdict extraction from a teacher/verifier model's prose (run when -# JSON parsing fails). `["\'\s:]*` already consumes whitespace, so the original -# trailing `\s*` made two adjacent \s-matching quantifiers that backtrack O(n^2) -# on a `verdict` + whitespace flood in untrusted model output (CodeQL -# py/polynomial-redos). Without it a single unbounded quantifier remains — the -# matched text is identical, and the scan is linear. -_VERDICT_PROSE_RE = re.compile( - r'verdict["\'\s:]*["\']?(pass|needs_work|fail|inconclusive)', re.I -) +_VERDICT_PROSE_RE = _evaluation._VERDICT_PROSE_RE +_verdict_efficiency = _evaluation.verdict_efficiency -def _verdict_efficiency(verdict: Optional[dict]) -> dict: - if not isinstance(verdict, dict): - return {} - out = {"audit_summary": str(verdict.get("summary") or "")[:2000]} - for key in ("saved_turns", "saved_tool_calls"): - if key not in verdict: - continue - try: - out[key] = int(verdict.get(key)) - except (TypeError, ValueError): - pass - baseline = str(verdict.get("baseline_verdict") or "").lower().strip() - if baseline in {"better", "same", "worse", "unknown"}: - out["baseline_verdict"] = baseline - if "usefulness" in verdict: - try: - out["usefulness"] = max(0.0, min(1.0, float(verdict.get("usefulness")))) - except (TypeError, ValueError): - pass - return out +# Compatibility bindings keep existing imports and offline patch targets. +# Implementations receive current collaborators through _audit_dependencies. class SkillAddRequest(BaseModel): @@ -116,54 +95,18 @@ class SkillUpdateRequest(BaseModel): steps: Optional[List[str]] = None -def _skill_test_task(skill: dict) -> str: - """Build the one shared task used by both sides of an audit comparison.""" - if not isinstance(skill, dict): - skill = {} - ctx = (skill.get("when_to_use") or skill.get("description") or skill.get("name") or "").strip() - return ( - "Complete this task end-to-end. Use this exact task context and do not ask " - "the user for more input. If a harmless fixture is needed, create the " - "smallest realistic one that satisfies the context, state it explicitly, " - "then complete and verify the result. Do not perform destructive or " - "externally visible actions. Task context: " + (ctx or "(general)") - ) +_skill_test_task = _prompts._skill_test_task def _skill_baseline_task(skill: dict) -> str: - """Backward-compatible alias; both audit arms must receive identical text.""" + """Compatibility alias using the currently bound shared task builder.""" return _skill_test_task(skill) -def _skill_test_messages(md: str, task: str) -> list[dict]: - """Keep user-editable skill text out of the trusted system role.""" - return [ - { - "role": "system", - "content": ( - "You are TESTING a skill. Follow the supplied reusable procedure " - "to complete the user's task for real, using available tools step " - "by step. If the skill is wrong, unclear, or references tools that " - "do not exist, do your best; the problems will be reviewed afterward." - ), - }, - untrusted_context_message("skill under test", md), - {"role": "user", "content": task}, - ] +_skill_test_messages = _prompts._skill_test_messages -def _skill_baseline_messages(task: str) -> list[dict]: - return [ - { - "role": "system", - "content": ( - "You are completing a baseline audit run. Do not use any saved " - "skill text. Solve the user's task using only your normal tools " - "and reasoning." - ), - }, - {"role": "user", "content": task}, - ] +_skill_baseline_messages = _prompts._skill_baseline_messages async def _eval_skill_run(skill_md: str, task: str, transcript: str, @@ -172,358 +115,38 @@ async def _eval_skill_run(skill_md: str, task: str, transcript: str, skill_stats: Optional[dict] = None, baseline_stats: Optional[dict] = None, workload: str = "foreground", owner: Optional[str] = None) -> dict: - """LLM-as-judge: grade a skill test run from its transcript. Advisory only. - - Robust against local reasoning models (strips , lenient JSON, - generous token budget) — same defensive parsing used elsewhere. - """ - import json as _json - import re as _re - from src.llm_core import llm_call_async - - sys_prompt = ( - "You are a strict QA reviewer judging whether an AI 'skill' (a reusable " - "procedure) actually works. You are given the SKILL, the TASK it was tested " - "on, and the TRANSCRIPT of the agent's run.\n\n" - "Judge honestly:\n" - "- The functional verdict judges ONLY whether the SKILL RUN completed the " - "task correctly and whether the skill procedure is usable. A correct skill " - "run is pass even when the baseline is equally good. Record comparative " - "value separately in baseline_verdict/usefulness.\n" - "- Did following the skill accomplish the task accurately?\n" - "- Are the steps clear, correct, and reproducible?\n" - "- Did it reference tools/commands that don't exist or that errored?\n" - "- Separately, compared with the baseline run WITHOUT the skill, did the skill make " - "the agent more accurate, use fewer turns/tool calls, or avoid avoidable " - "thrashing?\n" - "- Do NOT penalize a skill merely for being broad or generic. A broad " - "skill is useful if it makes the agent finish correctly in fewer turns " - "or with fewer tools than baseline.\n" - "- METADATA: do the frontmatter fields match what the skill actually does? " - "Flag wrong/misleading/missing tags, a wrong category, a when_to_use that " - "doesn't describe the real trigger, or a description that oversells or " - "mismatches the body. List each metadata problem in 'issues' (prefix it " - "with 'metadata:'). Metadata problems alone do NOT make the verdict 'fail' " - "if the procedure works — note them as issues on an otherwise-passing run.\n\n" - "Never use inconclusive merely because the baseline was the same or better. " - "IMPORTANT — fairness rule: if the SKILL RUN could NOT proceed because it lacked " - "an input or target the test never provided (e.g. there was no document/" - "email/data to act on, so the agent reasonably asked for it), that is NOT " - "the skill's fault. Return verdict \"inconclusive\" — do NOT mark it fail " - "or needs_work. Only judge the skill's PROCEDURE; reserve fail/needs_work " - "for when the steps themselves are wrong, vague, or reference missing tools.\n\n" - "If you need to reason, do it inside FIRST. Then output " - "ONLY this JSON (no fences):\n" - 'Set baseline_verdict to how the SKILL RUN compares to the BASELINE RUN.\n\n' - '{"verdict": "pass" | "needs_work" | "fail" | "inconclusive", ' - '"confidence": 0.0-1.0, "summary": "one short sentence", ' - '"issues": ["short issue", ...], ' - '"baseline_verdict": "better" | "same" | "worse" | "unknown", ' - '"usefulness": 0.0-1.0, "saved_turns": integer, "saved_tool_calls": integer}' + return await _evaluation.evaluate_run( + skill_md, task, transcript, url, model, headers, baseline_transcript, + skill_stats, baseline_stats, workload, owner, _effort_for_call=effort_for_call, ) - # Give the judge plenty of transcript, and when it must trim, keep the TAIL - # (the final result lives at the end) plus a bit of the head — truncating to - # a short prefix made the judge wrongly call complete runs "incomplete / - # missing sections" because it never saw the end. - def _clip(t: str, limit: int = 24000) -> str: - t = (t or "").strip() or "(no output produced)" - if len(t) <= limit: - return t - head = limit // 4 - return t[:head] + "\n\n…[transcript trimmed for length]…\n\n" + t[-(limit - head):] - skill_stats = skill_stats or {} - baseline_stats = baseline_stats or {} - user_msg = ( - f"=== SKILL ===\n{(skill_md or '')[:4000]}\n\n" - f"=== TASK ===\n{task}\n\n" - f"=== SKILL RUN STATS ===\n" - f"turns={skill_stats.get('turns', 'unknown')} " - f"tool_calls={skill_stats.get('tool_calls', 'unknown')}\n\n" - f"=== SKILL RUN TRANSCRIPT ===\n{_clip(transcript)}\n\n" - f"=== BASELINE RUN WITHOUT SKILL STATS ===\n" - f"turns={baseline_stats.get('turns', 'unknown')} " - f"tool_calls={baseline_stats.get('tool_calls', 'unknown')}\n\n" - f"=== BASELINE RUN WITHOUT SKILL TRANSCRIPT ===\n{_clip(baseline_transcript)}" - ) - _VERDICTS = ("pass", "needs_work", "fail", "inconclusive") - - def _parse(raw: str): - """Return a final result dict on success, or None if unparseable.""" - text = (raw or '') - # Strip closed think blocks. If a was opened but never closed - # (the model ran out of budget mid-reasoning), drop everything from it - # onward so its stray braces don't poison JSON extraction. - text = strip_closed_think_blocks(text) - text = _re.sub(r'[\s\S]*$', '', text, flags=_re.I).strip() - - def _coerce(d): - return d if (isinstance(d, dict) and "verdict" in d) else None - - data = None - # Scan every balanced {...} candidate and keep the LAST one that parses - # and carries a "verdict" — the transcript is full of JSON API bodies, - # so a naive first-brace/last-brace span almost never parses. - for m in _re.finditer(r'\{[\s\S]*?\}', text): - frag = m.group(0) - for cand in (frag, _re.sub(r',(\s*[}\]])', r'\1', frag)): - try: - d = _coerce(_json.loads(cand)) - except Exception: - d = None - if d is not None: - data = d - # Fallback to the greedy outermost span (handles nested objects the - # non-greedy scan above splits apart). - if data is None: - a, b = text.find('{'), text.rfind('}') - if a >= 0 and b > a: - frag = text[a:b + 1] - for cand in (frag, _re.sub(r',(\s*[}\]])', r'\1', frag)): - try: - d = _coerce(_json.loads(cand)) - except Exception: - d = None - if d is not None: - data = d - break - - v = str(data.get("verdict", "")).lower().strip() if isinstance(data, dict) else None - # Last resort: pull the verdict keyword straight out of the prose so a - # clearly-decided run isn't thrown away as "unparseable". - if v not in _VERDICTS: - km = _VERDICT_PROSE_RE.search(text) - if km: - v = km.group(1).lower() - if data is None: - data = {} - if not isinstance(data, dict) or v not in _VERDICTS: - return None - try: - conf = float(data.get("confidence", 0)) - except (TypeError, ValueError): - conf = 0 - try: - usefulness = float(data.get("usefulness", 0.0)) - except (TypeError, ValueError): - usefulness = 0.0 - try: - saved_turns = int(data.get("saved_turns", 0)) - except (TypeError, ValueError): - saved_turns = 0 - try: - saved_tool_calls = int(data.get("saved_tool_calls", 0)) - except (TypeError, ValueError): - saved_tool_calls = 0 - return { - "verdict": v, - "confidence": max(0.0, min(1.0, conf)), - "summary": str(data.get("summary", ""))[:400], - "issues": [str(x)[:200] for x in (data.get("issues") or []) if str(x).strip()][:8], - "baseline_verdict": ( - str(data.get("baseline_verdict", "unknown")).lower().strip() - if str(data.get("baseline_verdict", "unknown")).lower().strip() in {"better", "same", "worse", "unknown"} - else "unknown" - ), - "usefulness": max(0.0, min(1.0, usefulness)), - "saved_turns": saved_turns, - "saved_tool_calls": saved_tool_calls, - } - - # Two attempts: the first lets the judge reason; if a heavy reasoning model - # burns its budget inside and never emits the JSON, the second - # forbids thinking and demands the JSON immediately. - last_text = "" - last_err = None - for attempt in range(2): - msgs = [{"role": "system", "content": sys_prompt}, - {"role": "user", "content": user_msg}] - if attempt == 1: - msgs[0]["content"] = ( - sys_prompt + "\n\nDO NOT use or any reasoning. Your reply " - "must START with '{' and be ONLY the JSON object, nothing else." - ) - try: - raw = await llm_call_async( - # Generous budget so a heavy reasoner can think AND still have - # room to emit the JSON afterwards (reasoning tokens come out of - # this same cap; the server clamps to its own max). - url, model, msgs, - temperature=0.1, max_tokens=32768, headers=headers, timeout=180, - workload=workload, - reasoning_effort=effort_for_call(url, model, owner), - ) - except Exception as e: - # Don't give up on a transient first-attempt error — let the second - # (no-think) attempt run before reporting failure. - last_err = e - continue - last_text = (raw or '') - parsed = _parse(raw) - if parsed is not None: - return parsed - - if last_err is not None and not last_text: - return {"verdict": "unknown", "confidence": 0, "summary": f"Review call failed: {last_err}", "issues": []} - return {"verdict": "unknown", "confidence": 0, - "summary": "Reviewer returned unparseable output.", "issues": [], "raw": last_text[:300]} async def _eval_skill_necessity(skill_md: str, others: list, url: str, model: str, headers: Optional[dict], workload: str = "foreground", owner: Optional[str] = None) -> Optional[dict]: - """Advisory judge: is this skill worth keeping, or is it redundant / trivially - unnecessary? Sees the OTHER skills' names+descriptions so it can spot - duplicates. Returns {necessary, redundant_with, reason} or None. Never acts — - purely a flag the UI surfaces.""" - import json as _json - import re as _re - from src.llm_core import llm_call_async - - catalog = "\n".join(f"- {o.get('name')}: {o.get('description', '')}" for o in others) or "(no other skills)" - sys_prompt = ( - "You assess whether a reusable AI 'skill' (a saved procedure) is worth keeping. " - "A skill is UNNECESSARY if it essentially duplicates another skill in the library. " - "Do NOT call a skill unnecessary merely because it is broad or generic; broad " - "skills can be worth keeping when they make a capable assistant finish accurately " - "with fewer turns or fewer tool calls than it would without the skill. A skill IS " - "necessary if it captures a reusable trigger, procedure, tool sequence, or " - "hard-won detail that could improve future runs.\n\n" - "Be conservative: only call it unnecessary when you're confident. Reason in " - " first if needed, then output ONLY this JSON:\n" - '{"necessary": true|false, "redundant_with": ["skill-name", ...], ' - '"reason": "one short sentence"}' + return await _evaluation.evaluate_necessity( + skill_md, others, url, model, headers, workload, owner, + _effort_for_call=effort_for_call, ) - user_msg = ( - f"=== SKILL UNDER REVIEW ===\n{(skill_md or '')[:3000]}\n\n" - f"=== OTHER SKILLS IN THE LIBRARY ===\n{catalog[:4000]}" - ) - try: - raw = await llm_call_async( - url, model, - [{"role": "system", "content": sys_prompt}, {"role": "user", "content": user_msg}], - temperature=0.1, max_tokens=8192, headers=headers, timeout=120, - workload=workload, - reasoning_effort=effort_for_call(url, model, owner), - ) - except Exception as e: - logger.warning(f"Necessity check failed: {e}") - return None - text = strip_closed_think_blocks(raw or '') - text = _re.sub(r'[\s\S]*$', '', text, flags=_re.I).strip() - data = None - a, b = text.find('{'), text.rfind('}') - if a >= 0 and b > a: - frag = text[a:b + 1] - for cand in (frag, _re.sub(r',(\s*[}\]])', r'\1', frag)): - try: - data = _json.loads(cand) - break - except Exception: - continue - if not isinstance(data, dict) or "necessary" not in data: - return None - return { - "necessary": bool(data.get("necessary", True)), - "redundant_with": [str(x)[:80] for x in (data.get("redundant_with") or []) if str(x).strip()][:6], - "reason": str(data.get("reason", ""))[:200], - } -def _should_check_retrieval_precision(skill: dict) -> bool: - """Cheap prefilter for the expensive retrieval-precision judge. - - Skills with broad tags like "network" or "document" are the ones most likely - to over-inject. Narrow command/vendor tags alone are fine. - """ - broad = { - "arch", "arch linux", "linux", "network", "networking", "wifi", - "installation", "install", "system", "ssh", "document", "documents", - "search", "email", "calendar", "gpu", "server", "python", - } - if not isinstance(skill, dict): - return False - tags = {str(t or "").strip().lower() for t in (skill.get("tags") or [])} - if tags & broad: - return True - text = " ".join([ - str(skill.get("name") or ""), - str(skill.get("description") or ""), - str(skill.get("when_to_use") or ""), - ]).lower() - return sum(1 for t in broad if t in text) >= 2 +_should_check_retrieval_precision = _evaluation.should_check_retrieval_precision async def _eval_skill_retrieval_precision(skill_md: str, others: list, url: str, model: str, headers: Optional[dict], workload: str = "foreground", owner: Optional[str] = None) -> Optional[dict]: - """Advisory judge: would this skill's metadata make retrieval over-select it? - - This is distinct from "does the procedure work?". It asks whether tags, - description, and when_to_use are specific enough that the skill won't be - injected into adjacent but wrong tasks. - """ - import json as _json - import re as _re - from src.llm_core import llm_call_async - - catalog = "\n".join(f"- {o.get('name')}: {o.get('description', '')}" for o in others[:80]) or "(no other skills)" - sys_prompt = ( - "You are auditing retrieval metadata for a reusable AI skill. The app selects " - "skills by matching user requests against the skill name, description, tags, " - "when_to_use, and procedure text. Judge whether this skill is likely to be " - "over-selected for nearby but wrong requests.\n\n" - "Focus ONLY on metadata/retrieval precision, not whether the procedure works. " - "Flag broad tags such as network, installation, system, document, search, ssh, " - "python, gpu, or server when they would cause this narrow skill to match too " - "many adjacent tasks. Recommend narrower tags/when_to_use wording. Compare " - "against the other skills to spot boundaries.\n\n" - "Return ok=true only when the trigger metadata is narrow enough. If not ok, " - "issues MUST start with 'metadata: retrieval:' and be actionable. Output ONLY JSON:\n" - '{"ok": true|false, "summary": "one short sentence", "issues": ["metadata: retrieval: ..."]}' + return await _evaluation.evaluate_retrieval_precision( + skill_md, others, url, model, headers, workload, owner, + _effort_for_call=effort_for_call, ) - user_msg = ( - f"=== SKILL UNDER REVIEW ===\n{(skill_md or '')[:5000]}\n\n" - f"=== OTHER SKILLS IN LIBRARY ===\n{catalog[:5000]}\n\n" - "Decide if this skill's retrieval metadata should be narrowed so it only " - "fires for its intended scenario and not for adjacent skills above." - ) - try: - raw = await llm_call_async( - url, model, - [{"role": "system", "content": sys_prompt}, {"role": "user", "content": user_msg}], - temperature=0.1, max_tokens=4096, headers=headers, timeout=90, - workload=workload, - reasoning_effort=effort_for_call(url, model, owner), - ) - except Exception as e: - logger.warning(f"Retrieval precision check failed: {e}") - return None - text = strip_closed_think_blocks(raw or '') - text = _re.sub(r'[\s\S]*$', '', text, flags=_re.I).strip() - data = None - a, b = text.find('{'), text.rfind('}') - if a >= 0 and b > a: - frag = text[a:b + 1] - for cand in (frag, _re.sub(r',(\s*[}\]])', r'\1', frag)): - try: - data = _json.loads(cand) - break - except Exception: - continue - if not isinstance(data, dict) or "ok" not in data: - return None - return { - "ok": bool(data.get("ok")), - "summary": str(data.get("summary", ""))[:300], - "issues": [str(x)[:220] for x in (data.get("issues") or []) if str(x).strip()][:6], - } # In-memory skill-test jobs, keyed by (owner, skill_name). Runs server-side so # the test survives the modal being closed; the UI polls /test-status. (Not # persisted across restart — it's a "come back in a bit" convenience.) -_skill_test_jobs: dict = {} +_skill_test_jobs = _agent_runner._skill_test_jobs async def _run_skill_test_job( @@ -542,1175 +165,125 @@ async def _run_skill_test_job( exact_approval=None, request_authority=None, ): - """Background coroutine: run the skill in an agent loop, capture a condensed - log + transcript, then have the judge grade it. Writes into _skill_test_jobs.""" - import json as _json - from src.agent_loop import stream_agent_loop - from src.agent_runtime.authority import RequestAuthority - - job = _skill_test_jobs.get(key) - if job is None: - return - log = job["log"] - transcript = transcript if isinstance(transcript, list) else [] - say_buf = [] - skill_stats = {"turns": 0, "tool_calls": 0} - - def _flush_say(): - if say_buf: - log.append({"type": "say", "text": "".join(say_buf)}) - say_buf.clear() - - messages = list(messages) if isinstance(messages, list) else _skill_test_messages(md, task) - try: - # aclosing: pausing for an approval leaves this loop early, and the - # stream's ContextVar cleanup must run in this task, not in asyncio's - # finalizer (#6635). - async with aclosing(stream_agent_loop( - url, model, messages, headers=headers, - temperature=0.3, max_tokens=0, max_rounds=8, owner=owner, - exact_approval=exact_approval, - request_authority=(request_authority or ( - exact_approval.pending.request_authority if exact_approval is not None else None - ) or RequestAuthority.empty(owner=owner)), - )) as stream: - async for chunk in stream: - if not chunk.startswith("data: ") or chunk.strip() == "data: [DONE]": - continue - try: - d = _json.loads(chunk[6:]) - except Exception: - continue - if d.get("delta"): - say_buf.append(d["delta"]); transcript.append(d["delta"]) - elif d.get("type") == "tool_start": - _flush_say() - skill_stats["tool_calls"] += 1 - cmd = str(d.get("command") or d.get("args") or "")[:300] - log.append({"type": "tool_start", "tool": d.get("tool"), "command": cmd}) - transcript.append(f"\n[tool {d.get('tool')}] {cmd}\n") - elif d.get("type") == "tool_output": - _flush_say() - out = str(d.get("output") or "")[:600] - tool_log = {"type": "tool_output", "output": out} - approval = d.get("ask_user") - if isinstance(approval, dict): - tool_log["ask_user"] = approval - log.append(tool_log) - transcript.append(f"[output] {out}\n") - if ( - isinstance(approval, dict) - and approval.get("kind") == "tool_approval" - and approval.get("approval_id") - ): - # Manual skill tests have their own polling UI instead of a - # chat session. Pause the run and retain only server-side - # continuation state until the same owner approves/denies - # this exact sealed action. - job["status"] = "awaiting_approval" - job["approval"] = approval - job["_transcript"] = transcript - return - elif d.get("type") == "agent_step": - _flush_say() - try: - skill_stats["turns"] = max(skill_stats["turns"], int(d.get("round") or 0)) - except (TypeError, ValueError): - pass - log.append({"type": "agent_step", "round": d.get("round")}) - transcript.append(f"\n--- round {d.get('round')} ---\n") - elif d.get("type") == "metrics": - data = d.get("data") or {} - try: - skill_stats["turns"] = max(skill_stats["turns"], int(data.get("agent_rounds") or 0)) - except (TypeError, ValueError): - pass - try: - skill_stats["tool_calls"] = max(skill_stats["tool_calls"], int(data.get("tool_calls") or 0)) - except (TypeError, ValueError): - pass - if len(log) > 600: - del log[0:len(log) - 600] - _flush_say() - except Exception as e: - _flush_say() - log.append({"type": "error", "error": str(e)}) - - job.pop("approval", None) - job.pop("_transcript", None) - job.pop("_run", None) - log.append({"type": "evaluating"}) - try: - baseline_task = task - log.append({"type": "agent_step", "round": "baseline"}) - baseline_transcript, baseline_stats, baseline_approval = await _run_skill_audit_arm( - _skill_baseline_messages(baseline_task), - url, - model, - headers, - owner, - ) - if baseline_approval is not None: - try: - from src.tool_approvals import tool_approval_store - tool_approval_store.consume( - baseline_approval.get("approval_id"), - decision="deny", - owner=owner, - session_id=None, - ) - except Exception: - logger.debug("Could not retire manual-test baseline approval", exc_info=True) - job["verdict"] = await _eval_skill_run( - md, - task, - "".join(transcript), - url, - model, - headers, - baseline_transcript=baseline_transcript, - skill_stats=skill_stats, - baseline_stats=baseline_stats, - owner=owner, - ) - except Exception as e: - job["verdict"] = {"verdict": "unknown", "confidence": 0, "summary": f"Eval failed: {e}", "issues": []} - # Record the result so the card shows a 'verified' check (a manual test - # never involves the teacher) and nudge the confidence score to match the - # verdict — same scale as Audit-all's pass=0.95. inconclusive/unknown leave - # the score alone (missing-fixture or parse failures shouldn't punish it). - if skills_manager is not None: - v = (job["verdict"] or {}).get("verdict") or "unknown" - try: - skills_manager.set_audit( - name, - v, - by_teacher=False, - worker_model=model, - owner=owner, - **_verdict_efficiency(job.get("verdict")), - ) - except Exception: - pass - conf = {"pass": 0.95, "needs_work": 0.6, "fail": 0.4}.get(v) - if conf is not None: - try: - skills_manager.update_skill(name, {"confidence": conf}, owner=owner) - except Exception: - pass - job["status"] = "done" + return await _agent_runner.run_test_job( + key, name, md, task, url, model, headers, owner, skills_manager, + messages=messages, transcript=transcript, exact_approval=exact_approval, + request_authority=request_authority, deps=_audit_dependencies(), + ) # ── Autonomous skill audit: test → judge → self-edit → retry → teacher → flag ── -_skill_audit_jobs: dict = {} +_skill_audit_jobs = _audit_workflow._skill_audit_jobs -def _audit_auto_publish_policy(owner) -> tuple[bool, float]: - """Return (auto_publish_enabled, minimum_confidence) for audit finalization.""" - try: - from routes.prefs_routes import _load_for_user - prefs = _load_for_user(owner) or {} - except Exception: - prefs = {} - try: - from src.settings import get_setting - default_min = get_setting("skill_autosave_min_confidence", 0.85) - except Exception: - default_min = 0.85 - enabled = bool(prefs.get("auto_approve_skills", True)) - try: - min_conf = float(prefs.get("skill_min_confidence", default_min)) - except (TypeError, ValueError): - min_conf = 0.85 - return enabled, max(0.0, min(1.0, min_conf)) +_audit_auto_publish_policy = _publication.auto_publish_policy -def _skill_duplicate_blocker(skills_manager, name: str, owner) -> Optional[str]: - """Cheap duplicate guard matching the UI's duplicate grouping. - - The LLM necessity check catches semantic redundancy, but the UI also has a - cheap similarity pass. Use the same broad signal before auto-publishing so - a high-scoring lower-priority duplicate stays draft. - """ - import re as _re - - def _tokens(sk: dict) -> set[str]: - text = " ".join([ - str(sk.get("name") or ""), - str(sk.get("description") or ""), - str(sk.get("when_to_use") or ""), - " ".join(sk.get("procedure") or []), - " ".join(sk.get("tags") or []), - ]).lower() - text = _re.sub(r"-\d+\b", "", text) - return { - t for t in _re.split(r"[^a-z0-9]+", text) - if len(t) > 2 and t not in {"the", "and", "with", "for", "from", "using"} - } - - def _sim(a: dict, b: dict) -> float: - A, B = _tokens(a), _tokens(b) - if not A or not B: - return 0.0 - return len(A & B) / max(1, len(A | B)) - - def _base(n: str) -> str: - return _re.sub(r"-\d+$", "", str(n or "")) - - def _score(sk: dict) -> float: - return ( - (100000 if (sk.get("status") == "published") else 0) - + int(sk.get("uses") or 0) * 100 - + round(float(sk.get("confidence") or 0) * 100) - + (-5 if sk.get("audit_by_teacher") else 0) - - (len(str(sk.get("name") or "")) / 1000) - ) - - skills = [s for s in skills_manager.load(owner=owner) if s.get("status") != "binned"] - current = next((s for s in skills if (s.get("name") or s.get("id")) == name), None) - if not current: - return None - duplicates = [] - cur_name = current.get("name") or current.get("id") or name - for other in skills: - other_name = other.get("name") or other.get("id") - if not other_name or other_name == cur_name: - continue - if _base(cur_name) == _base(other_name) or _sim(current, other) >= 0.38: - duplicates.append(other) - if not duplicates: - return None - keeper = sorted([current, *duplicates], key=_score, reverse=True)[0] - keeper_name = keeper.get("name") or keeper.get("id") or "" - if keeper_name and keeper_name != cur_name: - try: - skills_manager.set_necessity( - cur_name, - False, - [keeper_name], - f"Lower-priority duplicate of {keeper_name}", - owner=owner, - ) - except Exception: - pass - return keeper_name - return None +_skill_duplicate_blocker = _publication.duplicate_blocker def _finalize_audit_batch(skills_manager, results: list[dict], owner, log) -> None: - """Draft audited failures, bin duplicate losers, and publish the best copy. - - Binned skills remain on disk for recovery and inspection, but the Skills - manager excludes them from retrieval. Only skills actually processed by - this audit job are changed here; an unrelated existing skill is never - moved just because it resembles an audited one. - """ - import re as _re - - auto_publish, min_conf = _audit_auto_publish_policy(owner) - current = [ - s for s in skills_manager.load(owner=owner) - if s.get("source") != "builtin" and s.get("status") != "binned" - ] - by_name = {s.get("name"): s for s in current if s.get("name")} - processed = {str(r.get("skill")) for r in results if r.get("skill")} - protected = { - str(r.get("skill")) for r in results - if r.get("skill") and r.get("result") == "approval_required" - } - - def tokens(sk: dict) -> set[str]: - text = " ".join([ - str(sk.get("name") or ""), str(sk.get("description") or ""), - str(sk.get("when_to_use") or ""), " ".join(sk.get("procedure") or []), - " ".join(sk.get("tags") or []), - ]).lower() - text = _re.sub(r"-\d+\b", "", text) - return { - t for t in _re.split(r"[^a-z0-9]+", text) - if len(t) > 2 and t not in {"the", "and", "with", "for", "from", "using"} - } - - def similar(a: dict, b: dict) -> float: - left, right = tokens(a), tokens(b) - return len(left & right) / max(1, len(left | right)) if left and right else 0.0 - - def base(name: str) -> str: - return _re.sub(r"-\d+$", "", str(name or "")) - - def score(sk: dict) -> float: - try: - confidence = float(sk.get("confidence") or 0) - except (TypeError, ValueError): - confidence = 0.0 - return ( - (100000 if sk.get("status") == "published" else 0) - + int(sk.get("uses") or 0) * 100 - + round(confidence * 100) - + (-5 if sk.get("audit_by_teacher") else 0) - - len(str(sk.get("name") or "")) / 1000 - ) - - # Anything that does not clear the configured policy stays a draft. Drafts - # are excluded from retrieval/injection by SkillsManager.index_for(). - for name in processed - protected: - skill = by_name.get(name) - if not skill: - continue - verdict = str(skill.get("audit_verdict") or "").lower() - try: - confidence = float(skill.get("confidence") or 0) - except (TypeError, ValueError): - confidence = 0.0 - if verdict in {"needs_work", "fail"} or ( - verdict == "pass" and confidence < min_conf - ): - try: - skills_manager.update_skill(name, {"status": "draft"}, owner=owner) - log(f"{name}: kept as draft after audit") - except Exception: - logger.warning("Could not bin audited skill %s", name, exc_info=True) - - # Build the same connected duplicate groups shown by the UI. - parent = {s["name"]: s["name"] for s in current} - - def find(name: str) -> str: - while parent[name] != name: - parent[name] = parent[parent[name]] - name = parent[name] - return name - - def unite(left: str, right: str) -> None: - left, right = find(left), find(right) - if left != right: - parent[right] = left - - for index, left in enumerate(current): - for right in current[index + 1:]: - if base(left["name"]) == base(right["name"]) or similar(left, right) >= 0.38: - unite(left["name"], right["name"]) - groups: dict[str, list[dict]] = {} - for skill in current: - groups.setdefault(find(skill["name"]), []).append(skill) - - for group in groups.values(): - if len(group) < 2: - continue - passing = [] - for skill in group: - if skill["name"] not in processed or skill["name"] in protected: - continue - if str(skill.get("audit_verdict") or "").lower() != "pass": - continue - try: - confidence = float(skill.get("confidence") or 0) - except (TypeError, ValueError): - confidence = 0.0 - if confidence >= min_conf: - passing.append(skill) - if not passing: - continue - keeper = max(passing, key=score) - if auto_publish: - try: - skills_manager.update_skill(keeper["name"], {"status": "published"}, owner=owner) - log(f"{keeper['name']}: auto-approved as best passing duplicate") - except Exception: - logger.warning("Could not auto-approve skill %s", keeper["name"], exc_info=True) - for skill in group: - name = skill["name"] - if name == keeper["name"] or name not in processed or name in protected: - continue - try: - skills_manager.update_skill(name, {"status": "binned"}, owner=owner) - log(f"{name}: moved to bin as duplicate of {keeper['name']}") - except Exception: - logger.warning("Could not bin duplicate skill %s", name, exc_info=True) - - -def _audit_flag_text(*parts) -> str: - text_parts = [] - for part in parts: - if isinstance(part, dict): - text_parts.extend(str(v or "") for v in part.values()) - elif isinstance(part, (list, tuple, set)): - text_parts.extend(str(v or "") for v in part) - else: - text_parts.append(str(part or "")) - return " ".join(text_parts).lower() - - -def _audit_utility_blocker(skill: Optional[dict], necessity: Optional[dict], - verdict_data: Optional[dict]) -> Optional[str]: - """Return a short reason when a passing skill still should stay draft. - - Broad/generic wording is not a blocker by itself. The blocker is whether - the skill failed to improve the agent versus a no-skill baseline, or whether - it duplicates another skill. - """ - generic_re = re.compile( - r"\b(duplicat\w*|redundan\w*|overlap\w*|same skill|same procedure)\b", - re.I, + return _publication.finalize_batch( + skills_manager, results, owner, log, deps=_audit_dependencies(), ) - if isinstance(necessity, dict): - reason = str(necessity.get("reason") or "") - if necessity.get("necessary") is False and generic_re.search(reason): - return reason or "Duplicate or redundant skill" - if isinstance(verdict_data, dict): - baseline_verdict = str(verdict_data.get("baseline_verdict") or "unknown").lower() - try: - usefulness = float(verdict_data.get("usefulness", 0.0) or 0.0) - except (TypeError, ValueError): - usefulness = 0.0 - try: - saved_turns = int(verdict_data.get("saved_turns", 0) or 0) - except (TypeError, ValueError): - saved_turns = 0 - try: - saved_tool_calls = int(verdict_data.get("saved_tool_calls", 0) or 0) - except (TypeError, ValueError): - saved_tool_calls = 0 - if baseline_verdict == "worse": - return "Skill performed worse than the no-skill baseline" - verdict_text = _audit_flag_text( - verdict_data.get("summary"), - verdict_data.get("issues") or [], - ) - if generic_re.search(verdict_text): - return "Audit flagged the skill as duplicate or redundant" - return None + +_audit_flag_text = _publication.audit_flag_text + + +_audit_utility_blocker = _publication.utility_blocker def _audit_finalize_status(skills_manager, name: str, owner, verdict: str, confidence: Optional[float], necessity: Optional[dict] = None, verdict_data: Optional[dict] = None) -> str: - """Apply the user's audit publishing policy. - - Audit is the final pass: skills that pass at/above the threshold are - published; failing or unnecessary/redundant skills are returned to draft. - Inconclusive runs preserve the existing state because they provide no - evidence either way. The completed batch moves duplicate losers to the bin. - """ - auto_publish, min_conf = _audit_auto_publish_policy(owner) - necessary = True - current = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) - if verdict in {"inconclusive", "unknown"}: - return (current or {}).get("status") or "draft" - utility_reason = _audit_utility_blocker(current, necessity, verdict_data) - if ( - isinstance(necessity, dict) - and necessity.get("necessary") is False - and necessity.get("redundant_with") - ): - necessary = False - if utility_reason: - necessary = False - try: - skills_manager.set_necessity(name, False, [], utility_reason, owner=owner) - except Exception: - pass - duplicate_of = _skill_duplicate_blocker(skills_manager, name, owner) if verdict == "pass" else None - if duplicate_of: - necessary = False - c = float(confidence or 0.0) - status = "published" if (auto_publish and necessary and verdict == "pass" and c >= min_conf) else "draft" - try: - skills_manager.update_skill(name, {"status": status}, owner=owner) - except Exception: - pass - return status + return _publication.finalize_status( + skills_manager, name, owner, verdict, confidence, necessity, verdict_data, + deps=_audit_dependencies(), + ) -def _apply_skill_md(skills_manager, name: str, md: str, owner) -> bool: - """Parse + persist an edited SKILL.md. Returns True on success.""" - try: - from services.memory.skill_format import Skill, slugify - sk = Skill.from_markdown(md) - # Pin the identity: the audit's fixer is now allowed to edit frontmatter - # (tags/category/when_to_use/description), but it must NEVER rename the - # skill — a changed `name` would move the dir and orphan the usage/audit - # sidecar entries that the caller keeps writing under the original name. - sk.name = name - return bool(skills_manager.update_skill(name, { - "name": sk.name, "description": sk.description, "version": sk.version, - "category": sk.category, "tags": sk.tags, "platforms": sk.platforms, - "requires_toolsets": sk.requires_toolsets, "fallback_for_toolsets": sk.fallback_for_toolsets, - "status": sk.status, "confidence": sk.confidence, "source": sk.source, - "teacher_model": sk.teacher_model, "owner": sk.owner or owner, - "when_to_use": sk.when_to_use, "procedure": sk.procedure, - "pitfalls": sk.pitfalls, "verification": sk.verification, "body_extra": sk.body_extra, - }, owner=owner)) - except Exception as e: - logger.warning(f"Audit: could not save edited skill {name}: {e}") - return False +_apply_skill_md = _publication.apply_markdown -class SkillAuditUnavailable(RuntimeError): - """The test infrastructure failed; this is not evidence about a skill.""" +SkillAuditUnavailable = _agent_runner.SkillAuditUnavailable async def _run_skill_audit_arm(messages: list[dict], url, model, headers, owner, workload: str = "foreground") -> tuple[str, dict, Optional[dict]]: - """Run one audit arm in the agent loop; return transcript, stats, approval.""" - import json as _json - from src.agent_loop import stream_agent_loop - from src.agent_runtime.authority import RequestAuthority, active_request_authority - transcript = [] - approval_required = None - stats = {"turns": 0, "tool_calls": 0} - try: - # max_tokens explicitly set: passing 0 lets some upstreams (Ollama, - # OpenAI-compat) generate an empty completion, which manifested as - # the skill test returning nothing while chat (which carries its - # preset's max_tokens) worked. 4096 matches the chat default. - # aclosing: stopping for an approval leaves this loop early (#6635). - async with aclosing(stream_agent_loop( - url, model, messages, headers=headers, - temperature=0.3, max_tokens=4096, max_rounds=8, - owner=owner, workload=workload, suppress_skills=True, - request_authority=(active_request_authority() or RequestAuthority.empty(owner=owner)), - )) as stream: - async for chunk in stream: - # Streams can include an SSE event line before the data line, - # notably `event: error`. Do not silently discard those failures. - payload = next((line[6:] for line in chunk.splitlines() if line.startswith("data: ")), None) - if payload is None or payload == "[DONE]": - continue - try: - d = _json.loads(payload) - except Exception: - continue - if d.get("error") or d.get("type") == "error": - raise SkillAuditUnavailable(str(d.get("error") or d.get("message") or "Audit stream failed")) - if d.get("delta"): - transcript.append(d["delta"]) - elif d.get("type") == "tool_start": - stats["tool_calls"] += 1 - transcript.append(f"\n[tool {d.get('tool')}] {str(d.get('command') or d.get('args') or '')[:300]}\n") - elif d.get("type") == "tool_output": - transcript.append(f"[output] {str(d.get('output') or '')[:600]}\n") - approval = d.get("ask_user") - if ( - isinstance(approval, dict) - and approval.get("kind") == "tool_approval" - ): - approval_required = approval - break - elif d.get("type") == "agent_step": - try: - stats["turns"] = max(stats["turns"], int(d.get("round") or 0)) - except (TypeError, ValueError): - pass - transcript.append(f"\n--- round {d.get('round')} ---\n") - elif d.get("type") == "metrics": - data = d.get("data") or {} - try: - stats["turns"] = max(stats["turns"], int(data.get("agent_rounds") or 0)) - except (TypeError, ValueError): - pass - try: - stats["tool_calls"] = max(stats["tool_calls"], int(data.get("tool_calls") or 0)) - except (TypeError, ValueError): - pass - except SkillAuditUnavailable: - raise - except Exception as e: - raise SkillAuditUnavailable(str(e)) from e - return "".join(transcript), stats, approval_required + return await _agent_runner.run_audit_arm(messages, url, model, headers, owner, workload) async def _run_skill_test_once(md: str, task: str, url, model, headers, owner, workload: str = "foreground") -> tuple: - """Run the skill once in the agent loop; return (transcript, verdict).""" - messages = _skill_test_messages(md, task) - text, stats, approval_required = await _run_skill_audit_arm( - messages, url, model, headers, owner, workload=workload, + return await _agent_runner.run_test_once( + md, task, url, model, headers, owner, workload, deps=_audit_dependencies(), ) - if approval_required is not None: - # Unattended audits have no authority to approve and no UI that could - # resume this record. Destructively deny it now instead of leaving a - # reusable opaque grant pending until TTL/cap eviction. - try: - from src.tool_approvals import tool_approval_store - tool_approval_store.consume( - approval_required.get("approval_id"), - decision="deny", - owner=owner, - session_id=None, - ) - except Exception: - logger.debug("Could not retire unattended skill approval", exc_info=True) - return text, { - "verdict": "inconclusive", - "confidence": 1.0, - "summary": ( - "This automated audit reached an exact action that requires " - "a human approval; no action was executed." - ), - "issues": [ - "Run this skill's manual test and review the sealed action." - ], - "approval_required": True, - } - verdict = await _eval_skill_run( - md, - task, - text, - url, - model, - headers, - skill_stats=stats, - workload=workload, - owner=owner, - ) - return text, verdict async def _improve_skill_md(skill_md: str, verdict: dict, transcript: str, url, model, headers, workload: str = "foreground", owner: Optional[str] = None): - """Have a model rewrite SKILL.md to fix the reviewer's issues. Returns the - corrected markdown, or None if it couldn't produce a usable change.""" - import re as _re - from src.llm_core import llm_call_async - issues = "\n".join("- " + str(i) for i in (verdict.get("issues") or [])) - sys_prompt = ( - "You are improving a reusable AI SKILL written in Markdown (frontmatter + body). " - "A QA reviewer found problems after a test run. Rewrite the SKILL.md to fix them: " - "make vague steps concrete, correct or remove references to tools that don't exist, " - "ensure the procedure is reproducible. " - "Keep the `name` field EXACTLY as-is (it is the skill's identity / filename). You MAY " - "correct the OTHER frontmatter — tags, category, when_to_use, description — when the " - "reviewer flagged them (issues prefixed 'metadata:') or they don't match the body; keep " - "retrieval metadata narrow: remove broad tags that would over-select the skill, and make " - "`when_to_use` say when NOT to use the skill if adjacent tasks are easy to confuse. Keep " - "valid frontmatter structure. Do NOT invent capabilities the agent lacks. Reason in " - " first if needed, then output ONLY the full corrected SKILL.md (no " - "fences, no commentary)." + return await _evaluation.improve_markdown( + skill_md, verdict, transcript, url, model, headers, workload, owner, + _effort_for_call=effort_for_call, ) - user_msg = ( - f"=== CURRENT SKILL.md ===\n{skill_md}\n\n" - f"=== REVIEWER VERDICT ===\n{verdict.get('summary', '')}\nIssues:\n{issues}\n\n" - f"=== TEST TRANSCRIPT ===\n{(transcript or '')[:6000]}" - ) - try: - raw = await llm_call_async(url, model, - [{"role": "system", "content": sys_prompt}, - {"role": "user", "content": user_msg}], - temperature=0.2, max_tokens=16384, headers=headers, timeout=180, - workload=workload, - reasoning_effort=effort_for_call(url, model, owner)) - except Exception as e: - logger.warning(f"Audit: improve call failed: {e}") - return None - text = strip_closed_think_blocks(raw or '') - text = _re.sub(r'[\s\S]*$', '', text, flags=_re.I) - text = _re.sub(r'', '', text, flags=_re.I).strip() - if text.startswith("```"): - text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip() - # Some reasoning models still prepend analysis or echo the old skill before - # the final corrected document. Keep only the last complete-looking - # frontmatter document so reviewer prose never gets persisted into SKILL.md. - starts = list(_re.finditer(r'(?m)^---\s*\n(?=[\s\S]*?^name\s*:)', text)) - if starts: - text = text[starts[-1].start():].strip() - return text or None async def _audit_one_skill(skills_manager, skill, url, model, headers, teacher, owner, log, workload: str = "foreground") -> dict: - """Test → judge → self-edit+retry → (teacher edit+retry) → flag. Never deletes; - a skill the teacher still can't fix is demoted to draft for manual review. - `teacher` is (url, model, headers) or None. `log(msg)` records progress.""" - name = skill.get("name") - - # Reflect the audit outcome in the skill's confidence so the main list - # updates: a clean pass earns high confidence; a pass that needed fixing - # earns a bit less; a skill that still fails is marked low. - def _set_conf(c): - try: - skills_manager.update_skill(name, {"confidence": c}, owner=owner) - except Exception: - pass - - md = skills_manager.read_skill_md(name, owner=owner) - if not md: - log(f"{name}: no source — skipped") - return {"skill": name, "result": "skipped"} - - # Cheap deterministic cleanup first. If this is an obvious lower-priority - # duplicate, do not spend LLM turns on necessity, retrieval precision, - # skill-vs-baseline testing, self-edit, or teacher escalation. - duplicate_of = _skill_duplicate_blocker(skills_manager, name, owner) - if duplicate_of: - reason = f"Lower-priority duplicate of {duplicate_of}" - try: - skills_manager.update_skill(name, {"status": "draft", "confidence": 0.35}, owner=owner) - skills_manager.set_audit(name, "skipped", by_teacher=False, worker_model=model, owner=owner) - skills_manager.set_necessity(name, False, [duplicate_of], reason, owner=owner) - except Exception: - pass - log(f"{name}: draft — skipped audit ({reason[:100]})") - return {"skill": name, "result": "skipped_duplicate", "reason": reason, "confidence": 0.35, "status": "draft"} - - # Advisory necessity/redundancy check — runs once, independent of the test - # outcome, and only records a flag the UI surfaces (never deletes/demotes). - others = [] - nec = None - try: - # Only compare against skills the SAME owner can see, so the necessity - # judge never sees (or flags "redundant_with") another user's skills. - sk_owner = skill.get("owner") - others = [ - {"name": s.get("name"), "description": s.get("description", "")} - for s in skills_manager.load(owner=owner) - if s.get("name") and s.get("name") != name - and (not sk_owner or not s.get("owner") or s.get("owner") == sk_owner) - ] - nec = await _eval_skill_necessity( - md, others, url, model, headers, workload=workload, - owner=owner, - ) - if nec is not None: - skills_manager.set_necessity(name, nec.get("necessary", True), - nec.get("redundant_with"), nec.get("reason"), - owner=owner) - if not nec.get("necessary", True): - log(f"{name}: possibly unnecessary — {nec.get('reason', '')[:80]}") - except Exception as e: - log(f"{name}: necessity check skipped — {e}") - - # Retrieval precision check: if broad tags/trigger text would make this - # narrow skill over-inject, fix only metadata before the functional test. - try: - if _should_check_retrieval_precision(skill): - rp = await _eval_skill_retrieval_precision( - md, others, url, model, headers, workload=workload, - owner=owner, - ) - if rp and not rp.get("ok"): - issues = rp.get("issues") or ["metadata: retrieval: narrow tags and when_to_use to the intended trigger"] - log(f"{name}: narrowing retrieval metadata — {(rp.get('summary') or issues[0])[:80]}") - fixed = await _improve_skill_md(md, { - "verdict": "pass", - "confidence": 1.0, - "summary": rp.get("summary") or "Retrieval metadata is too broad.", - "issues": issues, - }, "Retrieval audit only: the procedure may work, but matching metadata is too broad.", - url, model, headers, workload=workload, owner=owner) - if fixed and fixed.strip() != md.strip() and _apply_skill_md(skills_manager, name, fixed, owner): - md = fixed - refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) - if refreshed: - skill = refreshed - except Exception as e: - log(f"{name}: retrieval precision check skipped — {e}") - - task = _skill_test_task(skill) - log(f"{name}: testing…") - skill_messages = _skill_test_messages(md, task) - transcript, skill_stats, approval_required = await _run_skill_audit_arm( - skill_messages, - url, - model, - headers, - owner, - workload=workload, + return await _audit_workflow.audit_one_skill( + skills_manager, skill, url, model, headers, teacher, owner, log, workload, + deps=_audit_dependencies(), ) - if approval_required is not None: - try: - from src.tool_approvals import tool_approval_store - tool_approval_store.consume( - approval_required.get("approval_id"), - decision="deny", - owner=owner, - session_id=None, - ) - except Exception: - logger.debug("Could not retire unattended skill approval", exc_info=True) - verdict = { - "verdict": "inconclusive", - "confidence": 1.0, - "summary": ( - "This automated audit reached an exact action that requires " - "a human approval; no action was executed." - ), - "issues": [ - "Run this skill's manual test and review the sealed action." - ], - "approval_required": True, - } - else: - baseline_task = task - log(f"{name}: running no-skill baseline…") - baseline_transcript, baseline_stats, baseline_approval = await _run_skill_audit_arm( - _skill_baseline_messages(baseline_task), - url, - model, - headers, - owner, - workload=workload, - ) - if baseline_approval is not None: - try: - from src.tool_approvals import tool_approval_store - tool_approval_store.consume( - baseline_approval.get("approval_id"), - decision="deny", - owner=owner, - session_id=None, - ) - except Exception: - logger.debug("Could not retire unattended baseline approval", exc_info=True) - verdict = await _eval_skill_run( - md, - task, - transcript, - url, - model, - headers, - baseline_transcript=baseline_transcript, - skill_stats=skill_stats, - baseline_stats=baseline_stats, - workload=workload, - owner=owner, - ) - v = verdict.get("verdict") - log(f"{name}: verdict = {v} ({verdict.get('summary', '')[:80]})") - if verdict.get("approval_required"): - # An unattended audit is not authority for an action influenced by the - # skill under test. Preserve the skill's current publication/confidence - # state and route the exact action to the manual test UI instead of - # letting a safety pause demote, rewrite, or auto-publish the skill. - skills_manager.set_audit( - name, - "inconclusive", - by_teacher=False, - worker_model=model, - owner=owner, - audit_summary=verdict.get("summary") or "The test requires approval for an external action.", - ) - status = skill.get("status") or "draft" - log(f"{name}: {status} unchanged — exact action needs manual approval") - return { - "skill": name, - "result": "approval_required", - "verdict": verdict, - "status": status, - } - if v == "pass": - # Procedure works. If the reviewer still flagged metadata (tags/category/ - # when_to_use/description), do ONE fixer pass to correct the frontmatter - # without re-testing — a metadata-only fix can't break a passing run. - meta_issues = [i for i in (verdict.get("issues") or []) if str(i).lower().lstrip().startswith("metadata:")] - if meta_issues: - log(f"{name}: pass, but fixing {len(meta_issues)} metadata issue(s)…") - fixed = await _improve_skill_md( - md, verdict, transcript, url, model, headers, workload=workload, - owner=owner, - ) - if fixed and fixed.strip() != md.strip(): - _apply_skill_md(skills_manager, name, fixed, owner) - _set_conf(0.95) - skills_manager.set_audit( - name, - "pass", - by_teacher=False, - worker_model=model, - owner=owner, - **_verdict_efficiency(verdict), - ) - refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) - status = _audit_finalize_status(skills_manager, name, owner, "pass", 0.95, (refreshed or {}).get("necessity"), verdict) - log(f"{name}: {status} — confidence 95%") - return {"skill": name, "result": "pass", "verdict": verdict, "confidence": 0.95, "status": status} - if v in ("unknown", "inconclusive"): - skills_manager.set_audit( - name, - "inconclusive", - by_teacher=False, - worker_model=model, - owner=owner, - **_verdict_efficiency(verdict), - ) - status = _audit_finalize_status(skills_manager, name, owner, "inconclusive", skill.get("confidence") or 0.0, skill.get("necessity")) - log(f"{name}: {status} — inconclusive") - return {"skill": name, "result": "inconclusive", "verdict": verdict, "status": status} - - # Self-edit + retry. - log(f"{name}: self-editing to fix issues…") - new_md = await _improve_skill_md( - md, verdict, transcript, url, model, headers, workload=workload, - owner=owner, - ) - if new_md and new_md.strip() != md.strip() and _apply_skill_md(skills_manager, name, new_md, owner): - md = new_md - transcript, verdict = await _run_skill_test_once( - md, task, url, model, headers, owner, workload=workload, - ) - v = verdict.get("verdict") - log(f"{name}: retry (self) = {v}") - if v == "pass": - _set_conf(0.85) - skills_manager.set_audit( - name, - "pass", - by_teacher=False, - worker_model=model, - owner=owner, - **_verdict_efficiency(verdict), - ) - refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) - status = _audit_finalize_status(skills_manager, name, owner, "pass", 0.85, (refreshed or {}).get("necessity"), verdict) - log(f"{name}: {status} — confidence 85% after self-edit") - return {"skill": name, "result": "pass_after_self_edit", "verdict": verdict, "confidence": 0.85, "status": status} - - # Teacher escalation (if a distinct teacher model is configured). The - # teacher only REWRITES the skill — it does NOT run the test. The point is - # to verify the regular (student) model can now succeed with the teacher's - # improved procedure, so the retry runs on the worker model, not the teacher. - teacher_ran = False - if teacher and teacher[0] and teacher[1] and (teacher[1] != model or teacher[0] != url): - teacher_ran = True - t_url, t_model, t_headers = teacher - log(f"{name}: teacher {t_model} rewriting the skill…") - t_md = await _improve_skill_md( - md, verdict, transcript, t_url, t_model, t_headers, workload=workload, - owner=owner, - ) - if t_md and t_md.strip() != md.strip() and _apply_skill_md(skills_manager, name, t_md, owner): - md = t_md - # Re-test with the STUDENT model (the model the skill runs under in use). - transcript, verdict = await _run_skill_test_once( - md, task, url, model, headers, owner, workload=workload, - ) - v = verdict.get("verdict") - log(f"{name}: retry on student after teacher rewrite = {v}") - if v == "pass": - _set_conf(0.8) - skills_manager.set_audit( - name, - "pass", - by_teacher=True, - worker_model=model, - teacher_model=t_model, - owner=owner, - **_verdict_efficiency(verdict), - ) - refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) - status = _audit_finalize_status(skills_manager, name, owner, "pass", 0.8, (refreshed or {}).get("necessity"), verdict) - log(f"{name}: {status} — confidence 80% after teacher rewrite") - return {"skill": name, "result": "pass_after_teacher", "verdict": verdict, "confidence": 0.8, "status": status} - - # Still failing → demote to draft + low confidence + flag (do NOT delete). - try: - skills_manager.update_skill(name, {"status": "draft", "confidence": 0.35}, owner=owner) - except Exception: - pass - skills_manager.set_audit( - name, v or "fail", by_teacher=teacher_ran, - worker_model=model, - teacher_model=(teacher[1] if teacher_ran and teacher else ""), - owner=owner, - **_verdict_efficiency(verdict), - ) - log(f"{name}: flagged — confidence lowered, kept as draft for manual review") - return {"skill": name, "result": "flagged", "verdict": verdict, "confidence": 0.35} async def _run_audit_all_job(key, skills_manager, names, url, model, headers, teacher, owner, workload: str = "foreground"): - """Background: audit each named skill in sequence, recording progress.""" - import asyncio as _asyncio - import time as _time - - job = _skill_audit_jobs.get(key) - if job is None: - return - - def log(msg): - job["log"].append(msg) - if len(job["log"]) > 1000: - del job["log"][0:len(job["log"]) - 1000] - - cancelled = False - try: - for nm in names: - if job.get("cancel"): - cancelled = True - log("(cancelled)") - break - job["current"] = nm - skills = skills_manager.load(owner=owner) - sk = next((s for s in skills if s.get("name") == nm), None) - if not sk: - continue - try: - res = await _audit_one_skill( - skills_manager, sk, url, model, headers, teacher, owner, log, - workload=workload, - ) - except _asyncio.CancelledError: - cancelled = True - job["cancel"] = True - log("(cancelled)") - raise - except SkillAuditUnavailable as e: - job["unavailable"] = str(e) - log(f"Audit paused: {e}. Skill verdicts unchanged; retry when the model is available.") - break - except Exception as e: - log(f"{nm}: error — {e}") - res = {"skill": nm, "result": "error"} - skills_manager.set_audit(nm, "inconclusive", worker_model=model, owner=owner) - try: - refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == nm), None) - if refreshed: - res["skill_state"] = { - "name": refreshed.get("name"), - "status": refreshed.get("status"), - "confidence": refreshed.get("confidence"), - "audit_verdict": refreshed.get("audit_verdict"), - "audit_by_teacher": refreshed.get("audit_by_teacher"), - "audit_worker_model": refreshed.get("audit_worker_model"), - "audit_teacher_model": refreshed.get("audit_teacher_model"), - "audited_at": refreshed.get("audited_at"), - "saved_turns": refreshed.get("saved_turns"), - "saved_tool_calls": refreshed.get("saved_tool_calls"), - "baseline_verdict": refreshed.get("baseline_verdict"), - "usefulness": refreshed.get("usefulness"), - "necessity": refreshed.get("necessity"), - } - except Exception: - pass - job["results"].append(res) - job["done"] = len(job["results"]) - except _asyncio.CancelledError: - cancelled = True - finally: - if not cancelled and not job.get("cancel") and not job.get("unavailable"): - try: - _finalize_audit_batch(skills_manager, job.get("results") or [], owner, log) - except Exception: - logger.warning("Could not finalize skills audit batch", exc_info=True) - job["current"] = None - job["status"] = "cancelled" if cancelled or job.get("cancel") else "error" if job.get("unavailable") else "done" - job["finished"] = _time.time() - job.pop("task", None) + return await _audit_workflow.run_audit_batch( + key, skills_manager, names, url, model, headers, teacher, owner, workload, + deps=_audit_dependencies(), + ) def _resolve_audit_models(owner=None, model_spec=None, endpoint_url=None): - """Resolve (url, model, headers, teacher) for an audit run from Settings. - - Worker = Utility model (falling back to Default, normalized to a served - model id); teacher = the optional Settings → Teacher Model config. Shared - by the manual /audit-all route and scheduled/event audits. Raises - ValueError if no worker model. - """ - from src.endpoint_resolver import resolve_endpoint, resolve_utility_fallback_candidates - if model_spec and endpoint_url: - # Scheduled tasks store the endpoint URL and model separately. Resolve - # the endpoint directly so an explicit task choice cannot be replaced - # by the global Utility setting. - from src.endpoint_resolver import build_headers, resolve_endpoint_runtime - from src.database import ModelEndpoint, SessionLocal - from src.endpoint_resolver import normalize_base, same_endpoint_base - from src.auth_helpers import owner_filter - url = endpoint_url - model = model_spec - headers = {} - db = SessionLocal() - try: - query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) - for ep in owner_filter(query, ModelEndpoint, owner).all(): - base = normalize_base(getattr(ep, "base_url", "") or "") - if same_endpoint_base(url, base): - runtime_base, api_key = resolve_endpoint_runtime(ep, owner=owner) - headers = build_headers(api_key, runtime_base or base) - break - finally: - db.close() - elif model_spec: - from src.ai_interaction import _resolve_model - url, model, headers = _resolve_model(str(model_spec), owner=owner) - else: - url, model, headers = resolve_endpoint("utility", owner=owner) - if not url or not model: - # Utility fallbacks are an explicit part of the user's model - # configuration. Audits must use the same chain as other background - # work instead of treating an empty primary Utility slot as fatal. - for fallback_url, fallback_model, fallback_headers in resolve_utility_fallback_candidates(owner=owner): - url, model, headers = fallback_url, fallback_model, fallback_headers - break - if not url or not model: - raise ValueError("No model configured — set a Default or Utility model in Settings.") - try: - from src.llm_core import list_model_ids - import os as _os - _avail = list_model_ids(url, headers=headers) - if _avail and model not in _avail: - _base = _os.path.basename((model or "").rstrip("/")) - model = next((a for a in _avail if _os.path.basename(a.rstrip("/")) == _base), None) or _avail[0] - except Exception: - pass - - teacher = None - try: - from src.settings import get_setting - if get_setting("teacher_enabled", False): - spec = (get_setting("teacher_model", "") or "").strip() - if spec: - from src.ai_interaction import _resolve_model - t_url, t_model, t_headers = _resolve_model(spec, owner=owner) - if t_url and t_model: - teacher = (t_url, t_model, t_headers) - except Exception as e: - logger.warning(f"Audit teacher resolve failed: {e}") - return url, model, headers, teacher + return _model_selection.resolve_audit_models(owner, model_spec, endpoint_url) async def run_scheduled_skill_audit(skills_manager: SkillsManager, owner: Optional[str] = None, max_skills: int = 8) -> dict: - """Nightly audit pass. Audits the LEAST-recently-audited skills first and - caps the batch so it rotates through the library over successive nights - instead of re-checking the same ones every run. Reuses the same job store - the manual 'Audit all' button uses, so its progress shows in the UI too.""" - import time as _time + return await _audit_workflow.run_scheduled_skill_audit( + skills_manager, owner, max_skills, deps=_audit_dependencies(), + ) - key = (owner or "",) - existing = _skill_audit_jobs.get(key) - if existing and existing.get("status") == "running": - logger.info("Scheduled skill audit skipped — a run is already active.") - return {"status": "running", "skipped": True} - try: - url, model, headers, teacher = _resolve_audit_models(owner=owner) - except ValueError as e: - logger.info(f"Scheduled skill audit skipped — {e}") - return {"status": "skipped", "reason": str(e)} - - from services.memory.skill_lifecycle import automatic_audit_candidates - skills = automatic_audit_candidates(skills_manager.load(owner=owner), limit=max_skills) - names = [s["name"] for s in skills] - if not names: - return {"status": "done", "total": 0} - - _skill_audit_jobs[key] = { - "status": "running", "scope": "scheduled", "model": model, - "teacher": teacher[1] if teacher else None, - "total": len(names), "done": 0, "current": None, - "results": [], "log": [f"Nightly audit of {len(names)} least-recently-checked skill(s) with {model}" - + (f"; teacher {teacher[1]}" if teacher else "")], - "started": _time.time(), "cancel": False, - } - logger.info(f"Scheduled skill audit starting: {len(names)} skill(s) (owner={owner or 'all'})") - await _run_audit_all_job( - key, skills_manager, names, url, model, headers, teacher, owner, - workload="background", +def _audit_dependencies() -> AuditDependencies: + """Bind current compatibility symbols without shared mutable rebinding.""" + return AuditDependencies( + test_jobs=_skill_test_jobs, + audit_jobs=_skill_audit_jobs, + test_task=_skill_test_task, + test_messages=_skill_test_messages, + baseline_messages=_skill_baseline_messages, + evaluate_run=_eval_skill_run, + evaluate_necessity=_eval_skill_necessity, + evaluate_retrieval_precision=_eval_skill_retrieval_precision, + should_check_retrieval_precision=_should_check_retrieval_precision, + improve_markdown=_improve_skill_md, + apply_markdown=_apply_skill_md, + duplicate_blocker=_skill_duplicate_blocker, + finalize_status=_audit_finalize_status, + finalize_batch=_finalize_audit_batch, + auto_publish_policy=_audit_auto_publish_policy, + utility_blocker=_audit_utility_blocker, + verdict_efficiency=_verdict_efficiency, + run_test_job=_run_skill_test_job, + run_arm=_run_skill_audit_arm, + run_test_once=_run_skill_test_once, + audit_one=_audit_one_skill, + run_batch=_run_audit_all_job, + resolve_models=_resolve_audit_models, ) - job = _skill_audit_jobs.get(key, {}) - return {"status": "done", "total": len(names), "results": job.get("results", [])} def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter: @@ -2022,8 +595,6 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter: On completion it records the verdict and nudges the skill's confidence to match (pass→0.95, needs_work→0.6, fail→0.4; inconclusive/unknown leave it untouched). It never changes the skill's published/draft STATUS.""" - import time as _time - import asyncio as _asyncio from core.database import SessionLocal from src.endpoint_resolver import ( build_chat_url, build_headers, resolve_endpoint, @@ -2079,46 +650,14 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter: except Exception as _e: logger.warning(f"Skill-test model resolve failed: {_e}") - key = (user or "", name) - previous_job = _skill_test_jobs.get(key) or {} - previous_approval = previous_job.get("approval") or {} - if previous_approval.get("approval_id"): - try: - from src.tool_approvals import tool_approval_store - tool_approval_store.consume( - previous_approval["approval_id"], - decision="deny", - owner=user, - session_id=None, - ) - except Exception: - logger.debug("Could not retire replaced skill approval", exc_info=True) - _skill_test_jobs[key] = { - "status": "running", - "task": task, - "model": model, - "skill": name, - "started": _time.time(), - "log": [{"type": "skill_test_start", "task": task, "skill": name, "model": model}], - "verdict": None, - "_run": { - "md": md, - "url": url, - "model": model, - "headers": headers, - "owner": user, - "request_authority": request_authority, - }, - } - _asyncio.create_task(_run_skill_test_job( - key, name, md, task, url, model, headers, user, skills_manager, - request_authority=request_authority)) - return {"ok": True, "status": "running", "skill": name, "model": model} + return _job_coordinator.start_test_job( + name, md, task, url, model, headers, user, skills_manager, + request_authority, deps=_audit_dependencies(), + ) @router.post("/{skill_id}/test-approval") async def approve_skill_test_action(request: Request, skill_id: str): """Resume a manual skill test with one exact server-sealed action.""" - import asyncio as _asyncio from src.tool_approvals import tool_approval_store from src.agent_runtime.authority import require_user_approval_request require_user_approval_request(request) @@ -2169,74 +708,10 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter: if decision == "approve" and exact_approval is None: raise HTTPException(409, "This tool approval could not be consumed.") - job.pop("approval", None) - if decision == "deny": - job.pop("_transcript", None) - job.pop("_run", None) - job["log"].append({ - "type": "approval_denied", - "text": "Exact action denied; the skill test stopped without executing it.", - }) - job["verdict"] = { - "verdict": "inconclusive", - "confidence": 1.0, - "summary": "The test stopped because its exact action was denied.", - "issues": [], - } - job["status"] = "done" - return {"ok": True, "status": "done", "decision": "deny"} - - run = job.get("_run") or {} - transcript = job.pop("_transcript", []) - # stream_agent_loop owns its per-round message list internally. Rebuild - # continuation context from the original untrusted skill plus the - # accumulated transcript so repeated approvals do not lose earlier - # approved results, while keeping every transcript byte tainted. - messages = _skill_test_messages( - run.get("md", ""), - job.get("task", ""), + return _job_coordinator.continue_test_job( + job, key, name, decision, expected, exact_approval, + skills_manager, deps=_audit_dependencies(), ) - if transcript: - messages.append(untrusted_context_message( - "skill test transcript", - "".join(str(item) for item in transcript), - )) - messages.extend([ - { - "role": "assistant", - "content": str(expected.get("question") or "Allow this exact action once?"), - }, - { - "role": "user", - "content": ( - f"Approved the exact {exact_approval.pending.tool_name} " - "action shown above once." - ), - }, - ]) - job["status"] = "running" - job["log"].append({ - "type": "approval_granted", - "text": ( - f"Approved exact {exact_approval.pending.tool_name} action once; " - "resuming test." - ), - }) - _asyncio.create_task(_run_skill_test_job( - key, - name, - run.get("md", ""), - job.get("task", ""), - run.get("url"), - run.get("model"), - run.get("headers"), - run.get("owner"), - skills_manager, - messages=messages, - transcript=transcript, - exact_approval=exact_approval, - )) - return {"ok": True, "status": "running", "decision": "approve"} @router.get("/{skill_id}/test-status") async def test_skill_status(request: Request, skill_id: str): @@ -2245,17 +720,7 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter: skills = skills_manager.load(owner=user) match = next((s for s in skills if s.get("name") == skill_id or s.get("id") == skill_id), None) name = (match or {}).get("name", skill_id) - job = _skill_test_jobs.get((user or "", name)) - if not job: - return {"status": "none"} - return { - "status": job["status"], - "task": job.get("task"), - "model": job.get("model"), - "log": job.get("log", []), - "verdict": job.get("verdict"), - "approval": job.get("approval"), - } + return _job_coordinator.test_status(user, name, deps=_audit_dependencies()) @router.post("/audit-all") async def audit_all_skills(request: Request): @@ -2266,8 +731,6 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter: {scope: 'drafts'|'unchecked'|'all', names?: [...], skip_audited?: bool}. Default 'all' means every visible skill, including already-published skills, so audit can publish or demote according to the confidence threshold.""" - import asyncio as _asyncio - import time as _time user = _owner(request) body = await request.json() if request.headers.get("content-type", "").startswith("application/json") else {} @@ -2290,78 +753,23 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter: except ValueError as e: raise HTTPException(400, str(e)) - skills = skills_manager.load(owner=user) - # Built-ins are tracked, pre-approved application procedures. They do - # not consume audit turns and cannot be demoted by an audit result. - auditable_skills = [s for s in skills if s.get("source") != "builtin"] - by_name = {s.get("name"): s for s in auditable_skills if s.get("name")} - if isinstance(requested_names, list): - names = [] - seen = set() - for raw in requested_names: - nm = str(raw or "").strip() - if not nm or nm in seen or nm not in by_name: - continue - if scope not in ("all", "selected") and (by_name[nm].get("status") or "draft") == "published": - continue - if skip_audited and by_name[nm].get("audit_verdict"): - continue - names.append(nm) - seen.add(nm) - scope = "selected" if requested_names else scope - elif scope == "all": - names = [ - s.get("name") for s in auditable_skills - if s.get("name") and (not skip_audited or not s.get("audit_verdict")) - ] - else: - scope = "unchecked" if scope == "drafts" else scope - names = [ - s.get("name") for s in auditable_skills - if s.get("name") - and (s.get("status") or "draft") != "published" - and not s.get("audit_verdict") - ] - if not names: - return {"ok": True, "status": "done", "total": 0, "results": [], "log": ["No skills to audit."]} - - _skill_audit_jobs[key] = { - "status": "running", "scope": scope, "model": model, - "teacher": teacher[1] if teacher else None, - "total": len(names), "done": 0, "current": None, - "results": [], "log": [f"Auditing {len(names)} skill(s) with {model}" + (f"; teacher {teacher[1]}" if teacher else "")], - "started": _time.time(), "cancel": False, - } - task = _asyncio.create_task(_run_audit_all_job(key, skills_manager, names, url, model, headers, teacher, user)) - _skill_audit_jobs[key]["task"] = task - return {"ok": True, "status": "running", "total": len(names), "model": model} + names, scope = _job_coordinator.select_audit_names( + skills_manager, user, scope, requested_names, skip_audited, + ) + return _job_coordinator.start_audit_job( + key, skills_manager, names, url, model, headers, teacher, user, + scope, deps=_audit_dependencies(), + ) @router.get("/audit-all/status") async def audit_status(request: Request): user = _owner(request) - job = _skill_audit_jobs.get((user or "",)) - if not job: - return {"status": "none"} - return { - "status": job["status"], "scope": job.get("scope"), - "total": job.get("total", 0), "done": job.get("done", 0), - "current": job.get("current"), "model": job.get("model"), "teacher": job.get("teacher"), - "results": job.get("results", []), "log": job.get("log", []), - "started": job.get("started"), "finished": job.get("finished"), - } + return _job_coordinator.audit_status(user, deps=_audit_dependencies()) @router.post("/audit-all/cancel") async def audit_cancel(request: Request): user = _owner(request) - job = _skill_audit_jobs.get((user or "",)) - if job: - job["cancel"] = True - job["status"] = "cancelled" - job["current"] = None - task = job.get("task") - if task and not task.done(): - task.cancel() - return {"ok": True, "status": "cancelled" if job else "none"} + return _job_coordinator.cancel_audit(user, deps=_audit_dependencies()) @router.post("/{skill_id}/markdown") async def save_skill_markdown(request: Request, skill_id: str): diff --git a/services/skills/__init__.py b/services/skills/__init__.py new file mode 100644 index 000000000..1b6236117 --- /dev/null +++ b/services/skills/__init__.py @@ -0,0 +1 @@ +"""Skills domain operations; persistence and lifecycle remain in SkillsManager.""" diff --git a/services/skills/agent_runner.py b/services/skills/agent_runner.py new file mode 100644 index 000000000..bf81860f1 --- /dev/null +++ b/services/skills/agent_runner.py @@ -0,0 +1,308 @@ +"""Agent execution, transcripts and same-task stream cleanup.""" + +import logging +from contextlib import aclosing +from typing import Optional + +from .dependencies import AuditDependencies + +logger = logging.getLogger(__name__) + +_skill_test_jobs: dict = {} + + +class SkillAuditUnavailable(RuntimeError): + """The test infrastructure failed; this is not evidence about a skill.""" + + +async def run_test_job( + key, + name, + md, + task, + url, + model, + headers, + owner, + skills_manager=None, + *, + messages=None, + transcript=None, + exact_approval=None, + request_authority=None, + deps: AuditDependencies, +): + """Background coroutine: run the skill in an agent loop, capture a condensed + log + transcript, then have the judge grade it. Writes into deps.test_jobs.""" + import json as _json + from src.agent_loop import stream_agent_loop + from src.agent_runtime.authority import RequestAuthority + + job = deps.test_jobs.get(key) + if job is None: + return + log = job["log"] + transcript = transcript if isinstance(transcript, list) else [] + say_buf = [] + skill_stats = {"turns": 0, "tool_calls": 0} + + def _flush_say(): + if say_buf: + log.append({"type": "say", "text": "".join(say_buf)}) + say_buf.clear() + + messages = list(messages) if isinstance(messages, list) else deps.test_messages(md, task) + try: + # aclosing: pausing for an approval leaves this loop early, and the + # stream's ContextVar cleanup must run in this task, not in asyncio's + # finalizer (#6635). + async with aclosing(stream_agent_loop( + url, model, messages, headers=headers, + temperature=0.3, max_tokens=0, max_rounds=8, owner=owner, + exact_approval=exact_approval, + request_authority=(request_authority or ( + exact_approval.pending.request_authority if exact_approval is not None else None + ) or RequestAuthority.empty(owner=owner)), + )) as stream: + async for chunk in stream: + if not chunk.startswith("data: ") or chunk.strip() == "data: [DONE]": + continue + try: + d = _json.loads(chunk[6:]) + except Exception: + continue + if d.get("delta"): + say_buf.append(d["delta"]); transcript.append(d["delta"]) + elif d.get("type") == "tool_start": + _flush_say() + skill_stats["tool_calls"] += 1 + cmd = str(d.get("command") or d.get("args") or "")[:300] + log.append({"type": "tool_start", "tool": d.get("tool"), "command": cmd}) + transcript.append(f"\n[tool {d.get('tool')}] {cmd}\n") + elif d.get("type") == "tool_output": + _flush_say() + out = str(d.get("output") or "")[:600] + tool_log = {"type": "tool_output", "output": out} + approval = d.get("ask_user") + if isinstance(approval, dict): + tool_log["ask_user"] = approval + log.append(tool_log) + transcript.append(f"[output] {out}\n") + if ( + isinstance(approval, dict) + and approval.get("kind") == "tool_approval" + and approval.get("approval_id") + ): + # Manual skill tests have their own polling UI instead of a + # chat session. Pause the run and retain only server-side + # continuation state until the same owner approves/denies + # this exact sealed action. + job["status"] = "awaiting_approval" + job["approval"] = approval + job["_transcript"] = transcript + return + elif d.get("type") == "agent_step": + _flush_say() + try: + skill_stats["turns"] = max(skill_stats["turns"], int(d.get("round") or 0)) + except (TypeError, ValueError): + pass + log.append({"type": "agent_step", "round": d.get("round")}) + transcript.append(f"\n--- round {d.get('round')} ---\n") + elif d.get("type") == "metrics": + data = d.get("data") or {} + try: + skill_stats["turns"] = max(skill_stats["turns"], int(data.get("agent_rounds") or 0)) + except (TypeError, ValueError): + pass + try: + skill_stats["tool_calls"] = max(skill_stats["tool_calls"], int(data.get("tool_calls") or 0)) + except (TypeError, ValueError): + pass + if len(log) > 600: + del log[0:len(log) - 600] + _flush_say() + except Exception as e: + _flush_say() + log.append({"type": "error", "error": str(e)}) + + job.pop("approval", None) + job.pop("_transcript", None) + job.pop("_run", None) + log.append({"type": "evaluating"}) + try: + baseline_task = task + log.append({"type": "agent_step", "round": "baseline"}) + baseline_transcript, baseline_stats, baseline_approval = await deps.run_arm( + deps.baseline_messages(baseline_task), + url, + model, + headers, + owner, + ) + if baseline_approval is not None: + try: + from src.tool_approvals import tool_approval_store + tool_approval_store.consume( + baseline_approval.get("approval_id"), + decision="deny", + owner=owner, + session_id=None, + ) + except Exception: + logger.debug("Could not retire manual-test baseline approval", exc_info=True) + job["verdict"] = await deps.evaluate_run( + md, + task, + "".join(transcript), + url, + model, + headers, + baseline_transcript=baseline_transcript, + skill_stats=skill_stats, + baseline_stats=baseline_stats, + owner=owner, + ) + except Exception as e: + job["verdict"] = {"verdict": "unknown", "confidence": 0, "summary": f"Eval failed: {e}", "issues": []} + # Record the result so the card shows a 'verified' check (a manual test + # never involves the teacher) and nudge the confidence score to match the + # verdict — same scale as Audit-all's pass=0.95. inconclusive/unknown leave + # the score alone (missing-fixture or parse failures shouldn't punish it). + if skills_manager is not None: + v = (job["verdict"] or {}).get("verdict") or "unknown" + try: + skills_manager.set_audit( + name, + v, + by_teacher=False, + worker_model=model, + owner=owner, + **deps.verdict_efficiency(job.get("verdict")), + ) + except Exception: + pass + conf = {"pass": 0.95, "needs_work": 0.6, "fail": 0.4}.get(v) + if conf is not None: + try: + skills_manager.update_skill(name, {"confidence": conf}, owner=owner) + except Exception: + pass + job["status"] = "done" + + +async def run_audit_arm(messages: list[dict], url, model, headers, owner, + workload: str = "foreground") -> tuple[str, dict, Optional[dict]]: + """Run one audit arm in the agent loop; return transcript, stats, approval.""" + import json as _json + from src.agent_loop import stream_agent_loop + from src.agent_runtime.authority import RequestAuthority, active_request_authority + transcript = [] + approval_required = None + stats = {"turns": 0, "tool_calls": 0} + try: + # max_tokens explicitly set: passing 0 lets some upstreams (Ollama, + # OpenAI-compat) generate an empty completion, which manifested as + # the skill test returning nothing while chat (which carries its + # preset's max_tokens) worked. 4096 matches the chat default. + # aclosing: stopping for an approval leaves this loop early (#6635). + async with aclosing(stream_agent_loop( + url, model, messages, headers=headers, + temperature=0.3, max_tokens=4096, max_rounds=8, + owner=owner, workload=workload, suppress_skills=True, + request_authority=(active_request_authority() or RequestAuthority.empty(owner=owner)), + )) as stream: + async for chunk in stream: + # Streams can include an SSE event line before the data line, + # notably `event: error`. Do not silently discard those failures. + payload = next((line[6:] for line in chunk.splitlines() if line.startswith("data: ")), None) + if payload is None or payload == "[DONE]": + continue + try: + d = _json.loads(payload) + except Exception: + continue + if d.get("error") or d.get("type") == "error": + raise SkillAuditUnavailable(str(d.get("error") or d.get("message") or "Audit stream failed")) + if d.get("delta"): + transcript.append(d["delta"]) + elif d.get("type") == "tool_start": + stats["tool_calls"] += 1 + transcript.append(f"\n[tool {d.get('tool')}] {str(d.get('command') or d.get('args') or '')[:300]}\n") + elif d.get("type") == "tool_output": + transcript.append(f"[output] {str(d.get('output') or '')[:600]}\n") + approval = d.get("ask_user") + if ( + isinstance(approval, dict) + and approval.get("kind") == "tool_approval" + ): + approval_required = approval + break + elif d.get("type") == "agent_step": + try: + stats["turns"] = max(stats["turns"], int(d.get("round") or 0)) + except (TypeError, ValueError): + pass + transcript.append(f"\n--- round {d.get('round')} ---\n") + elif d.get("type") == "metrics": + data = d.get("data") or {} + try: + stats["turns"] = max(stats["turns"], int(data.get("agent_rounds") or 0)) + except (TypeError, ValueError): + pass + try: + stats["tool_calls"] = max(stats["tool_calls"], int(data.get("tool_calls") or 0)) + except (TypeError, ValueError): + pass + except SkillAuditUnavailable: + raise + except Exception as e: + raise SkillAuditUnavailable(str(e)) from e + return "".join(transcript), stats, approval_required + + +async def run_test_once(md: str, task: str, url, model, headers, owner, + workload: str = "foreground", *, deps: AuditDependencies) -> tuple: + """Run the skill once in the agent loop; return (transcript, verdict).""" + messages = deps.test_messages(md, task) + text, stats, approval_required = await deps.run_arm( + messages, url, model, headers, owner, workload=workload, + ) + if approval_required is not None: + # Unattended audits have no authority to approve and no UI that could + # resume this record. Destructively deny it now instead of leaving a + # reusable opaque grant pending until TTL/cap eviction. + try: + from src.tool_approvals import tool_approval_store + tool_approval_store.consume( + approval_required.get("approval_id"), + decision="deny", + owner=owner, + session_id=None, + ) + except Exception: + logger.debug("Could not retire unattended skill approval", exc_info=True) + return text, { + "verdict": "inconclusive", + "confidence": 1.0, + "summary": ( + "This automated audit reached an exact action that requires " + "a human approval; no action was executed." + ), + "issues": [ + "Run this skill's manual test and review the sealed action." + ], + "approval_required": True, + } + verdict = await deps.evaluate_run( + md, + task, + text, + url, + model, + headers, + skill_stats=stats, + workload=workload, + owner=owner, + ) + return text, verdict diff --git a/services/skills/audit_workflow.py b/services/skills/audit_workflow.py new file mode 100644 index 000000000..f7af6682f --- /dev/null +++ b/services/skills/audit_workflow.py @@ -0,0 +1,439 @@ +"""Sequential skill audit coordination and transient owner-scoped jobs.""" + +import logging +from typing import Optional + +from services.memory.skills import SkillsManager +from .dependencies import AuditDependencies + +from .agent_runner import SkillAuditUnavailable + +logger = logging.getLogger(__name__) + +_skill_audit_jobs: dict = {} + + +async def audit_one_skill(skills_manager, skill, url, model, headers, + teacher, owner, log, workload: str = "foreground", *, deps: AuditDependencies) -> dict: + """Test → judge → self-edit+retry → (teacher edit+retry) → flag. Never deletes; + a skill the teacher still can't fix is demoted to draft for manual review. + `teacher` is (url, model, headers) or None. `log(msg)` records progress.""" + name = skill.get("name") + + # Reflect the audit outcome in the skill's confidence so the main list + # updates: a clean pass earns high confidence; a pass that needed fixing + # earns a bit less; a skill that still fails is marked low. + def _set_conf(c): + try: + skills_manager.update_skill(name, {"confidence": c}, owner=owner) + except Exception: + pass + + md = skills_manager.read_skill_md(name, owner=owner) + if not md: + log(f"{name}: no source — skipped") + return {"skill": name, "result": "skipped"} + + # Cheap deterministic cleanup first. If this is an obvious lower-priority + # duplicate, do not spend LLM turns on necessity, retrieval precision, + # skill-vs-baseline testing, self-edit, or teacher escalation. + duplicate_of = deps.duplicate_blocker(skills_manager, name, owner) + if duplicate_of: + reason = f"Lower-priority duplicate of {duplicate_of}" + try: + skills_manager.update_skill(name, {"status": "draft", "confidence": 0.35}, owner=owner) + skills_manager.set_audit(name, "skipped", by_teacher=False, worker_model=model, owner=owner) + skills_manager.set_necessity(name, False, [duplicate_of], reason, owner=owner) + except Exception: + pass + log(f"{name}: draft — skipped audit ({reason[:100]})") + return {"skill": name, "result": "skipped_duplicate", "reason": reason, "confidence": 0.35, "status": "draft"} + + # Advisory necessity/redundancy check — runs once, independent of the test + # outcome, and only records a flag the UI surfaces (never deletes/demotes). + others = [] + nec = None + try: + # Only compare against skills the SAME owner can see, so the necessity + # judge never sees (or flags "redundant_with") another user's skills. + sk_owner = skill.get("owner") + others = [ + {"name": s.get("name"), "description": s.get("description", "")} + for s in skills_manager.load(owner=owner) + if s.get("name") and s.get("name") != name + and (not sk_owner or not s.get("owner") or s.get("owner") == sk_owner) + ] + nec = await deps.evaluate_necessity( + md, others, url, model, headers, workload=workload, + owner=owner, + ) + if nec is not None: + skills_manager.set_necessity(name, nec.get("necessary", True), + nec.get("redundant_with"), nec.get("reason"), + owner=owner) + if not nec.get("necessary", True): + log(f"{name}: possibly unnecessary — {nec.get('reason', '')[:80]}") + except Exception as e: + log(f"{name}: necessity check skipped — {e}") + + # Retrieval precision check: if broad tags/trigger text would make this + # narrow skill over-inject, fix only metadata before the functional test. + try: + if deps.should_check_retrieval_precision(skill): + rp = await deps.evaluate_retrieval_precision( + md, others, url, model, headers, workload=workload, + owner=owner, + ) + if rp and not rp.get("ok"): + issues = rp.get("issues") or ["metadata: retrieval: narrow tags and when_to_use to the intended trigger"] + log(f"{name}: narrowing retrieval metadata — {(rp.get('summary') or issues[0])[:80]}") + fixed = await deps.improve_markdown(md, { + "verdict": "pass", + "confidence": 1.0, + "summary": rp.get("summary") or "Retrieval metadata is too broad.", + "issues": issues, + }, "Retrieval audit only: the procedure may work, but matching metadata is too broad.", + url, model, headers, workload=workload, owner=owner) + if fixed and fixed.strip() != md.strip() and deps.apply_markdown(skills_manager, name, fixed, owner): + md = fixed + refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) + if refreshed: + skill = refreshed + except Exception as e: + log(f"{name}: retrieval precision check skipped — {e}") + + task = deps.test_task(skill) + log(f"{name}: testing…") + skill_messages = deps.test_messages(md, task) + transcript, skill_stats, approval_required = await deps.run_arm( + skill_messages, + url, + model, + headers, + owner, + workload=workload, + ) + if approval_required is not None: + try: + from src.tool_approvals import tool_approval_store + tool_approval_store.consume( + approval_required.get("approval_id"), + decision="deny", + owner=owner, + session_id=None, + ) + except Exception: + logger.debug("Could not retire unattended skill approval", exc_info=True) + verdict = { + "verdict": "inconclusive", + "confidence": 1.0, + "summary": ( + "This automated audit reached an exact action that requires " + "a human approval; no action was executed." + ), + "issues": [ + "Run this skill's manual test and review the sealed action." + ], + "approval_required": True, + } + else: + baseline_task = task + log(f"{name}: running no-skill baseline…") + baseline_transcript, baseline_stats, baseline_approval = await deps.run_arm( + deps.baseline_messages(baseline_task), + url, + model, + headers, + owner, + workload=workload, + ) + if baseline_approval is not None: + try: + from src.tool_approvals import tool_approval_store + tool_approval_store.consume( + baseline_approval.get("approval_id"), + decision="deny", + owner=owner, + session_id=None, + ) + except Exception: + logger.debug("Could not retire unattended baseline approval", exc_info=True) + verdict = await deps.evaluate_run( + md, + task, + transcript, + url, + model, + headers, + baseline_transcript=baseline_transcript, + skill_stats=skill_stats, + baseline_stats=baseline_stats, + workload=workload, + owner=owner, + ) + v = verdict.get("verdict") + log(f"{name}: verdict = {v} ({verdict.get('summary', '')[:80]})") + if verdict.get("approval_required"): + # An unattended audit is not authority for an action influenced by the + # skill under test. Preserve the skill's current publication/confidence + # state and route the exact action to the manual test UI instead of + # letting a safety pause demote, rewrite, or auto-publish the skill. + skills_manager.set_audit( + name, + "inconclusive", + by_teacher=False, + worker_model=model, + owner=owner, + audit_summary=verdict.get("summary") or "The test requires approval for an external action.", + ) + status = skill.get("status") or "draft" + log(f"{name}: {status} unchanged — exact action needs manual approval") + return { + "skill": name, + "result": "approval_required", + "verdict": verdict, + "status": status, + } + if v == "pass": + # Procedure works. If the reviewer still flagged metadata (tags/category/ + # when_to_use/description), do ONE fixer pass to correct the frontmatter + # without re-testing — a metadata-only fix can't break a passing run. + meta_issues = [i for i in (verdict.get("issues") or []) if str(i).lower().lstrip().startswith("metadata:")] + if meta_issues: + log(f"{name}: pass, but fixing {len(meta_issues)} metadata issue(s)…") + fixed = await deps.improve_markdown( + md, verdict, transcript, url, model, headers, workload=workload, + owner=owner, + ) + if fixed and fixed.strip() != md.strip(): + deps.apply_markdown(skills_manager, name, fixed, owner) + _set_conf(0.95) + skills_manager.set_audit( + name, + "pass", + by_teacher=False, + worker_model=model, + owner=owner, + **deps.verdict_efficiency(verdict), + ) + refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) + status = deps.finalize_status(skills_manager, name, owner, "pass", 0.95, (refreshed or {}).get("necessity"), verdict) + log(f"{name}: {status} — confidence 95%") + return {"skill": name, "result": "pass", "verdict": verdict, "confidence": 0.95, "status": status} + if v in ("unknown", "inconclusive"): + skills_manager.set_audit( + name, + "inconclusive", + by_teacher=False, + worker_model=model, + owner=owner, + **deps.verdict_efficiency(verdict), + ) + status = deps.finalize_status(skills_manager, name, owner, "inconclusive", skill.get("confidence") or 0.0, skill.get("necessity")) + log(f"{name}: {status} — inconclusive") + return {"skill": name, "result": "inconclusive", "verdict": verdict, "status": status} + + # Self-edit + retry. + log(f"{name}: self-editing to fix issues…") + new_md = await deps.improve_markdown( + md, verdict, transcript, url, model, headers, workload=workload, + owner=owner, + ) + if new_md and new_md.strip() != md.strip() and deps.apply_markdown(skills_manager, name, new_md, owner): + md = new_md + transcript, verdict = await deps.run_test_once( + md, task, url, model, headers, owner, workload=workload, + ) + v = verdict.get("verdict") + log(f"{name}: retry (self) = {v}") + if v == "pass": + _set_conf(0.85) + skills_manager.set_audit( + name, + "pass", + by_teacher=False, + worker_model=model, + owner=owner, + **deps.verdict_efficiency(verdict), + ) + refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) + status = deps.finalize_status(skills_manager, name, owner, "pass", 0.85, (refreshed or {}).get("necessity"), verdict) + log(f"{name}: {status} — confidence 85% after self-edit") + return {"skill": name, "result": "pass_after_self_edit", "verdict": verdict, "confidence": 0.85, "status": status} + + # Teacher escalation (if a distinct teacher model is configured). The + # teacher only REWRITES the skill — it does NOT run the test. The point is + # to verify the regular (student) model can now succeed with the teacher's + # improved procedure, so the retry runs on the worker model, not the teacher. + teacher_ran = False + if teacher and teacher[0] and teacher[1] and (teacher[1] != model or teacher[0] != url): + teacher_ran = True + t_url, t_model, t_headers = teacher + log(f"{name}: teacher {t_model} rewriting the skill…") + t_md = await deps.improve_markdown( + md, verdict, transcript, t_url, t_model, t_headers, workload=workload, + owner=owner, + ) + if t_md and t_md.strip() != md.strip() and deps.apply_markdown(skills_manager, name, t_md, owner): + md = t_md + # Re-test with the STUDENT model (the model the skill runs under in use). + transcript, verdict = await deps.run_test_once( + md, task, url, model, headers, owner, workload=workload, + ) + v = verdict.get("verdict") + log(f"{name}: retry on student after teacher rewrite = {v}") + if v == "pass": + _set_conf(0.8) + skills_manager.set_audit( + name, + "pass", + by_teacher=True, + worker_model=model, + teacher_model=t_model, + owner=owner, + **deps.verdict_efficiency(verdict), + ) + refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) + status = deps.finalize_status(skills_manager, name, owner, "pass", 0.8, (refreshed or {}).get("necessity"), verdict) + log(f"{name}: {status} — confidence 80% after teacher rewrite") + return {"skill": name, "result": "pass_after_teacher", "verdict": verdict, "confidence": 0.8, "status": status} + + # Still failing → demote to draft + low confidence + flag (do NOT delete). + try: + skills_manager.update_skill(name, {"status": "draft", "confidence": 0.35}, owner=owner) + except Exception: + pass + skills_manager.set_audit( + name, v or "fail", by_teacher=teacher_ran, + worker_model=model, + teacher_model=(teacher[1] if teacher_ran and teacher else ""), + owner=owner, + **deps.verdict_efficiency(verdict), + ) + log(f"{name}: flagged — confidence lowered, kept as draft for manual review") + return {"skill": name, "result": "flagged", "verdict": verdict, "confidence": 0.35} + + +async def run_audit_batch(key, skills_manager, names, url, model, headers, teacher, owner, + workload: str = "foreground", *, deps: AuditDependencies): + """Background: audit each named skill in sequence, recording progress.""" + import asyncio as _asyncio + import time as _time + + job = deps.audit_jobs.get(key) + if job is None: + return + + def log(msg): + job["log"].append(msg) + if len(job["log"]) > 1000: + del job["log"][0:len(job["log"]) - 1000] + + cancelled = False + try: + for nm in names: + if job.get("cancel"): + cancelled = True + log("(cancelled)") + break + job["current"] = nm + skills = skills_manager.load(owner=owner) + sk = next((s for s in skills if s.get("name") == nm), None) + if not sk: + continue + try: + res = await deps.audit_one( + skills_manager, sk, url, model, headers, teacher, owner, log, + workload=workload, + ) + except _asyncio.CancelledError: + cancelled = True + job["cancel"] = True + log("(cancelled)") + raise + except SkillAuditUnavailable as e: + job["unavailable"] = str(e) + log(f"Audit paused: {e}. Skill verdicts unchanged; retry when the model is available.") + break + except Exception as e: + log(f"{nm}: error — {e}") + res = {"skill": nm, "result": "error"} + skills_manager.set_audit(nm, "inconclusive", worker_model=model, owner=owner) + try: + refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == nm), None) + if refreshed: + res["skill_state"] = { + "name": refreshed.get("name"), + "status": refreshed.get("status"), + "confidence": refreshed.get("confidence"), + "audit_verdict": refreshed.get("audit_verdict"), + "audit_by_teacher": refreshed.get("audit_by_teacher"), + "audit_worker_model": refreshed.get("audit_worker_model"), + "audit_teacher_model": refreshed.get("audit_teacher_model"), + "audited_at": refreshed.get("audited_at"), + "saved_turns": refreshed.get("saved_turns"), + "saved_tool_calls": refreshed.get("saved_tool_calls"), + "baseline_verdict": refreshed.get("baseline_verdict"), + "usefulness": refreshed.get("usefulness"), + "necessity": refreshed.get("necessity"), + } + except Exception: + pass + job["results"].append(res) + job["done"] = len(job["results"]) + except _asyncio.CancelledError: + cancelled = True + finally: + if not cancelled and not job.get("cancel") and not job.get("unavailable"): + try: + deps.finalize_batch(skills_manager, job.get("results") or [], owner, log) + except Exception: + logger.warning("Could not finalize skills audit batch", exc_info=True) + job["current"] = None + job["status"] = "cancelled" if cancelled or job.get("cancel") else "error" if job.get("unavailable") else "done" + job["finished"] = _time.time() + job.pop("task", None) + + +async def run_scheduled_skill_audit(skills_manager: SkillsManager, + owner: Optional[str] = None, + max_skills: int = 8, *, deps: AuditDependencies) -> dict: + """Nightly audit pass. Audits the LEAST-recently-audited skills first and + caps the batch so it rotates through the library over successive nights + instead of re-checking the same ones every run. Reuses the same job store + the manual 'Audit all' button uses, so its progress shows in the UI too.""" + import time as _time + + key = (owner or "",) + existing = deps.audit_jobs.get(key) + if existing and existing.get("status") == "running": + logger.info("Scheduled skill audit skipped — a run is already active.") + return {"status": "running", "skipped": True} + + try: + url, model, headers, teacher = deps.resolve_models(owner=owner) + except ValueError as e: + logger.info(f"Scheduled skill audit skipped — {e}") + return {"status": "skipped", "reason": str(e)} + + from services.memory.skill_lifecycle import automatic_audit_candidates + skills = automatic_audit_candidates(skills_manager.load(owner=owner), limit=max_skills) + names = [s["name"] for s in skills] + if not names: + return {"status": "done", "total": 0} + + deps.audit_jobs[key] = { + "status": "running", "scope": "scheduled", "model": model, + "teacher": teacher[1] if teacher else None, + "total": len(names), "done": 0, "current": None, + "results": [], "log": [f"Nightly audit of {len(names)} least-recently-checked skill(s) with {model}" + + (f"; teacher {teacher[1]}" if teacher else "")], + "started": _time.time(), "cancel": False, + } + logger.info(f"Scheduled skill audit starting: {len(names)} skill(s) (owner={owner or 'all'})") + await deps.run_batch( + key, skills_manager, names, url, model, headers, teacher, owner, + workload="background", + ) + job = deps.audit_jobs.get(key, {}) + return {"status": "done", "total": len(names), "results": job.get("results", [])} diff --git a/services/skills/dependencies.py b/services/skills/dependencies.py new file mode 100644 index 000000000..5ff669a8b --- /dev/null +++ b/services/skills/dependencies.py @@ -0,0 +1,35 @@ +"""Explicit collaborators for execution and audit coordination. + +The transport compatibility bindings supply the original patchable functions. +Services never import the skills transport module or copy credentials into a +second state store. +""" +from dataclasses import dataclass +from typing import Callable + + +@dataclass(frozen=True) +class AuditDependencies: + test_jobs: dict + audit_jobs: dict + test_task: Callable + test_messages: Callable + baseline_messages: Callable + evaluate_run: Callable + evaluate_necessity: Callable + evaluate_retrieval_precision: Callable + should_check_retrieval_precision: Callable + improve_markdown: Callable + apply_markdown: Callable + duplicate_blocker: Callable + finalize_status: Callable + finalize_batch: Callable + auto_publish_policy: Callable + utility_blocker: Callable + verdict_efficiency: Callable + run_test_job: Callable + run_arm: Callable + run_test_once: Callable + audit_one: Callable + run_batch: Callable + resolve_models: Callable diff --git a/services/skills/evaluation.py b/services/skills/evaluation.py new file mode 100644 index 000000000..85ebeeadf --- /dev/null +++ b/services/skills/evaluation.py @@ -0,0 +1,464 @@ +"""Model grading and advisory skill assessment with explicit offline call seams.""" + +import logging +import re +from typing import Optional + +from src.text_helpers import strip_closed_think_blocks +from src.utility_effort import effort_for_call + +logger = logging.getLogger(__name__) + +# Last-resort verdict extraction from a teacher/verifier model's prose (run when +# JSON parsing fails). `["\'\s:]*` already consumes whitespace, so the original +# trailing `\s*` made two adjacent \s-matching quantifiers that backtrack O(n^2) +# on a `verdict` + whitespace flood in untrusted model output (CodeQL +# py/polynomial-redos). Without it a single unbounded quantifier remains — the +# matched text is identical, and the scan is linear. +_VERDICT_PROSE_RE = re.compile( + r'verdict["\'\s:]*["\']?(pass|needs_work|fail|inconclusive)', re.I +) + + +def verdict_efficiency(verdict: Optional[dict]) -> dict: + if not isinstance(verdict, dict): + return {} + out = {"audit_summary": str(verdict.get("summary") or "")[:2000]} + for key in ("saved_turns", "saved_tool_calls"): + if key not in verdict: + continue + try: + out[key] = int(verdict.get(key)) + except (TypeError, ValueError): + pass + baseline = str(verdict.get("baseline_verdict") or "").lower().strip() + if baseline in {"better", "same", "worse", "unknown"}: + out["baseline_verdict"] = baseline + if "usefulness" in verdict: + try: + out["usefulness"] = max(0.0, min(1.0, float(verdict.get("usefulness")))) + except (TypeError, ValueError): + pass + return out + + +def parse_run_verdict(raw: str): + """Return a final result dict on success, or None if unparseable.""" + import json as _json + import re as _re + _VERDICTS = ("pass", "needs_work", "fail", "inconclusive") + + text = (raw or '') + # Strip closed think blocks. If a was opened but never closed + # (the model ran out of budget mid-reasoning), drop everything from it + # onward so its stray braces don't poison JSON extraction. + text = strip_closed_think_blocks(text) + text = _re.sub(r'[\s\S]*$', '', text, flags=_re.I).strip() + + def _coerce(d): + return d if (isinstance(d, dict) and "verdict" in d) else None + + data = None + # Scan every balanced {...} candidate and keep the LAST one that parses + # and carries a "verdict" — the transcript is full of JSON API bodies, + # so a naive first-brace/last-brace span almost never parses. + for m in _re.finditer(r'\{[\s\S]*?\}', text): + frag = m.group(0) + for cand in (frag, _re.sub(r',(\s*[}\]])', r'\1', frag)): + try: + d = _coerce(_json.loads(cand)) + except Exception: + d = None + if d is not None: + data = d + # Fallback to the greedy outermost span (handles nested objects the + # non-greedy scan above splits apart). + if data is None: + a, b = text.find('{'), text.rfind('}') + if a >= 0 and b > a: + frag = text[a:b + 1] + for cand in (frag, _re.sub(r',(\s*[}\]])', r'\1', frag)): + try: + d = _coerce(_json.loads(cand)) + except Exception: + d = None + if d is not None: + data = d + break + + v = str(data.get("verdict", "")).lower().strip() if isinstance(data, dict) else None + # Last resort: pull the verdict keyword straight out of the prose so a + # clearly-decided run isn't thrown away as "unparseable". + if v not in _VERDICTS: + km = _VERDICT_PROSE_RE.search(text) + if km: + v = km.group(1).lower() + if data is None: + data = {} + if not isinstance(data, dict) or v not in _VERDICTS: + return None + try: + conf = float(data.get("confidence", 0)) + except (TypeError, ValueError): + conf = 0 + try: + usefulness = float(data.get("usefulness", 0.0)) + except (TypeError, ValueError): + usefulness = 0.0 + try: + saved_turns = int(data.get("saved_turns", 0)) + except (TypeError, ValueError): + saved_turns = 0 + try: + saved_tool_calls = int(data.get("saved_tool_calls", 0)) + except (TypeError, ValueError): + saved_tool_calls = 0 + return { + "verdict": v, + "confidence": max(0.0, min(1.0, conf)), + "summary": str(data.get("summary", ""))[:400], + "issues": [str(x)[:200] for x in (data.get("issues") or []) if str(x).strip()][:8], + "baseline_verdict": ( + str(data.get("baseline_verdict", "unknown")).lower().strip() + if str(data.get("baseline_verdict", "unknown")).lower().strip() in {"better", "same", "worse", "unknown"} + else "unknown" + ), + "usefulness": max(0.0, min(1.0, usefulness)), + "saved_turns": saved_turns, + "saved_tool_calls": saved_tool_calls, + } + + +async def evaluate_run(skill_md: str, task: str, transcript: str, + url: str, model: str, headers: Optional[dict], + baseline_transcript: str = "", + skill_stats: Optional[dict] = None, + baseline_stats: Optional[dict] = None, + workload: str = "foreground", owner: Optional[str] = None, + *, _llm_call=None, _effort_for_call=None) -> dict: + """LLM-as-judge: grade a skill test run from its transcript. Advisory only. + + Robust against local reasoning models (strips , lenient JSON, + generous token budget) — same defensive parsing used elsewhere. + """ + if _llm_call is None: + from src.llm_core import llm_call_async + _llm_call = llm_call_async + _effort_for_call = _effort_for_call or effort_for_call + + sys_prompt = ( + "You are a strict QA reviewer judging whether an AI 'skill' (a reusable " + "procedure) actually works. You are given the SKILL, the TASK it was tested " + "on, and the TRANSCRIPT of the agent's run.\n\n" + "Judge honestly:\n" + "- The functional verdict judges ONLY whether the SKILL RUN completed the " + "task correctly and whether the skill procedure is usable. A correct skill " + "run is pass even when the baseline is equally good. Record comparative " + "value separately in baseline_verdict/usefulness.\n" + "- Did following the skill accomplish the task accurately?\n" + "- Are the steps clear, correct, and reproducible?\n" + "- Did it reference tools/commands that don't exist or that errored?\n" + "- Separately, compared with the baseline run WITHOUT the skill, did the skill make " + "the agent more accurate, use fewer turns/tool calls, or avoid avoidable " + "thrashing?\n" + "- Do NOT penalize a skill merely for being broad or generic. A broad " + "skill is useful if it makes the agent finish correctly in fewer turns " + "or with fewer tools than baseline.\n" + "- METADATA: do the frontmatter fields match what the skill actually does? " + "Flag wrong/misleading/missing tags, a wrong category, a when_to_use that " + "doesn't describe the real trigger, or a description that oversells or " + "mismatches the body. List each metadata problem in 'issues' (prefix it " + "with 'metadata:'). Metadata problems alone do NOT make the verdict 'fail' " + "if the procedure works — note them as issues on an otherwise-passing run.\n\n" + "Never use inconclusive merely because the baseline was the same or better. " + "IMPORTANT — fairness rule: if the SKILL RUN could NOT proceed because it lacked " + "an input or target the test never provided (e.g. there was no document/" + "email/data to act on, so the agent reasonably asked for it), that is NOT " + "the skill's fault. Return verdict \"inconclusive\" — do NOT mark it fail " + "or needs_work. Only judge the skill's PROCEDURE; reserve fail/needs_work " + "for when the steps themselves are wrong, vague, or reference missing tools.\n\n" + "If you need to reason, do it inside FIRST. Then output " + "ONLY this JSON (no fences):\n" + 'Set baseline_verdict to how the SKILL RUN compares to the BASELINE RUN.\n\n' + '{"verdict": "pass" | "needs_work" | "fail" | "inconclusive", ' + '"confidence": 0.0-1.0, "summary": "one short sentence", ' + '"issues": ["short issue", ...], ' + '"baseline_verdict": "better" | "same" | "worse" | "unknown", ' + '"usefulness": 0.0-1.0, "saved_turns": integer, "saved_tool_calls": integer}' + ) + # Give the judge plenty of transcript, and when it must trim, keep the TAIL + # (the final result lives at the end) plus a bit of the head — truncating to + # a short prefix made the judge wrongly call complete runs "incomplete / + # missing sections" because it never saw the end. + def _clip(t: str, limit: int = 24000) -> str: + t = (t or "").strip() or "(no output produced)" + if len(t) <= limit: + return t + head = limit // 4 + return t[:head] + "\n\n…[transcript trimmed for length]…\n\n" + t[-(limit - head):] + skill_stats = skill_stats or {} + baseline_stats = baseline_stats or {} + user_msg = ( + f"=== SKILL ===\n{(skill_md or '')[:4000]}\n\n" + f"=== TASK ===\n{task}\n\n" + f"=== SKILL RUN STATS ===\n" + f"turns={skill_stats.get('turns', 'unknown')} " + f"tool_calls={skill_stats.get('tool_calls', 'unknown')}\n\n" + f"=== SKILL RUN TRANSCRIPT ===\n{_clip(transcript)}\n\n" + f"=== BASELINE RUN WITHOUT SKILL STATS ===\n" + f"turns={baseline_stats.get('turns', 'unknown')} " + f"tool_calls={baseline_stats.get('tool_calls', 'unknown')}\n\n" + f"=== BASELINE RUN WITHOUT SKILL TRANSCRIPT ===\n{_clip(baseline_transcript)}" + ) + + # Two attempts: the first lets the judge reason; if a heavy reasoning model + # burns its budget inside and never emits the JSON, the second + # forbids thinking and demands the JSON immediately. + last_text = "" + last_err = None + for attempt in range(2): + msgs = [{"role": "system", "content": sys_prompt}, + {"role": "user", "content": user_msg}] + if attempt == 1: + msgs[0]["content"] = ( + sys_prompt + "\n\nDO NOT use or any reasoning. Your reply " + "must START with '{' and be ONLY the JSON object, nothing else." + ) + try: + raw = await _llm_call( + # Generous budget so a heavy reasoner can think AND still have + # room to emit the JSON afterwards (reasoning tokens come out of + # this same cap; the server clamps to its own max). + url, model, msgs, + temperature=0.1, max_tokens=32768, headers=headers, timeout=180, + workload=workload, + reasoning_effort=_effort_for_call(url, model, owner), + ) + except Exception as e: + # Don't give up on a transient first-attempt error — let the second + # (no-think) attempt run before reporting failure. + last_err = e + continue + last_text = (raw or '') + parsed = parse_run_verdict(raw) + if parsed is not None: + return parsed + + if last_err is not None and not last_text: + return {"verdict": "unknown", "confidence": 0, "summary": f"Review call failed: {last_err}", "issues": []} + return {"verdict": "unknown", "confidence": 0, + "summary": "Reviewer returned unparseable output.", "issues": [], "raw": last_text[:300]} + + +async def evaluate_necessity(skill_md: str, others: list, url: str, model: str, + headers: Optional[dict], + workload: str = "foreground", owner: Optional[str] = None, + *, _llm_call=None, _effort_for_call=None) -> Optional[dict]: + """Advisory judge: is this skill worth keeping, or is it redundant / trivially + unnecessary? Sees the OTHER skills' names+descriptions so it can spot + duplicates. Returns {necessary, redundant_with, reason} or None. Never acts — + purely a flag the UI surfaces.""" + import json as _json + import re as _re + if _llm_call is None: + from src.llm_core import llm_call_async + _llm_call = llm_call_async + _effort_for_call = _effort_for_call or effort_for_call + + catalog = "\n".join(f"- {o.get('name')}: {o.get('description', '')}" for o in others) or "(no other skills)" + sys_prompt = ( + "You assess whether a reusable AI 'skill' (a saved procedure) is worth keeping. " + "A skill is UNNECESSARY if it essentially duplicates another skill in the library. " + "Do NOT call a skill unnecessary merely because it is broad or generic; broad " + "skills can be worth keeping when they make a capable assistant finish accurately " + "with fewer turns or fewer tool calls than it would without the skill. A skill IS " + "necessary if it captures a reusable trigger, procedure, tool sequence, or " + "hard-won detail that could improve future runs.\n\n" + "Be conservative: only call it unnecessary when you're confident. Reason in " + " first if needed, then output ONLY this JSON:\n" + '{"necessary": true|false, "redundant_with": ["skill-name", ...], ' + '"reason": "one short sentence"}' + ) + user_msg = ( + f"=== SKILL UNDER REVIEW ===\n{(skill_md or '')[:3000]}\n\n" + f"=== OTHER SKILLS IN THE LIBRARY ===\n{catalog[:4000]}" + ) + try: + raw = await _llm_call( + url, model, + [{"role": "system", "content": sys_prompt}, {"role": "user", "content": user_msg}], + temperature=0.1, max_tokens=8192, headers=headers, timeout=120, + workload=workload, + reasoning_effort=_effort_for_call(url, model, owner), + ) + except Exception as e: + logger.warning(f"Necessity check failed: {e}") + return None + text = strip_closed_think_blocks(raw or '') + text = _re.sub(r'[\s\S]*$', '', text, flags=_re.I).strip() + data = None + a, b = text.find('{'), text.rfind('}') + if a >= 0 and b > a: + frag = text[a:b + 1] + for cand in (frag, _re.sub(r',(\s*[}\]])', r'\1', frag)): + try: + data = _json.loads(cand) + break + except Exception: + continue + if not isinstance(data, dict) or "necessary" not in data: + return None + return { + "necessary": bool(data.get("necessary", True)), + "redundant_with": [str(x)[:80] for x in (data.get("redundant_with") or []) if str(x).strip()][:6], + "reason": str(data.get("reason", ""))[:200], + } + + +def should_check_retrieval_precision(skill: dict) -> bool: + """Cheap prefilter for the expensive retrieval-precision judge. + + Skills with broad tags like "network" or "document" are the ones most likely + to over-inject. Narrow command/vendor tags alone are fine. + """ + broad = { + "arch", "arch linux", "linux", "network", "networking", "wifi", + "installation", "install", "system", "ssh", "document", "documents", + "search", "email", "calendar", "gpu", "server", "python", + } + if not isinstance(skill, dict): + return False + tags = {str(t or "").strip().lower() for t in (skill.get("tags") or [])} + if tags & broad: + return True + text = " ".join([ + str(skill.get("name") or ""), + str(skill.get("description") or ""), + str(skill.get("when_to_use") or ""), + ]).lower() + return sum(1 for t in broad if t in text) >= 2 + + +async def evaluate_retrieval_precision(skill_md: str, others: list, + url: str, model: str, + headers: Optional[dict], + workload: str = "foreground", owner: Optional[str] = None, + *, _llm_call=None, _effort_for_call=None) -> Optional[dict]: + """Advisory judge: would this skill's metadata make retrieval over-select it? + + This is distinct from "does the procedure work?". It asks whether tags, + description, and when_to_use are specific enough that the skill won't be + injected into adjacent but wrong tasks. + """ + import json as _json + import re as _re + if _llm_call is None: + from src.llm_core import llm_call_async + _llm_call = llm_call_async + _effort_for_call = _effort_for_call or effort_for_call + + catalog = "\n".join(f"- {o.get('name')}: {o.get('description', '')}" for o in others[:80]) or "(no other skills)" + sys_prompt = ( + "You are auditing retrieval metadata for a reusable AI skill. The app selects " + "skills by matching user requests against the skill name, description, tags, " + "when_to_use, and procedure text. Judge whether this skill is likely to be " + "over-selected for nearby but wrong requests.\n\n" + "Focus ONLY on metadata/retrieval precision, not whether the procedure works. " + "Flag broad tags such as network, installation, system, document, search, ssh, " + "python, gpu, or server when they would cause this narrow skill to match too " + "many adjacent tasks. Recommend narrower tags/when_to_use wording. Compare " + "against the other skills to spot boundaries.\n\n" + "Return ok=true only when the trigger metadata is narrow enough. If not ok, " + "issues MUST start with 'metadata: retrieval:' and be actionable. Output ONLY JSON:\n" + '{"ok": true|false, "summary": "one short sentence", "issues": ["metadata: retrieval: ..."]}' + ) + user_msg = ( + f"=== SKILL UNDER REVIEW ===\n{(skill_md or '')[:5000]}\n\n" + f"=== OTHER SKILLS IN LIBRARY ===\n{catalog[:5000]}\n\n" + "Decide if this skill's retrieval metadata should be narrowed so it only " + "fires for its intended scenario and not for adjacent skills above." + ) + try: + raw = await _llm_call( + url, model, + [{"role": "system", "content": sys_prompt}, {"role": "user", "content": user_msg}], + temperature=0.1, max_tokens=4096, headers=headers, timeout=90, + workload=workload, + reasoning_effort=_effort_for_call(url, model, owner), + ) + except Exception as e: + logger.warning(f"Retrieval precision check failed: {e}") + return None + text = strip_closed_think_blocks(raw or '') + text = _re.sub(r'[\s\S]*$', '', text, flags=_re.I).strip() + data = None + a, b = text.find('{'), text.rfind('}') + if a >= 0 and b > a: + frag = text[a:b + 1] + for cand in (frag, _re.sub(r',(\s*[}\]])', r'\1', frag)): + try: + data = _json.loads(cand) + break + except Exception: + continue + if not isinstance(data, dict) or "ok" not in data: + return None + return { + "ok": bool(data.get("ok")), + "summary": str(data.get("summary", ""))[:300], + "issues": [str(x)[:220] for x in (data.get("issues") or []) if str(x).strip()][:6], + } + + +async def improve_markdown(skill_md: str, verdict: dict, transcript: str, url, model, headers, + workload: str = "foreground", owner: Optional[str] = None, + *, _llm_call=None, _effort_for_call=None): + """Have a model rewrite SKILL.md to fix the reviewer's issues. Returns the + corrected markdown, or None if it couldn't produce a usable change.""" + import re as _re + if _llm_call is None: + from src.llm_core import llm_call_async + _llm_call = llm_call_async + _effort_for_call = _effort_for_call or effort_for_call + issues = "\n".join("- " + str(i) for i in (verdict.get("issues") or [])) + sys_prompt = ( + "You are improving a reusable AI SKILL written in Markdown (frontmatter + body). " + "A QA reviewer found problems after a test run. Rewrite the SKILL.md to fix them: " + "make vague steps concrete, correct or remove references to tools that don't exist, " + "ensure the procedure is reproducible. " + "Keep the `name` field EXACTLY as-is (it is the skill's identity / filename). You MAY " + "correct the OTHER frontmatter — tags, category, when_to_use, description — when the " + "reviewer flagged them (issues prefixed 'metadata:') or they don't match the body; keep " + "retrieval metadata narrow: remove broad tags that would over-select the skill, and make " + "`when_to_use` say when NOT to use the skill if adjacent tasks are easy to confuse. Keep " + "valid frontmatter structure. Do NOT invent capabilities the agent lacks. Reason in " + " first if needed, then output ONLY the full corrected SKILL.md (no " + "fences, no commentary)." + ) + user_msg = ( + f"=== CURRENT SKILL.md ===\n{skill_md}\n\n" + f"=== REVIEWER VERDICT ===\n{verdict.get('summary', '')}\nIssues:\n{issues}\n\n" + f"=== TEST TRANSCRIPT ===\n{(transcript or '')[:6000]}" + ) + try: + raw = await _llm_call(url, model, + [{"role": "system", "content": sys_prompt}, + {"role": "user", "content": user_msg}], + temperature=0.2, max_tokens=16384, headers=headers, timeout=180, + workload=workload, + reasoning_effort=_effort_for_call(url, model, owner)) + except Exception as e: + logger.warning(f"Audit: improve call failed: {e}") + return None + text = strip_closed_think_blocks(raw or '') + text = _re.sub(r'[\s\S]*$', '', text, flags=_re.I) + text = _re.sub(r'', '', text, flags=_re.I).strip() + if text.startswith("```"): + text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip() + # Some reasoning models still prepend analysis or echo the old skill before + # the final corrected document. Keep only the last complete-looking + # frontmatter document so reviewer prose never gets persisted into SKILL.md. + starts = list(_re.finditer(r'(?m)^---\s*\n(?=[\s\S]*?^name\s*:)', text)) + if starts: + text = text[starts[-1].start():].strip() + return text or None diff --git a/services/skills/job_coordinator.py b/services/skills/job_coordinator.py new file mode 100644 index 000000000..1f737a519 --- /dev/null +++ b/services/skills/job_coordinator.py @@ -0,0 +1,216 @@ +"""Transient job transitions and public polling projections. + +HTTP ownership and approval validation happen before these operations. These +functions use the supplied registries and preserve the existing task ownership. +""" +import asyncio as _asyncio +import logging +import time as _time + +from src.prompt_security import untrusted_context_message +from .dependencies import AuditDependencies + +logger = logging.getLogger(__name__) + + +def start_test_job(name, md, task, url, model, headers, user, skills_manager, + request_authority, *, deps: AuditDependencies): + key = (user or "", name) + previous_job = deps.test_jobs.get(key) or {} + previous_approval = previous_job.get("approval") or {} + if previous_approval.get("approval_id"): + try: + from src.tool_approvals import tool_approval_store + tool_approval_store.consume( + previous_approval["approval_id"], + decision="deny", + owner=user, + session_id=None, + ) + except Exception: + logger.debug("Could not retire replaced skill approval", exc_info=True) + deps.test_jobs[key] = { + "status": "running", + "task": task, + "model": model, + "skill": name, + "started": _time.time(), + "log": [{"type": "skill_test_start", "task": task, "skill": name, "model": model}], + "verdict": None, + "_run": { + "md": md, + "url": url, + "model": model, + "headers": headers, + "owner": user, + "request_authority": request_authority, + }, + } + _asyncio.create_task(deps.run_test_job( + key, name, md, task, url, model, headers, user, skills_manager, + request_authority=request_authority)) + return {"ok": True, "status": "running", "skill": name, "model": model} + + +def continue_test_job(job, key, name, decision, expected, exact_approval, + skills_manager, *, deps: AuditDependencies): + job.pop("approval", None) + if decision == "deny": + job.pop("_transcript", None) + job.pop("_run", None) + job["log"].append({ + "type": "approval_denied", + "text": "Exact action denied; the skill test stopped without executing it.", + }) + job["verdict"] = { + "verdict": "inconclusive", + "confidence": 1.0, + "summary": "The test stopped because its exact action was denied.", + "issues": [], + } + job["status"] = "done" + return {"ok": True, "status": "done", "decision": "deny"} + + run = job.get("_run") or {} + transcript = job.pop("_transcript", []) + # stream_agent_loop owns its per-round message list internally. Rebuild + # continuation context from the original untrusted skill plus the + # accumulated transcript so repeated approvals do not lose earlier + # approved results, while keeping every transcript byte tainted. + messages = deps.test_messages( + run.get("md", ""), + job.get("task", ""), + ) + if transcript: + messages.append(untrusted_context_message( + "skill test transcript", + "".join(str(item) for item in transcript), + )) + messages.extend([ + { + "role": "assistant", + "content": str(expected.get("question") or "Allow this exact action once?"), + }, + { + "role": "user", + "content": ( + f"Approved the exact {exact_approval.pending.tool_name} " + "action shown above once." + ), + }, + ]) + job["status"] = "running" + job["log"].append({ + "type": "approval_granted", + "text": ( + f"Approved exact {exact_approval.pending.tool_name} action once; " + "resuming test." + ), + }) + _asyncio.create_task(deps.run_test_job( + key, + name, + run.get("md", ""), + job.get("task", ""), + run.get("url"), + run.get("model"), + run.get("headers"), + run.get("owner"), + skills_manager, + messages=messages, + transcript=transcript, + exact_approval=exact_approval, + )) + return {"ok": True, "status": "running", "decision": "approve"} + + +def test_status(user, name, *, deps: AuditDependencies): + job = deps.test_jobs.get((user or "", name)) + if not job: + return {"status": "none"} + return { + "status": job["status"], + "task": job.get("task"), + "model": job.get("model"), + "log": job.get("log", []), + "verdict": job.get("verdict"), + "approval": job.get("approval"), + } + + +def select_audit_names(skills_manager, user, scope, requested_names, skip_audited): + skills = skills_manager.load(owner=user) + # Built-ins are tracked, pre-approved application procedures. They do + # not consume audit turns and cannot be demoted by an audit result. + auditable_skills = [s for s in skills if s.get("source") != "builtin"] + by_name = {s.get("name"): s for s in auditable_skills if s.get("name")} + if isinstance(requested_names, list): + names = [] + seen = set() + for raw in requested_names: + nm = str(raw or "").strip() + if not nm or nm in seen or nm not in by_name: + continue + if scope not in ("all", "selected") and (by_name[nm].get("status") or "draft") == "published": + continue + if skip_audited and by_name[nm].get("audit_verdict"): + continue + names.append(nm) + seen.add(nm) + scope = "selected" if requested_names else scope + elif scope == "all": + names = [ + s.get("name") for s in auditable_skills + if s.get("name") and (not skip_audited or not s.get("audit_verdict")) + ] + else: + scope = "unchecked" if scope == "drafts" else scope + names = [ + s.get("name") for s in auditable_skills + if s.get("name") + and (s.get("status") or "draft") != "published" + and not s.get("audit_verdict") + ] + return names, scope + + +def start_audit_job(key, skills_manager, names, url, model, headers, teacher, user, + scope, *, deps: AuditDependencies): + if not names: + return {"ok": True, "status": "done", "total": 0, "results": [], "log": ["No skills to audit."]} + + deps.audit_jobs[key] = { + "status": "running", "scope": scope, "model": model, + "teacher": teacher[1] if teacher else None, + "total": len(names), "done": 0, "current": None, + "results": [], "log": [f"Auditing {len(names)} skill(s) with {model}" + (f"; teacher {teacher[1]}" if teacher else "")], + "started": _time.time(), "cancel": False, + } + task = _asyncio.create_task(deps.run_batch(key, skills_manager, names, url, model, headers, teacher, user)) + deps.audit_jobs[key]["task"] = task + return {"ok": True, "status": "running", "total": len(names), "model": model} + + +def audit_status(user, *, deps: AuditDependencies): + job = deps.audit_jobs.get((user or "",)) + if not job: + return {"status": "none"} + return { + "status": job["status"], "scope": job.get("scope"), + "total": job.get("total", 0), "done": job.get("done", 0), + "current": job.get("current"), "model": job.get("model"), "teacher": job.get("teacher"), + "results": job.get("results", []), "log": job.get("log", []), + "started": job.get("started"), "finished": job.get("finished"), + } + + +def cancel_audit(user, *, deps: AuditDependencies): + job = deps.audit_jobs.get((user or "",)) + if job: + job["cancel"] = True + job["status"] = "cancelled" + job["current"] = None + task = job.get("task") + if task and not task.done(): + task.cancel() + return {"ok": True, "status": "cancelled" if job else "none"} diff --git a/services/skills/model_selection.py b/services/skills/model_selection.py new file mode 100644 index 000000000..cd446632c --- /dev/null +++ b/services/skills/model_selection.py @@ -0,0 +1,75 @@ +"""Owner-scoped audit worker and teacher endpoint resolution.""" + +import logging + + +logger = logging.getLogger(__name__) + +def resolve_audit_models(owner=None, model_spec=None, endpoint_url=None): + """Resolve (url, model, headers, teacher) for an audit run from Settings. + + Worker = Utility model (falling back to Default, normalized to a served + model id); teacher = the optional Settings → Teacher Model config. Shared + by the manual /audit-all route and scheduled/event audits. Raises + ValueError if no worker model. + """ + from src.endpoint_resolver import resolve_endpoint, resolve_utility_fallback_candidates + if model_spec and endpoint_url: + # Scheduled tasks store the endpoint URL and model separately. Resolve + # the endpoint directly so an explicit task choice cannot be replaced + # by the global Utility setting. + from src.endpoint_resolver import build_headers, resolve_endpoint_runtime + from src.database import ModelEndpoint, SessionLocal + from src.endpoint_resolver import normalize_base, same_endpoint_base + from src.auth_helpers import owner_filter + url = endpoint_url + model = model_spec + headers = {} + db = SessionLocal() + try: + query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) + for ep in owner_filter(query, ModelEndpoint, owner).all(): + base = normalize_base(getattr(ep, "base_url", "") or "") + if same_endpoint_base(url, base): + runtime_base, api_key = resolve_endpoint_runtime(ep, owner=owner) + headers = build_headers(api_key, runtime_base or base) + break + finally: + db.close() + elif model_spec: + from src.ai_interaction import _resolve_model + url, model, headers = _resolve_model(str(model_spec), owner=owner) + else: + url, model, headers = resolve_endpoint("utility", owner=owner) + if not url or not model: + # Utility fallbacks are an explicit part of the user's model + # configuration. Audits must use the same chain as other background + # work instead of treating an empty primary Utility slot as fatal. + for fallback_url, fallback_model, fallback_headers in resolve_utility_fallback_candidates(owner=owner): + url, model, headers = fallback_url, fallback_model, fallback_headers + break + if not url or not model: + raise ValueError("No model configured — set a Default or Utility model in Settings.") + try: + from src.llm_core import list_model_ids + import os as _os + _avail = list_model_ids(url, headers=headers) + if _avail and model not in _avail: + _base = _os.path.basename((model or "").rstrip("/")) + model = next((a for a in _avail if _os.path.basename(a.rstrip("/")) == _base), None) or _avail[0] + except Exception: + pass + + teacher = None + try: + from src.settings import get_setting + if get_setting("teacher_enabled", False): + spec = (get_setting("teacher_model", "") or "").strip() + if spec: + from src.ai_interaction import _resolve_model + t_url, t_model, t_headers = _resolve_model(spec, owner=owner) + if t_url and t_model: + teacher = (t_url, t_model, t_headers) + except Exception as e: + logger.warning(f"Audit teacher resolve failed: {e}") + return url, model, headers, teacher diff --git a/services/skills/prompts.py b/services/skills/prompts.py new file mode 100644 index 000000000..42e32e000 --- /dev/null +++ b/services/skills/prompts.py @@ -0,0 +1,52 @@ +"""Identical audit tasks and explicitly untrusted skill context.""" + +from src.prompt_security import untrusted_context_message + +def _skill_test_task(skill: dict) -> str: + """Build the one shared task used by both sides of an audit comparison.""" + if not isinstance(skill, dict): + skill = {} + ctx = (skill.get("when_to_use") or skill.get("description") or skill.get("name") or "").strip() + return ( + "Complete this task end-to-end. Use this exact task context and do not ask " + "the user for more input. If a harmless fixture is needed, create the " + "smallest realistic one that satisfies the context, state it explicitly, " + "then complete and verify the result. Do not perform destructive or " + "externally visible actions. Task context: " + (ctx or "(general)") + ) + + +def _skill_baseline_task(skill: dict) -> str: + """Backward-compatible alias; both audit arms must receive identical text.""" + return _skill_test_task(skill) + + +def _skill_test_messages(md: str, task: str) -> list[dict]: + """Keep user-editable skill text out of the trusted system role.""" + return [ + { + "role": "system", + "content": ( + "You are TESTING a skill. Follow the supplied reusable procedure " + "to complete the user's task for real, using available tools step " + "by step. If the skill is wrong, unclear, or references tools that " + "do not exist, do your best; the problems will be reviewed afterward." + ), + }, + untrusted_context_message("skill under test", md), + {"role": "user", "content": task}, + ] + + +def _skill_baseline_messages(task: str) -> list[dict]: + return [ + { + "role": "system", + "content": ( + "You are completing a baseline audit run. Do not use any saved " + "skill text. Solve the user's task using only your normal tools " + "and reasoning." + ), + }, + {"role": "user", "content": task}, + ] diff --git a/services/skills/publication.py b/services/skills/publication.py new file mode 100644 index 000000000..d5267d7e6 --- /dev/null +++ b/services/skills/publication.py @@ -0,0 +1,349 @@ +"""Audit publication and duplicate coordination through the canonical SkillsManager.""" + +import logging +import re +from typing import Optional + +from .dependencies import AuditDependencies + +logger = logging.getLogger(__name__) + +def auto_publish_policy(owner) -> tuple[bool, float]: + """Return (auto_publish_enabled, minimum_confidence) for audit finalization.""" + try: + from routes.prefs_routes import _load_for_user + prefs = _load_for_user(owner) or {} + except Exception: + prefs = {} + try: + from src.settings import get_setting + default_min = get_setting("skill_autosave_min_confidence", 0.85) + except Exception: + default_min = 0.85 + enabled = bool(prefs.get("auto_approve_skills", True)) + try: + min_conf = float(prefs.get("skill_min_confidence", default_min)) + except (TypeError, ValueError): + min_conf = 0.85 + return enabled, max(0.0, min(1.0, min_conf)) + + +def duplicate_blocker(skills_manager, name: str, owner) -> Optional[str]: + """Cheap duplicate guard matching the UI's duplicate grouping. + + The LLM necessity check catches semantic redundancy, but the UI also has a + cheap similarity pass. Use the same broad signal before auto-publishing so + a high-scoring lower-priority duplicate stays draft. + """ + import re as _re + + def _tokens(sk: dict) -> set[str]: + text = " ".join([ + str(sk.get("name") or ""), + str(sk.get("description") or ""), + str(sk.get("when_to_use") or ""), + " ".join(sk.get("procedure") or []), + " ".join(sk.get("tags") or []), + ]).lower() + text = _re.sub(r"-\d+\b", "", text) + return { + t for t in _re.split(r"[^a-z0-9]+", text) + if len(t) > 2 and t not in {"the", "and", "with", "for", "from", "using"} + } + + def _sim(a: dict, b: dict) -> float: + A, B = _tokens(a), _tokens(b) + if not A or not B: + return 0.0 + return len(A & B) / max(1, len(A | B)) + + def _base(n: str) -> str: + return _re.sub(r"-\d+$", "", str(n or "")) + + def _score(sk: dict) -> float: + return ( + (100000 if (sk.get("status") == "published") else 0) + + int(sk.get("uses") or 0) * 100 + + round(float(sk.get("confidence") or 0) * 100) + + (-5 if sk.get("audit_by_teacher") else 0) + - (len(str(sk.get("name") or "")) / 1000) + ) + + skills = [s for s in skills_manager.load(owner=owner) if s.get("status") != "binned"] + current = next((s for s in skills if (s.get("name") or s.get("id")) == name), None) + if not current: + return None + duplicates = [] + cur_name = current.get("name") or current.get("id") or name + for other in skills: + other_name = other.get("name") or other.get("id") + if not other_name or other_name == cur_name: + continue + if _base(cur_name) == _base(other_name) or _sim(current, other) >= 0.38: + duplicates.append(other) + if not duplicates: + return None + keeper = sorted([current, *duplicates], key=_score, reverse=True)[0] + keeper_name = keeper.get("name") or keeper.get("id") or "" + if keeper_name and keeper_name != cur_name: + try: + skills_manager.set_necessity( + cur_name, + False, + [keeper_name], + f"Lower-priority duplicate of {keeper_name}", + owner=owner, + ) + except Exception: + pass + return keeper_name + return None + + +def finalize_batch(skills_manager, results: list[dict], owner, log, *, deps: AuditDependencies) -> None: + """Draft audited failures, bin duplicate losers, and publish the best copy. + + Binned skills remain on disk for recovery and inspection, but the Skills + manager excludes them from retrieval. Only skills actually processed by + this audit job are changed here; an unrelated existing skill is never + moved just because it resembles an audited one. + """ + import re as _re + + auto_publish, min_conf = deps.auto_publish_policy(owner) + current = [ + s for s in skills_manager.load(owner=owner) + if s.get("source") != "builtin" and s.get("status") != "binned" + ] + by_name = {s.get("name"): s for s in current if s.get("name")} + processed = {str(r.get("skill")) for r in results if r.get("skill")} + protected = { + str(r.get("skill")) for r in results + if r.get("skill") and r.get("result") == "approval_required" + } + + def tokens(sk: dict) -> set[str]: + text = " ".join([ + str(sk.get("name") or ""), str(sk.get("description") or ""), + str(sk.get("when_to_use") or ""), " ".join(sk.get("procedure") or []), + " ".join(sk.get("tags") or []), + ]).lower() + text = _re.sub(r"-\d+\b", "", text) + return { + t for t in _re.split(r"[^a-z0-9]+", text) + if len(t) > 2 and t not in {"the", "and", "with", "for", "from", "using"} + } + + def similar(a: dict, b: dict) -> float: + left, right = tokens(a), tokens(b) + return len(left & right) / max(1, len(left | right)) if left and right else 0.0 + + def base(name: str) -> str: + return _re.sub(r"-\d+$", "", str(name or "")) + + def score(sk: dict) -> float: + try: + confidence = float(sk.get("confidence") or 0) + except (TypeError, ValueError): + confidence = 0.0 + return ( + (100000 if sk.get("status") == "published" else 0) + + int(sk.get("uses") or 0) * 100 + + round(confidence * 100) + + (-5 if sk.get("audit_by_teacher") else 0) + - len(str(sk.get("name") or "")) / 1000 + ) + + # Anything that does not clear the configured policy stays a draft. Drafts + # are excluded from retrieval/injection by SkillsManager.index_for(). + for name in processed - protected: + skill = by_name.get(name) + if not skill: + continue + verdict = str(skill.get("audit_verdict") or "").lower() + try: + confidence = float(skill.get("confidence") or 0) + except (TypeError, ValueError): + confidence = 0.0 + if verdict in {"needs_work", "fail"} or ( + verdict == "pass" and confidence < min_conf + ): + try: + skills_manager.update_skill(name, {"status": "draft"}, owner=owner) + log(f"{name}: kept as draft after audit") + except Exception: + logger.warning("Could not bin audited skill %s", name, exc_info=True) + + # Build the same connected duplicate groups shown by the UI. + parent = {s["name"]: s["name"] for s in current} + + def find(name: str) -> str: + while parent[name] != name: + parent[name] = parent[parent[name]] + name = parent[name] + return name + + def unite(left: str, right: str) -> None: + left, right = find(left), find(right) + if left != right: + parent[right] = left + + for index, left in enumerate(current): + for right in current[index + 1:]: + if base(left["name"]) == base(right["name"]) or similar(left, right) >= 0.38: + unite(left["name"], right["name"]) + groups: dict[str, list[dict]] = {} + for skill in current: + groups.setdefault(find(skill["name"]), []).append(skill) + + for group in groups.values(): + if len(group) < 2: + continue + passing = [] + for skill in group: + if skill["name"] not in processed or skill["name"] in protected: + continue + if str(skill.get("audit_verdict") or "").lower() != "pass": + continue + try: + confidence = float(skill.get("confidence") or 0) + except (TypeError, ValueError): + confidence = 0.0 + if confidence >= min_conf: + passing.append(skill) + if not passing: + continue + keeper = max(passing, key=score) + if auto_publish: + try: + skills_manager.update_skill(keeper["name"], {"status": "published"}, owner=owner) + log(f"{keeper['name']}: auto-approved as best passing duplicate") + except Exception: + logger.warning("Could not auto-approve skill %s", keeper["name"], exc_info=True) + for skill in group: + name = skill["name"] + if name == keeper["name"] or name not in processed or name in protected: + continue + try: + skills_manager.update_skill(name, {"status": "binned"}, owner=owner) + log(f"{name}: moved to bin as duplicate of {keeper['name']}") + except Exception: + logger.warning("Could not bin duplicate skill %s", name, exc_info=True) + + +def audit_flag_text(*parts) -> str: + text_parts = [] + for part in parts: + if isinstance(part, dict): + text_parts.extend(str(v or "") for v in part.values()) + elif isinstance(part, (list, tuple, set)): + text_parts.extend(str(v or "") for v in part) + else: + text_parts.append(str(part or "")) + return " ".join(text_parts).lower() + + +def utility_blocker(skill: Optional[dict], necessity: Optional[dict], + verdict_data: Optional[dict]) -> Optional[str]: + """Return a short reason when a passing skill still should stay draft. + + Broad/generic wording is not a blocker by itself. The blocker is whether + the skill failed to improve the agent versus a no-skill baseline, or whether + it duplicates another skill. + """ + generic_re = re.compile( + r"\b(duplicat\w*|redundan\w*|overlap\w*|same skill|same procedure)\b", + re.I, + ) + if isinstance(necessity, dict): + reason = str(necessity.get("reason") or "") + if necessity.get("necessary") is False and generic_re.search(reason): + return reason or "Duplicate or redundant skill" + + if isinstance(verdict_data, dict): + baseline_verdict = str(verdict_data.get("baseline_verdict") or "unknown").lower() + try: + usefulness = float(verdict_data.get("usefulness", 0.0) or 0.0) + except (TypeError, ValueError): + usefulness = 0.0 + try: + saved_turns = int(verdict_data.get("saved_turns", 0) or 0) + except (TypeError, ValueError): + saved_turns = 0 + try: + saved_tool_calls = int(verdict_data.get("saved_tool_calls", 0) or 0) + except (TypeError, ValueError): + saved_tool_calls = 0 + if baseline_verdict == "worse": + return "Skill performed worse than the no-skill baseline" + verdict_text = audit_flag_text( + verdict_data.get("summary"), + verdict_data.get("issues") or [], + ) + if generic_re.search(verdict_text): + return "Audit flagged the skill as duplicate or redundant" + return None + + +def finalize_status(skills_manager, name: str, owner, verdict: str, + confidence: Optional[float], necessity: Optional[dict] = None, + verdict_data: Optional[dict] = None, *, deps: AuditDependencies) -> str: + """Apply the user's audit publishing policy. + + Audit is the final pass: skills that pass at/above the threshold are + published; failing or unnecessary/redundant skills are returned to draft. + Inconclusive runs preserve the existing state because they provide no + evidence either way. The completed batch moves duplicate losers to the bin. + """ + auto_publish, min_conf = deps.auto_publish_policy(owner) + necessary = True + current = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None) + if verdict in {"inconclusive", "unknown"}: + return (current or {}).get("status") or "draft" + utility_reason = deps.utility_blocker(current, necessity, verdict_data) + if ( + isinstance(necessity, dict) + and necessity.get("necessary") is False + and necessity.get("redundant_with") + ): + necessary = False + if utility_reason: + necessary = False + try: + skills_manager.set_necessity(name, False, [], utility_reason, owner=owner) + except Exception: + pass + duplicate_of = deps.duplicate_blocker(skills_manager, name, owner) if verdict == "pass" else None + if duplicate_of: + necessary = False + c = float(confidence or 0.0) + status = "published" if (auto_publish and necessary and verdict == "pass" and c >= min_conf) else "draft" + try: + skills_manager.update_skill(name, {"status": status}, owner=owner) + except Exception: + pass + return status + + +def apply_markdown(skills_manager, name: str, md: str, owner) -> bool: + """Parse + persist an edited SKILL.md. Returns True on success.""" + try: + from services.memory.skill_format import Skill, slugify + sk = Skill.from_markdown(md) + # Pin the identity: the audit's fixer is now allowed to edit frontmatter + # (tags/category/when_to_use/description), but it must NEVER rename the + # skill — a changed `name` would move the dir and orphan the usage/audit + # sidecar entries that the caller keeps writing under the original name. + sk.name = name + return bool(skills_manager.update_skill(name, { + "name": sk.name, "description": sk.description, "version": sk.version, + "category": sk.category, "tags": sk.tags, "platforms": sk.platforms, + "requires_toolsets": sk.requires_toolsets, "fallback_for_toolsets": sk.fallback_for_toolsets, + "status": sk.status, "confidence": sk.confidence, "source": sk.source, + "teacher_model": sk.teacher_model, "owner": sk.owner or owner, + "when_to_use": sk.when_to_use, "procedure": sk.procedure, + "pitfalls": sk.pitfalls, "verification": sk.verification, "body_extra": sk.body_extra, + }, owner=owner)) + except Exception as e: + logger.warning(f"Audit: could not save edited skill {name}: {e}") + return False diff --git a/tests/fixtures/skills_api_baseline.json b/tests/fixtures/skills_api_baseline.json new file mode 100644 index 000000000..deed1b2d9 --- /dev/null +++ b/tests/fixtures/skills_api_baseline.json @@ -0,0 +1,4429 @@ +{ + "baseline": "86b886c611338282af6d4a717246ea4e256f58df", + "routes": [ + [ + [ + "GET" + ], + "/api/skills", + "list_skills" + ], + [ + [ + "GET" + ], + "/api/skills/index", + "get_index" + ], + [ + [ + "GET" + ], + "/api/skills/slash-catalog", + "get_slash_catalog" + ], + [ + [ + "GET" + ], + "/api/skills/builtin", + "list_builtin_skills" + ], + [ + [ + "GET" + ], + "/api/skills/builtin/{name}", + "get_builtin_skill" + ], + [ + [ + "PUT" + ], + "/api/skills/builtin/{name}", + "set_builtin_override" + ], + [ + [ + "DELETE" + ], + "/api/skills/builtin/{name}", + "reset_builtin_override" + ], + [ + [ + "POST" + ], + "/api/skills/import-from-url", + "import_skill_from_url" + ], + [ + [ + "POST" + ], + "/api/skills/add", + "add_skill" + ], + [ + [ + "POST" + ], + "/api/skills/{skill_id}/invoke", + "invoke_skill" + ], + [ + [ + "GET" + ], + "/api/skills/{skill_id}", + "get_skill" + ], + [ + [ + "GET" + ], + "/api/skills/{skill_id}/markdown", + "get_skill_markdown" + ], + [ + [ + "POST" + ], + "/api/skills/{skill_id}/test", + "test_skill" + ], + [ + [ + "POST" + ], + "/api/skills/{skill_id}/test-approval", + "approve_skill_test_action" + ], + [ + [ + "GET" + ], + "/api/skills/{skill_id}/test-status", + "test_skill_status" + ], + [ + [ + "POST" + ], + "/api/skills/audit-all", + "audit_all_skills" + ], + [ + [ + "GET" + ], + "/api/skills/audit-all/status", + "audit_status" + ], + [ + [ + "POST" + ], + "/api/skills/audit-all/cancel", + "audit_cancel" + ], + [ + [ + "POST" + ], + "/api/skills/{skill_id}/markdown", + "save_skill_markdown" + ], + [ + [ + "PUT" + ], + "/api/skills/{skill_id}", + "update_skill" + ], + [ + [ + "DELETE" + ], + "/api/skills/{skill_id}", + "delete_skill" + ], + [ + [ + "POST" + ], + "/api/skills/search", + "search_skills" + ] + ], + "signatures": { + "_verdict_efficiency": { + "parameters": [ + { + "name": "verdict", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "empty": true + } + } + ], + "return": { + "type": "builtins.dict" + } + }, + "SkillAddRequest": { + "parameters": [ + { + "name": "name", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 80 + } + } + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "description", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 200 + } + } + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "category", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "type": "builtins.str" + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 40 + } + } + } + ] + }, + "default": { + "type": "builtins.str", + "value": "general" + } + }, + { + "name": "tags", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "marker": "pydantic._internal._signature._HAS_DEFAULT_FACTORY_CLASS" + } + }, + { + "name": "platforms", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "marker": "pydantic._internal._signature._HAS_DEFAULT_FACTORY_CLASS" + } + }, + { + "name": "requires_toolsets", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "marker": "pydantic._internal._signature._HAS_DEFAULT_FACTORY_CLASS" + } + }, + { + "name": "fallback_for_toolsets", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "marker": "pydantic._internal._signature._HAS_DEFAULT_FACTORY_CLASS" + } + }, + { + "name": "when_to_use", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 2000 + } + } + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "procedure", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "marker": "pydantic._internal._signature._HAS_DEFAULT_FACTORY_CLASS" + } + }, + { + "name": "pitfalls", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "marker": "pydantic._internal._signature._HAS_DEFAULT_FACTORY_CLASS" + } + }, + { + "name": "verification", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "marker": "pydantic._internal._signature._HAS_DEFAULT_FACTORY_CLASS" + } + }, + { + "name": "status", + "kind": "KEYWORD_ONLY", + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "draft" + } + }, + { + "name": "version", + "kind": "KEYWORD_ONLY", + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "1.0.0" + } + }, + { + "name": "confidence", + "kind": "KEYWORD_ONLY", + "annotation": { + "type": "builtins.float" + }, + "default": { + "type": "builtins.float", + "value": 0.8 + } + }, + { + "name": "source", + "kind": "KEYWORD_ONLY", + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "user" + } + }, + { + "name": "teacher_model", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "session_id", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "title", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 200 + } + } + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "problem", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 2000 + } + } + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "solution", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 5000 + } + } + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "steps", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "marker": "pydantic._internal._signature._HAS_DEFAULT_FACTORY_CLASS" + } + } + ], + "return": { + "type": "builtins.NoneType", + "value": null + }, + "model_fields": { + "name": { + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 80 + } + } + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "description": { + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 200 + } + } + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "category": { + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "type": "builtins.str" + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 40 + } + } + } + ] + }, + "default": { + "type": "builtins.str", + "value": "general" + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "tags": { + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "takes_validated_data": false + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "platforms": { + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "takes_validated_data": false + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "requires_toolsets": { + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "takes_validated_data": false + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "fallback_for_toolsets": { + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "takes_validated_data": false + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "when_to_use": { + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 2000 + } + } + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "procedure": { + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "takes_validated_data": false + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "pitfalls": { + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "takes_validated_data": false + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "verification": { + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "takes_validated_data": false + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "status": { + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "draft" + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "version": { + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "1.0.0" + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "confidence": { + "annotation": { + "type": "builtins.float" + }, + "default": { + "type": "builtins.float", + "value": 0.8 + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "source": { + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "user" + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "teacher_model": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "session_id": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "title": { + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 200 + } + } + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "problem": { + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 2000 + } + } + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "solution": { + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "metadata": [ + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 5000 + } + } + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "steps": { + "annotation": { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + "default": { + "factory": "builtins.list", + "takes_validated_data": false + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + } + }, + "model_config": { + "type": "builtins.dict", + "items": {} + } + }, + "SkillImportUrlRequest": { + "parameters": [ + { + "name": "url", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "type": "builtins.str" + }, + "metadata": [ + { + "type": "annotated_types.MinLen", + "fields": { + "min_length": { + "type": "builtins.int", + "value": 8 + } + } + }, + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 2000 + } + } + } + ] + }, + "default": { + "empty": true + } + } + ], + "return": { + "type": "builtins.NoneType", + "value": null + }, + "model_fields": { + "url": { + "annotation": { + "origin": "typing.Annotated", + "annotation": { + "type": "builtins.str" + }, + "metadata": [ + { + "type": "annotated_types.MinLen", + "fields": { + "min_length": { + "type": "builtins.int", + "value": 8 + } + } + }, + { + "type": "annotated_types.MaxLen", + "fields": { + "max_length": { + "type": "builtins.int", + "value": 2000 + } + } + } + ] + }, + "default": { + "empty": true + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + } + }, + "model_config": { + "type": "builtins.dict", + "items": {} + } + }, + "SkillUpdateRequest": { + "parameters": [ + { + "name": "name", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "description", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "category", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "tags", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "platforms", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "requires_toolsets", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "fallback_for_toolsets", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "when_to_use", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "procedure", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "pitfalls", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "verification", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "status", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "version", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "confidence", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.float" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "body_extra", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "title", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "problem", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "solution", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "steps", + "kind": "KEYWORD_ONLY", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + } + ], + "return": { + "type": "builtins.NoneType", + "value": null + }, + "model_fields": { + "name": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "description": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "category": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "tags": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "platforms": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "requires_toolsets": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "fallback_for_toolsets": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "when_to_use": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "procedure": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "pitfalls": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "verification": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "status": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "version": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "confidence": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.float" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "body_extra": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "title": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "problem": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "solution": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + }, + "steps": { + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "origin": "builtins.list", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + } + ] + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + }, + "validation": { + "alias": { + "type": "builtins.NoneType", + "value": null + }, + "alias_priority": { + "type": "builtins.NoneType", + "value": null + }, + "validation_alias": { + "type": "builtins.NoneType", + "value": null + }, + "discriminator": { + "type": "builtins.NoneType", + "value": null + }, + "validate_default": { + "type": "builtins.NoneType", + "value": null + } + } + } + }, + "model_config": { + "type": "builtins.dict", + "items": {} + } + }, + "_skill_test_task": { + "parameters": [ + { + "name": "skill", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.dict" + }, + "default": { + "empty": true + } + } + ], + "return": { + "type": "builtins.str" + } + }, + "_skill_baseline_task": { + "parameters": [ + { + "name": "skill", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.dict" + }, + "default": { + "empty": true + } + } + ], + "return": { + "type": "builtins.str" + } + }, + "_skill_test_messages": { + "parameters": [ + { + "name": "md", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "task", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + } + ], + "return": { + "origin": "builtins.list", + "module": "builtins", + "arguments": [ + { + "type": "builtins.dict" + } + ] + } + }, + "_skill_baseline_messages": { + "parameters": [ + { + "name": "task", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + } + ], + "return": { + "origin": "builtins.list", + "module": "builtins", + "arguments": [ + { + "type": "builtins.dict" + } + ] + } + }, + "_eval_skill_run": { + "parameters": [ + { + "name": "skill_md", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "task", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "transcript", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "url", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "model", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "headers", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "empty": true + } + }, + { + "name": "baseline_transcript", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "" + } + }, + { + "name": "skill_stats", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "baseline_stats", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "workload", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "foreground" + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + } + ], + "return": { + "type": "builtins.dict" + } + }, + "_eval_skill_necessity": { + "parameters": [ + { + "name": "skill_md", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "others", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.list" + }, + "default": { + "empty": true + } + }, + { + "name": "url", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "model", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "headers", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "empty": true + } + }, + { + "name": "workload", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "foreground" + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + } + ], + "return": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + } + }, + "_should_check_retrieval_precision": { + "parameters": [ + { + "name": "skill", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.dict" + }, + "default": { + "empty": true + } + } + ], + "return": { + "type": "builtins.bool" + } + }, + "_eval_skill_retrieval_precision": { + "parameters": [ + { + "name": "skill_md", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "others", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.list" + }, + "default": { + "empty": true + } + }, + { + "name": "url", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "model", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "headers", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "empty": true + } + }, + { + "name": "workload", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "foreground" + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + } + ], + "return": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + } + }, + "_run_skill_test_job": { + "parameters": [ + { + "name": "key", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "name", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "md", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "task", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "url", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "model", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "headers", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "skills_manager", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "messages", + "kind": "KEYWORD_ONLY", + "annotation": { + "empty": true + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "transcript", + "kind": "KEYWORD_ONLY", + "annotation": { + "empty": true + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "exact_approval", + "kind": "KEYWORD_ONLY", + "annotation": { + "empty": true + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "request_authority", + "kind": "KEYWORD_ONLY", + "annotation": { + "empty": true + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + } + ], + "return": { + "empty": true + } + }, + "_audit_auto_publish_policy": { + "parameters": [ + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + } + ], + "return": { + "origin": "builtins.tuple", + "module": "builtins", + "arguments": [ + { + "type": "builtins.bool" + }, + { + "type": "builtins.float" + } + ] + } + }, + "_skill_duplicate_blocker": { + "parameters": [ + { + "name": "skills_manager", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "name", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + } + ], + "return": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + } + }, + "_finalize_audit_batch": { + "parameters": [ + { + "name": "skills_manager", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "results", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "builtins.list", + "module": "builtins", + "arguments": [ + { + "type": "builtins.dict" + } + ] + }, + "default": { + "empty": true + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "log", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + } + ], + "return": { + "type": "builtins.NoneType", + "value": null + } + }, + "_audit_flag_text": { + "parameters": [ + { + "name": "parts", + "kind": "VAR_POSITIONAL", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + } + ], + "return": { + "type": "builtins.str" + } + }, + "_audit_utility_blocker": { + "parameters": [ + { + "name": "skill", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "empty": true + } + }, + { + "name": "necessity", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "empty": true + } + }, + { + "name": "verdict_data", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "empty": true + } + } + ], + "return": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + } + }, + "_audit_finalize_status": { + "parameters": [ + { + "name": "skills_manager", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "name", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "verdict", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "confidence", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.float" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "empty": true + } + }, + { + "name": "necessity", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "verdict_data", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + } + ], + "return": { + "type": "builtins.str" + } + }, + "_apply_skill_md": { + "parameters": [ + { + "name": "skills_manager", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "name", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "md", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + } + ], + "return": { + "type": "builtins.bool" + } + }, + "_run_skill_audit_arm": { + "parameters": [ + { + "name": "messages", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "builtins.list", + "module": "builtins", + "arguments": [ + { + "type": "builtins.dict" + } + ] + }, + "default": { + "empty": true + } + }, + { + "name": "url", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "model", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "headers", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "workload", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "foreground" + } + } + ], + "return": { + "origin": "builtins.tuple", + "module": "builtins", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.dict" + }, + { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.dict" + }, + { + "type": "builtins.NoneType" + } + ] + } + ] + } + }, + "_run_skill_test_once": { + "parameters": [ + { + "name": "md", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "task", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "url", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "model", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "headers", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "workload", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "foreground" + } + } + ], + "return": { + "type": "builtins.tuple" + } + }, + "_improve_skill_md": { + "parameters": [ + { + "name": "skill_md", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "verdict", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.dict" + }, + "default": { + "empty": true + } + }, + { + "name": "transcript", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "empty": true + } + }, + { + "name": "url", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "model", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "headers", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "workload", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "foreground" + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + } + ], + "return": { + "empty": true + } + }, + "_audit_one_skill": { + "parameters": [ + { + "name": "skills_manager", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "skill", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "url", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "model", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "headers", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "teacher", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "log", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "workload", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "foreground" + } + } + ], + "return": { + "type": "builtins.dict" + } + }, + "_run_audit_all_job": { + "parameters": [ + { + "name": "key", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "skills_manager", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "names", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "url", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "model", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "headers", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "teacher", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "empty": true + } + }, + { + "name": "workload", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.str" + }, + "default": { + "type": "builtins.str", + "value": "foreground" + } + } + ], + "return": { + "empty": true + } + }, + "_resolve_audit_models": { + "parameters": [ + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "model_spec", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "endpoint_url", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "empty": true + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + } + ], + "return": { + "empty": true + } + }, + "run_scheduled_skill_audit": { + "parameters": [ + { + "name": "skills_manager", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "services.memory.skills.SkillsManager" + }, + "default": { + "empty": true + } + }, + { + "name": "owner", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "origin": "typing.Union", + "module": "typing", + "arguments": [ + { + "type": "builtins.str" + }, + { + "type": "builtins.NoneType" + } + ] + }, + "default": { + "type": "builtins.NoneType", + "value": null + } + }, + { + "name": "max_skills", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "builtins.int" + }, + "default": { + "type": "builtins.int", + "value": 8 + } + } + ], + "return": { + "type": "builtins.dict" + } + }, + "setup_skills_routes": { + "parameters": [ + { + "name": "skills_manager", + "kind": "POSITIONAL_OR_KEYWORD", + "annotation": { + "type": "services.memory.skills.SkillsManager" + }, + "default": { + "empty": true + } + } + ], + "return": { + "type": "fastapi.routing.APIRouter" + } + } + } +} diff --git a/tests/fixtures/skills_openapi_baseline.json b/tests/fixtures/skills_openapi_baseline.json new file mode 100644 index 000000000..e34aa171f --- /dev/null +++ b/tests/fixtures/skills_openapi_baseline.json @@ -0,0 +1,1226 @@ +{ + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "SkillAddRequest": { + "properties": { + "category": { + "default": "general", + "maxLength": 40, + "title": "Category", + "type": "string" + }, + "confidence": { + "default": 0.8, + "title": "Confidence", + "type": "number" + }, + "description": { + "anyOf": [ + { + "maxLength": 200, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "fallback_for_toolsets": { + "items": { + "type": "string" + }, + "title": "Fallback For Toolsets", + "type": "array" + }, + "name": { + "anyOf": [ + { + "maxLength": 80, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "pitfalls": { + "items": { + "type": "string" + }, + "title": "Pitfalls", + "type": "array" + }, + "platforms": { + "items": { + "type": "string" + }, + "title": "Platforms", + "type": "array" + }, + "problem": { + "anyOf": [ + { + "maxLength": 2000, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Problem" + }, + "procedure": { + "items": { + "type": "string" + }, + "title": "Procedure", + "type": "array" + }, + "requires_toolsets": { + "items": { + "type": "string" + }, + "title": "Requires Toolsets", + "type": "array" + }, + "session_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Session Id" + }, + "solution": { + "anyOf": [ + { + "maxLength": 5000, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Solution" + }, + "source": { + "default": "user", + "title": "Source", + "type": "string" + }, + "status": { + "default": "draft", + "title": "Status", + "type": "string" + }, + "steps": { + "items": { + "type": "string" + }, + "title": "Steps", + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "teacher_model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Teacher Model" + }, + "title": { + "anyOf": [ + { + "maxLength": 200, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "verification": { + "items": { + "type": "string" + }, + "title": "Verification", + "type": "array" + }, + "version": { + "default": "1.0.0", + "title": "Version", + "type": "string" + }, + "when_to_use": { + "anyOf": [ + { + "maxLength": 2000, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "When To Use" + } + }, + "title": "SkillAddRequest", + "type": "object" + }, + "SkillImportUrlRequest": { + "properties": { + "url": { + "maxLength": 2000, + "minLength": 8, + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "SkillImportUrlRequest", + "type": "object" + }, + "SkillUpdateRequest": { + "properties": { + "body_extra": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Body Extra" + }, + "category": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Category" + }, + "confidence": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "fallback_for_toolsets": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Fallback For Toolsets" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "pitfalls": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Pitfalls" + }, + "platforms": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Platforms" + }, + "problem": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Problem" + }, + "procedure": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Procedure" + }, + "requires_toolsets": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Requires Toolsets" + }, + "solution": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Solution" + }, + "status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Status" + }, + "steps": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Steps" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tags" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "verification": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Verification" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + }, + "when_to_use": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "When To Use" + } + }, + "title": "SkillUpdateRequest", + "type": "object" + }, + "ValidationError": { + "properties": { + "ctx": { + "title": "Context", + "type": "object" + }, + "input": { + "title": "Input" + }, + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "info": { + "title": "FastAPI", + "version": "0.1.0" + }, + "openapi": "3.1.0", + "paths": { + "/api/skills": { + "get": { + "operationId": "list_skills_api_skills_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "List Skills", + "tags": [ + "skills" + ] + } + }, + "/api/skills/add": { + "post": { + "operationId": "add_skill_api_skills_add_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillAddRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Add Skill", + "tags": [ + "skills" + ] + } + }, + "/api/skills/audit-all": { + "post": { + "description": "Kick off a background audit of skills: each is tested + judged; if it\nneeds work the model self-edits and retries; if a teacher model is\nconfigured it escalates; a skill that still fails is demoted to draft\n(never deleted). Poll GET /audit-status. Body:\n{scope: 'drafts'|'unchecked'|'all', names?: [...], skip_audited?: bool}. Default 'all'\nmeans every visible skill, including already-published skills, so audit\ncan publish or demote according to the confidence threshold.", + "operationId": "audit_all_skills_api_skills_audit_all_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Audit All Skills", + "tags": [ + "skills" + ] + } + }, + "/api/skills/audit-all/cancel": { + "post": { + "operationId": "audit_cancel_api_skills_audit_all_cancel_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Audit Cancel", + "tags": [ + "skills" + ] + } + }, + "/api/skills/audit-all/status": { + "get": { + "operationId": "audit_status_api_skills_audit_all_status_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Audit Status", + "tags": [ + "skills" + ] + } + }, + "/api/skills/builtin": { + "get": { + "description": "Read-only list of the agent's built-in tool capabilities (research,\nsessions, tasks, email, etc.) \u2014 the things it natively knows how to do.\nSurfaced so the Skills tab can show them in a separate \"Built-in\"\nsection alongside the user's learned SKILL.md skills. Sourced from\nagent_loop.TOOL_SECTIONS (the same descriptions the model is given).", + "operationId": "list_builtin_skills_api_skills_builtin_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "List Builtin Skills", + "tags": [ + "skills" + ] + } + }, + "/api/skills/builtin/{name}": { + "delete": { + "description": "Revert a built-in tool to its shipped instruction block.", + "operationId": "reset_builtin_override_api_skills_builtin__name__delete", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Reset Builtin Override", + "tags": [ + "skills" + ] + }, + "get": { + "description": "Full text of a built-in tool's instruction block \u2014 the override\nif one is set, plus the shipped default (for the revert button).", + "operationId": "get_builtin_skill_api_skills_builtin__name__get", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Builtin Skill", + "tags": [ + "skills" + ] + }, + "put": { + "description": "Save a user override for a built-in tool's instruction block.\nWARNING surfaced in the UI \u2014 this changes how the assistant is\ntold to use a native tool.", + "operationId": "set_builtin_override_api_skills_builtin__name__put", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "title": "Name", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Set Builtin Override", + "tags": [ + "skills" + ] + } + }, + "/api/skills/import-from-url": { + "post": { + "description": "Install a SKILL.md bundle from a public GitHub URL (skills.sh links supported).", + "operationId": "import_skill_from_url_api_skills_import_from_url_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillImportUrlRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Import Skill From Url", + "tags": [ + "skills" + ] + } + }, + "/api/skills/index": { + "get": { + "description": "The lightweight `[{name, description, category}]` list that the\nagent's system prompt sees. Useful for the UI's \"what does the model\nactually have access to?\" view.", + "operationId": "get_index_api_skills_index_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Get Index", + "tags": [ + "skills" + ] + } + }, + "/api/skills/search": { + "post": { + "operationId": "search_skills_api_skills_search_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Search Skills", + "tags": [ + "skills" + ] + } + }, + "/api/skills/slash-catalog": { + "get": { + "description": "Return skills that are available as slash commands.\n\nMirrors the agent prompt's published-skill index so the UI never offers\na slash command the model would not normally be allowed to discover.", + "operationId": "get_slash_catalog_api_skills_slash_catalog_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Get Slash Catalog", + "tags": [ + "skills" + ] + } + }, + "/api/skills/{skill_id}": { + "delete": { + "operationId": "delete_skill_api_skills__skill_id__delete", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Delete Skill", + "tags": [ + "skills" + ] + }, + "get": { + "operationId": "get_skill_api_skills__skill_id__get", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Skill", + "tags": [ + "skills" + ] + }, + "put": { + "operationId": "update_skill_api_skills__skill_id__put", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Update Skill", + "tags": [ + "skills" + ] + } + }, + "/api/skills/{skill_id}/invoke": { + "post": { + "description": "Build a skill-pinned prompt for slash-command invocation.\n\nThis is intentionally server-side so availability, ownership, and usage\naccounting use the same rules as the SkillsManager.", + "operationId": "invoke_skill_api_skills__skill_id__invoke_post", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Invoke Skill", + "tags": [ + "skills" + ] + } + }, + "/api/skills/{skill_id}/markdown": { + "get": { + "description": "Return the raw SKILL.md text \u2014 used by the slash-invocation flow\nand the editor's 'view source' affordance.", + "operationId": "get_skill_markdown_api_skills__skill_id__markdown_get", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Skill Markdown", + "tags": [ + "skills" + ] + }, + "post": { + "description": "Replace SKILL.md with new raw content. Parses + validates first.", + "operationId": "save_skill_markdown_api_skills__skill_id__markdown_post", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Save Skill Markdown", + "tags": [ + "skills" + ] + } + }, + "/api/skills/{skill_id}/test": { + "post": { + "description": "Kick off a background skill test (agent run + LLM judge). Returns\nimmediately; the run executes server-side so it survives the modal being\nclosed. Poll GET /{skill_id}/test-status for progress + verdict.\nOn completion it records the verdict and nudges the skill's confidence\nto match (pass\u21920.95, needs_work\u21920.6, fail\u21920.4; inconclusive/unknown leave\nit untouched). It never changes the skill's published/draft STATUS.", + "operationId": "test_skill_api_skills__skill_id__test_post", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Test Skill", + "tags": [ + "skills" + ] + } + }, + "/api/skills/{skill_id}/test-approval": { + "post": { + "description": "Resume a manual skill test with one exact server-sealed action.", + "operationId": "approve_skill_test_action_api_skills__skill_id__test_approval_post", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Approve Skill Test Action", + "tags": [ + "skills" + ] + } + }, + "/api/skills/{skill_id}/test-status": { + "get": { + "description": "Current background-test state for a skill (status / log / verdict).", + "operationId": "test_skill_status_api_skills__skill_id__test_status_get", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Test Skill Status", + "tags": [ + "skills" + ] + } + } + } +} diff --git a/tests/test_redos_residual_python.py b/tests/test_redos_residual_python.py index 276c71059..8ee889ab5 100644 --- a/tests/test_redos_residual_python.py +++ b/tests/test_redos_residual_python.py @@ -26,7 +26,7 @@ def _fast(fn, *args): return result -# -- closed blocks (skills_routes x4, memory_extractor.audit_memories) -- +# -- closed blocks (skills.evaluation x4, memory_extractor.audit_memories) -- _THINK_REF = re.compile(r"[\s\S]*?", re.I) @@ -51,7 +51,11 @@ def test_strip_closed_think_blocks_unclosed_opener_flood_is_linear(): def test_lazy_think_regex_is_gone_from_llm_review_parsers(): - for rel in ("routes/skills_routes.py", "services/memory/memory_extractor.py"): + for rel in ( + "routes/skills_routes.py", + "services/skills/evaluation.py", + "services/memory/memory_extractor.py", + ): source = (_ROOT / rel).read_text(encoding="utf-8") assert r"[\s\S]*?" not in source, rel diff --git a/tests/test_redos_verdict_continuation.py b/tests/test_redos_verdict_continuation.py index 1304f0200..685a4dfa6 100644 --- a/tests/test_redos_verdict_continuation.py +++ b/tests/test_redos_verdict_continuation.py @@ -3,7 +3,7 @@ Both had two adjacent `\\s`-matching quantifiers that backtrack O(n^2) when the rest of the pattern fails on a whitespace flood: - * `routes/skills_routes.py` `_VERDICT_PROSE_RE` — `["\\'\\s:]*\\s*` (the class + * `services/skills/evaluation.py` `_VERDICT_PROSE_RE`: `["\\'\\s:]*\\s*` (the class already matches `\\s`) over a teacher/verifier model's prose verdict. * `src/agent_loop.py` `_EXPLICIT_CONTINUATION_RE` — `\\s*[.!?]*\\s*$` over a user's terse reply. @@ -18,7 +18,7 @@ import time import pytest import src.agent_tools # noqa: F401 (break agent_tools<->agent_loop import cycle) -from routes.skills_routes import _VERDICT_PROSE_RE +from services.skills.evaluation import _VERDICT_PROSE_RE from src.agent_loop import _EXPLICIT_CONTINUATION_RE, _is_explicit_continuation _BUDGET_S = 4.0 diff --git a/tests/test_skills_audit_characterization.py b/tests/test_skills_audit_characterization.py new file mode 100644 index 000000000..14d942ff1 --- /dev/null +++ b/tests/test_skills_audit_characterization.py @@ -0,0 +1,485 @@ +"""Offline contracts captured against the merged skills baseline before extraction.""" +import asyncio +from dataclasses import fields, is_dataclass +from difflib import unified_diff +import inspect +import json +from contextvars import ContextVar +from pathlib import Path +from typing import Annotated, get_args, get_origin + +import pytest +from fastapi import FastAPI +from starlette.requests import Request + +from routes import skills_routes as routes + + +FIXTURES = Path(__file__).parent / 'fixtures' + + +def _qualified_name(value): + return f'{value.__module__}.{value.__qualname__}' + + +def _value_contract(value): + if value is inspect.Signature.empty: + return {'empty': True} + if value is None or type(value) in (bool, int, float, str): + return {'type': _qualified_name(type(value)), 'value': value} + if type(value) in (list, tuple): + return {'type': _qualified_name(type(value)), + 'items': [_value_contract(item) for item in value]} + if type(value) is dict and all(type(key) is str for key in value): + return {'type': 'builtins.dict', + 'items': {key: _value_contract(item) for key, item in value.items()}} + if is_dataclass(value) and not isinstance(value, type): + return {'type': _qualified_name(type(value)), 'fields': { + field.name: _value_contract(getattr(value, field.name)) + for field in fields(value) + }} + raise TypeError(f'Unsupported contract value: {type(value)}') + + +def _annotation_contract(annotation): + if annotation is inspect.Signature.empty or annotation is None: + return _value_contract(annotation) + origin = get_origin(annotation) + if origin is Annotated: + base, *metadata = get_args(annotation) + return {'origin': _qualified_name(origin), + 'annotation': _annotation_contract(base), + 'metadata': [_value_contract(item) for item in metadata]} + if origin is not None: + return {'origin': _qualified_name(origin), 'module': annotation.__module__, + 'arguments': [_annotation_contract(arg) for arg in get_args(annotation)]} + return {'type': _qualified_name(annotation)} + + +def _factory_contract(factory): + # Names do not identify closure state, partial arguments or arbitrary code. + # Only these known builtin constructors have a supported default contract. + if not any(factory is builtin for builtin in (list, tuple, dict, set, frozenset)): + raise TypeError('Unsupported default factory; requires an explicit compatibility contract') + return _qualified_name(factory) + + +def _signature_contract(target): + """Freeze supported constructor structure without executing default factories.""" + signature = inspect.signature(target) + model_fields = getattr(target, 'model_fields', {}) + parameters = [] + for parameter in signature.parameters.values(): + field = model_fields.get(parameter.name) + if field is not None and field.default_factory is not None: + default = {'factory': _factory_contract(field.default_factory), + 'marker': _qualified_name(type(parameter.default))} + else: + default = _value_contract(parameter.default) + parameters.append({'name': parameter.name, 'kind': parameter.kind.name, + 'annotation': _annotation_contract(parameter.annotation), + 'default': default}) + contract = {'parameters': parameters, + 'return': _annotation_contract(signature.return_annotation)} + if model_fields: + # Field names can be absent from the Python signature (validation aliases). + # These public FieldInfo attributes affect input validation independently + # of the generated signature and JSON schema. Rebuilt annotations retain + # field metadata, including constraints and strict/union validation modes. + contract['model_fields'] = { + name: { + 'annotation': _annotation_contract(field.rebuild_annotation()), + 'default': ({'factory': _factory_contract(field.default_factory), + 'takes_validated_data': field.default_factory_takes_validated_data} + if field.default_factory is not None else + _value_contract(inspect.Signature.empty if field.is_required() else field.default)), + 'validation': {attribute: _value_contract(getattr(field, attribute)) + for attribute in ('alias', 'alias_priority', 'validation_alias', + 'discriminator', 'validate_default')}, + } + for name, field in model_fields.items() + } + # Includes inherited model settings such as validate_default and strict. + # Unsupported callables or values fail closed through _value_contract. + contract['model_config'] = _value_contract(target.model_config) + return contract + + +def _canonical_openapi(schema): + return json.dumps(schema, sort_keys=True, indent=2, ensure_ascii=True) + '\n' + + +def _assert_openapi_matches_baseline(schema): + expected = (FIXTURES / 'skills_openapi_baseline.json').read_text(encoding='utf-8') + actual = _canonical_openapi(schema) + if actual != expected: + baseline = json.loads(expected) + affected = [] + for label, before, after in [ + ('paths', baseline.get('paths', {}), schema.get('paths', {})), + ('schemas', baseline.get('components', {}).get('schemas', {}), + schema.get('components', {}).get('schemas', {})), + ]: + changed = [key for key in sorted(before.keys() | after.keys()) + if _canonical_openapi(before.get(key)) != _canonical_openapi(after.get(key))] + if changed: + affected.append(f'Changed {label}: {", ".join(changed)}\n') + pytest.fail('OpenAPI contract changed:\n' + ''.join(affected) + ''.join(unified_diff( + expected.splitlines(keepends=True), actual.splitlines(keepends=True), + fromfile='skills_openapi_baseline.json', tofile='generated OpenAPI', + ))) + + +def endpoint(method, path, manager=None): + return next(r.endpoint for r in routes.setup_skills_routes(manager).routes + if method in r.methods and r.path == '/api/skills' + path) + + +def request(owner='alice', body=None): + async def receive(): + return {'type': 'http.request', 'body': json.dumps(body or {}).encode()} + return Request({'type': 'http', 'method': 'POST', 'path': '/', + 'headers': [(b'content-type', b'application/json')], + 'state': {'user': owner}}, receive) + + +def test_api_and_python_signatures_match_baseline(): + baseline = json.loads((FIXTURES / 'skills_api_baseline.json').read_text(encoding='utf-8')) + app = FastAPI() + router = routes.setup_skills_routes(object()) + app.include_router(router) + schema = app.openapi() + _assert_openapi_matches_baseline(schema) + declarations = [[sorted(r.methods), r.path, r.name] for r in router.routes] + assert declarations == baseline['routes'] + for name, signature in baseline['signatures'].items(): + assert _signature_contract(getattr(routes, name)) == signature, name + + +def test_signature_contract_ignores_annotated_repr(monkeypatch): + target = routes.SkillImportUrlRequest + before = _signature_contract(target) + annotation = inspect.signature(target).parameters['url'].annotation + monkeypatch.setattr(type(annotation), '__repr__', lambda self: 'different Python rendering') + assert _signature_contract(target) == before + + +@pytest.mark.parametrize('change', ['name', 'order', 'kind', 'default', 'annotation', 'return']) +def test_signature_contract_detects_interface_changes(change): + def original(first: str, second: int = 1) -> bool: + pass + signature = inspect.signature(original) + first, second = signature.parameters.values() + parameters = [first, second] + if change == 'order': + parameters = [second.replace(default=inspect.Parameter.empty), + first.replace(default='')] + elif change == 'return': + signature = signature.replace(return_annotation=str) + else: + changes = { + 'name': {'name': 'renamed'}, + 'kind': {'kind': inspect.Parameter.KEYWORD_ONLY}, + 'default': {'default': 2}, + 'annotation': {'annotation': float}, + } + parameters[1] = second.replace(**changes[change]) + def changed(): + pass + changed.__signature__ = signature.replace(parameters=parameters) + assert _signature_contract(changed) != _signature_contract(original) + + +@pytest.mark.parametrize('change', ['constraint', 'factory']) +def test_signature_contract_detects_model_changes(change): + from pydantic import BaseModel, Field + + class Original(BaseModel): + text: str = Field(max_length=8) + tags: list[str] = Field(default_factory=list) + + class Changed(BaseModel): + text: str = Field(max_length=9 if change == 'constraint' else 8) + tags: list[str] = Field(default_factory=tuple if change == 'factory' else list) + + assert _signature_contract(Changed) != _signature_contract(Original) + + +@pytest.mark.parametrize('validated_data', [False, True]) +def test_signature_contract_rejects_custom_factories_without_calling_them(validated_data): + from pydantic import BaseModel, Field + + calls = [] + def make_factory(value): + def plain(): + calls.append(value) + return [value] + def with_data(data): + calls.append(value) + return [data['prefix'], value] + return with_data if validated_data else plain + + class Before(BaseModel): + prefix: str = 'validated' + tags: list[str] = Field(default_factory=make_factory('before')) + + class After(BaseModel): + prefix: str = 'validated' + tags: list[str] = Field(default_factory=make_factory('after')) + + assert Before().tags != After().tags + assert Before.model_fields['tags'].default_factory_takes_validated_data is validated_data + calls.clear() + for model in (Before, After): + with pytest.raises(TypeError, match='Unsupported default factory'): + _signature_contract(model) + assert calls == [] + + +def test_signature_contract_detects_default_validation_outside_signature_and_openapi(): + from pydantic import BaseModel, Field, ValidationError + + class Before(BaseModel): + count: int = Field(default='invalid', validate_default=False) + + class After(BaseModel): + count: int = Field(default='invalid', validate_default=True) + + assert Before().count == 'invalid' + with pytest.raises(ValidationError): + After() + assert inspect.signature(Before) == inspect.signature(After) + before_schema, after_schema = Before.model_json_schema(), After.model_json_schema() + before_schema.pop('title') + after_schema.pop('title') + assert before_schema == after_schema + assert _signature_contract(Before) != _signature_contract(After) + + +def test_signature_contract_detects_validation_alias_paths(): + from pydantic import AliasPath, BaseModel, Field, ValidationError + + class Before(BaseModel): + count: int = Field(validation_alias=AliasPath('before', 0)) + + class After(BaseModel): + count: int = Field(validation_alias=AliasPath('after', 0)) + + assert Before(before=[1]).count == 1 + with pytest.raises(ValidationError): + After(before=[1]) + assert inspect.signature(Before) == inspect.signature(After) + before_schema, after_schema = Before.model_json_schema(), After.model_json_schema() + before_schema.pop('title') + after_schema.pop('title') + assert before_schema == after_schema + assert _signature_contract(Before) != _signature_contract(After) + + +@pytest.mark.parametrize('setting', ['validate_default', 'strict']) +def test_signature_contract_detects_model_validation_configuration(setting): + from pydantic import BaseModel, ConfigDict, ValidationError + + class Before(BaseModel): + model_config = ConfigDict({setting: False}) + count: int = 'invalid' if setting == 'validate_default' else 1 + + class After(BaseModel): + model_config = ConfigDict({setting: True}) + count: int = 'invalid' if setting == 'validate_default' else 1 + + data = {} if setting == 'validate_default' else {'count': '1'} + Before(**data) + with pytest.raises(ValidationError): + After(**data) + assert inspect.signature(Before) == inspect.signature(After) + assert _signature_contract(Before) != _signature_contract(After) + + +def test_openapi_golden_is_canonical(): + golden = (FIXTURES / 'skills_openapi_baseline.json').read_text(encoding='utf-8') + schema = json.loads(golden) + assert golden == _canonical_openapi(schema) + + +@pytest.mark.parametrize('section, key', [ + ('paths', '/api/skills/add'), ('schemas', 'SkillAddRequest'), +]) +def test_openapi_failure_identifies_changed_endpoint_or_schema(section, key): + schema = json.loads((FIXTURES / 'skills_openapi_baseline.json').read_text(encoding='utf-8')) + target = schema['paths'] if section == 'paths' else schema['components']['schemas'] + if section == 'paths': + target[key]['post']['responses']['200']['description'] = 'changed contract' + else: + target[key]['properties']['solution']['anyOf'][0]['maxLength'] = 4999 + with pytest.raises(pytest.fail.Exception) as failure: + _assert_openapi_matches_baseline(schema) + diagnostic = str(failure.value) + assert '--- skills_openapi_baseline.json' in diagnostic + assert '+++ generated OpenAPI' in diagnostic + assert f'Changed {section}: {key}' in diagnostic + expected_change = '"description": "changed contract"' if section == 'paths' else '"maxLength": 4999' + assert any(line.startswith('+') and expected_change in line + for line in diagnostic.splitlines()) + + +@pytest.mark.parametrize('raw, expected', [ + ('{"verdict":"fail"}{"verdict":"PASS","confidence":2,}', 'pass'), + ('{"verdict":"fail"} text {"verdict":"needs_work"}', 'needs_work'), + ('Verdict: INCONCLUSIVE', 'inconclusive'), + ('{"verdict":"pass"}', 'unknown'), + ('{"verdict":"invalid"}', 'unknown'), +]) +async def test_run_verdict_parsing_and_retry(monkeypatch, raw, expected): + calls = [] + async def call(url, model, messages, **kwargs): + calls.append((messages, kwargs)) + return raw + monkeypatch.setattr('src.llm_core.llm_call_async', call) + monkeypatch.setattr(routes, 'effort_for_call', lambda url, model, owner: 'high' if owner == 'alice' else None) + result = await routes._eval_skill_run('md', 'task', 'transcript', 'url', 'm', {}, + workload='background', owner='alice') + assert result['verdict'] == expected + assert len(calls) == (2 if expected == 'unknown' else 1) + for _, kwargs in calls: + assert kwargs['reasoning_effort'] == 'high' + assert kwargs['workload'] == 'background' + assert kwargs['timeout'] == 180 + assert kwargs['max_tokens'] == 32768 + if len(calls) == 2: + assert "must START with '{'" in calls[1][0][0]['content'] + + +async def test_review_transient_failure_and_result_coercion(monkeypatch): + calls = [] + async def call(*args, **kwargs): + calls.append(kwargs) + if len(calls) == 1: + raise TimeoutError('offline') + return json.dumps({'verdict': 'pass', 'confidence': '-1', 'summary': 'x' * 500, + 'issues': ['y' * 250] * 10, 'baseline_verdict': 'invalid', + 'usefulness': '2', 'saved_turns': '-2', 'saved_tool_calls': 'bad'}) + monkeypatch.setattr('src.llm_core.llm_call_async', call) + result = await routes._eval_skill_run('', '', '', 'url', 'm', None) + assert result == {'verdict': 'pass', 'confidence': 0.0, 'summary': 'x' * 400, + 'issues': ['y' * 200] * 8, 'baseline_verdict': 'unknown', + 'usefulness': 1.0, 'saved_turns': -2, 'saved_tool_calls': 0} + + +async def test_review_failure_schema_and_transcript_tail(monkeypatch): + seen = [] + async def call(url, model, messages, **kwargs): + seen.append(messages) + raise RuntimeError('offline') + monkeypatch.setattr('src.llm_core.llm_call_async', call) + result = await routes._eval_skill_run('m' * 5000, 'TASK', 'HEAD' + 'x' * 30000 + 'TAIL', 'url', 'm', None) + assert result == {'verdict': 'unknown', 'confidence': 0, + 'summary': 'Review call failed: offline', 'issues': []} + assert len(seen) == 2 + assert 'HEAD' in seen[0][1]['content'] and 'TAIL' in seen[0][1]['content'] + assert 'm' * 4001 not in seen[0][1]['content'] + + +@pytest.mark.parametrize('name, raw, expected, tokens, timeout', [ + ('_eval_skill_necessity', 'x{"necessary":false,"redundant_with":["demo"],"reason":"duplicate",}', + {'necessary': False, 'redundant_with': ['demo'], 'reason': 'duplicate'}, 8192, 120), + ('_eval_skill_retrieval_precision', '{"ok":false,"summary":"broad","issues":["metadata: retrieval: narrow"]}', + {'ok': False, 'summary': 'broad', 'issues': ['metadata: retrieval: narrow']}, 4096, 90), + ('_eval_skill_necessity', 'unfinished', None, 8192, 120), + ('_eval_skill_retrieval_precision', '[]', None, 4096, 90), +]) +async def test_advisory_schema_budget_and_owner(monkeypatch, name, raw, expected, tokens, timeout): + async def call(*args, **kwargs): + assert kwargs['max_tokens'] == tokens + assert kwargs['timeout'] == timeout + assert kwargs['reasoning_effort'] == 'medium' + return raw + monkeypatch.setattr('src.llm_core.llm_call_async', call) + monkeypatch.setattr(routes, 'effort_for_call', lambda url, model, owner: 'medium' if owner == 'alice' else None) + assert await getattr(routes, name)('md', [], 'url', 'm', None, owner='alice') == expected + + +@pytest.mark.parametrize('name', ['_eval_skill_necessity', '_eval_skill_retrieval_precision', '_improve_skill_md']) +async def test_advisory_exceptions_return_none(monkeypatch, name): + async def call(*args, **kwargs): + raise TimeoutError('offline') + monkeypatch.setattr('src.llm_core.llm_call_async', call) + args = ('md', {}, '', 'url', 'm', None) if name == '_improve_skill_md' else ('md', [], 'url', 'm', None) + assert await getattr(routes, name)(*args) is None + + +async def test_cancelled_audit_closes_stream_in_originating_context(monkeypatch): + bound = ContextVar('audit_characterization', default=None) + opened = asyncio.Event() + closed = [] + async def stream(*args, **kwargs): + token = bound.set('bound') + try: + yield 'data: {"delta":"started"}\n\n' + opened.set() + await asyncio.Event().wait() + finally: + bound.reset(token) + closed.append(asyncio.current_task()) + monkeypatch.setattr('src.agent_loop.stream_agent_loop', stream) + task = asyncio.create_task(routes._run_skill_audit_arm([], 'url', 'm', {}, 'alice')) + await opened.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert closed == [task] + assert bound.get() is None + + +async def test_cancelled_batch_skips_finalization_and_cleans_task(monkeypatch): + opened = asyncio.Event() + async def audit(*args, **kwargs): + opened.set() + await asyncio.Event().wait() + class Manager: + def load(self, owner=None): + assert owner == 'alice' + return [{'name': 'demo'}] + job = {'status': 'running', 'log': [], 'results': [], 'done': 0} + monkeypatch.setattr(routes, '_skill_audit_jobs', {('alice',): job}) + monkeypatch.setattr(routes, '_audit_one_skill', audit) + monkeypatch.setattr(routes, '_finalize_audit_batch', lambda *args: pytest.fail('cancelled batch finalized')) + task = asyncio.create_task(routes._run_audit_all_job(('alice',), Manager(), ['demo'], 'url', 'm', {}, None, 'alice')) + job['task'] = task + await opened.wait() + task.cancel() + await task + assert job['status'] == 'cancelled' + assert job['current'] is None and 'task' not in job and 'finished' in job + assert job['results'] == [] and job['done'] == 0 + + +async def test_test_polling_is_owner_scoped_and_hides_continuation(monkeypatch): + class Manager: + def load(self, owner=None): + return [{'name': 'demo'}] + monkeypatch.setattr(routes, 'get_current_user', lambda req: req.state.user) + job = {'status': 'awaiting_approval', 'task': 'task', 'model': 'm', 'log': [], + 'verdict': None, 'approval': {'approval_id': 'id'}, + '_run': {'headers': {'Authorization': 'secret'}}, '_transcript': ['private']} + monkeypatch.setattr(routes, '_skill_test_jobs', {('alice', 'demo'): job}) + poll = endpoint('GET', '/{skill_id}/test-status', Manager()) + assert await poll(request('bob'), 'demo') == {'status': 'none'} + assert await poll(request(), 'demo') == {k: job[k] for k in ('status', 'task', 'model', 'log', 'verdict', 'approval')} + + +def test_batch_finalizer_protects_approval_required_and_unprocessed_skills(monkeypatch): + class Manager: + def __init__(self): + self.updated = [] + def load(self, owner=None): + assert owner == 'alice' + return [{'name': name, 'status': 'published', 'source': 'learned', + 'audit_verdict': 'fail', 'confidence': .9} for name in ('demo', 'demo-1', 'demo-2')] + def update_skill(self, name, fields, owner=None): + self.updated.append((name, fields, owner)) + monkeypatch.setattr(routes, '_audit_auto_publish_policy', lambda owner: (True, .8)) + manager = Manager() + routes._finalize_audit_batch(manager, [{'skill': 'demo', 'result': 'approval_required'}, + {'skill': 'demo-1', 'result': 'flagged'}], 'alice', lambda msg: None) + assert manager.updated == [('demo-1', {'status': 'draft'}, 'alice')] diff --git a/tests/test_skills_audit_services.py b/tests/test_skills_audit_services.py new file mode 100644 index 000000000..1aae77a4e --- /dev/null +++ b/tests/test_skills_audit_services.py @@ -0,0 +1,402 @@ +"""Service and HTTP integration contracts without live model or skill state.""" +import asyncio +from dataclasses import replace + +import pytest +from fastapi import HTTPException + +from routes import skills_routes as routes +from services.skills import ( + agent_runner, audit_workflow, evaluation, job_coordinator, model_selection, publication, +) +from src.agent_runtime.authority import RequestAuthority, bind_request_authority +from src.tool_approvals import ToolApprovalStore +from src.tool_capabilities import capabilities_for_action +from tests.test_skills_audit_characterization import endpoint, request + + +@pytest.fixture +def prompt_execution(monkeypatch): + """Capture actual agent ingress, keeping runners and coordination real.""" + from copy import deepcopy + + class Manager: + def __init__(self): + self.row = {'name': 'demo', 'owner': 'alice', 'source': 'learned', + 'status': 'draft', 'confidence': .5, 'description': 'fixture'} + def load(self, owner=None): + assert owner == 'alice' + return [dict(self.row)] + def read_skill_md(self, name, owner=None): + assert (name, owner) == ('demo', 'alice') + return 'private skill <<>>' + def update_skill(self, name, fields, owner=None): + assert (name, owner) == ('demo', 'alice') + self.row.update(fields) + return True + def set_audit(self, name, verdict, **kwargs): + assert (name, kwargs['owner']) == ('demo', 'alice') + self.row['audit_verdict'] = verdict + + seen, grades = [], [] + async def stream(url, model, messages, **kwargs): + seen.append((deepcopy(messages), kwargs)) + yield 'data: {"delta":"executed"}\n\n' + async def evaluate(*args, **kwargs): + return {'verdict': grades.pop(0) if grades else 'pass', 'issues': []} + async def necessity(*args, **kwargs): + return None + monkeypatch.setattr('src.agent_loop.stream_agent_loop', stream) + monkeypatch.setattr(routes, '_eval_skill_run', evaluate) + monkeypatch.setattr(routes, '_eval_skill_necessity', necessity) + monkeypatch.setattr(routes, '_should_check_retrieval_precision', lambda *a: False) + monkeypatch.setattr(routes, '_skill_duplicate_blocker', lambda *a: None) + monkeypatch.setattr(routes, '_audit_auto_publish_policy', lambda owner: (False, .85)) + monkeypatch.setattr(routes, '_skill_test_jobs', {}) + monkeypatch.setattr(routes, '_skill_audit_jobs', {}) + monkeypatch.setattr(routes, 'get_current_user', lambda req: req.state.user) + monkeypatch.setattr('src.endpoint_resolver.resolve_endpoint', lambda *a, **kw: ('url', 'worker', {})) + monkeypatch.setattr('src.llm_core.list_model_ids', lambda *a, **kw: ['worker']) + monkeypatch.setattr(routes, '_resolve_audit_models', lambda **kw: ('url', 'worker', {}, None)) + return Manager(), seen, grades + + +def patch_prompt_builders(monkeypatch): + originals = routes._skill_test_messages, routes._skill_baseline_messages + def test_messages(md, task): + return originals[0]('patched skill: ' + md, task) + def baseline_messages(task): + return originals[1]('patched baseline: ' + task) + monkeypatch.setattr(routes, '_skill_test_task', lambda skill: 'patched task') + monkeypatch.setattr(routes, '_skill_test_messages', test_messages) + monkeypatch.setattr(routes, '_skill_baseline_messages', baseline_messages) + + +@pytest.mark.parametrize('patched', [False, True]) +@pytest.mark.parametrize('entry', ['manual', 'audit', 'scheduled']) +async def test_route_prompt_builders_reach_agent_execution(prompt_execution, monkeypatch, entry, patched): + manager, seen, _ = prompt_execution + if patched: + patch_prompt_builders(monkeypatch) + expected_task = routes._skill_test_task(manager.row) + if entry == 'manual': + response = await endpoint('POST', '/{skill_id}/test', manager)(request(), 'demo') + assert response['status'] == 'running' + # The stub stream and judge never suspend, so one loop turn completes the job. + await asyncio.sleep(0) + assert routes._skill_test_jobs[('alice', 'demo')]['status'] == 'done' + elif entry == 'audit': + result = await routes._audit_one_skill(manager, manager.row, 'url', 'worker', {}, None, + 'alice', lambda msg: None) + assert result['result'] == 'pass' + else: + result = await routes.run_scheduled_skill_audit(manager, owner='alice') + assert result['total'] == 1 and result['results'][0]['result'] == 'pass' + assert all(kwargs['workload'] == 'background' for _, kwargs in seen) + assert [messages for messages, _ in seen] == [ + routes._skill_test_messages(manager.read_skill_md('demo', owner='alice'), expected_task), + routes._skill_baseline_messages(expected_task), + ] + tainted = seen[0][0][1] + assert tainted['metadata']['trusted'] is False + assert tainted['metadata']['tool_gate_untrusted'] is True + assert '<<<_END_UNTRUSTED_DATA>>>' in tainted['content'] + assert all(kwargs['request_authority'].owner == 'alice' for _, kwargs in seen) + + +async def test_route_prompt_patches_reach_self_and_teacher_retries(prompt_execution, monkeypatch): + manager, seen, grades = prompt_execution + patch_prompt_builders(monkeypatch) + grades.extend(['fail', 'fail', 'pass']) + rewrites = iter(['self rewrite', 'teacher rewrite']) + async def improve(*args, **kwargs): + return next(rewrites) + monkeypatch.setattr(routes, '_improve_skill_md', improve) + monkeypatch.setattr(routes, '_apply_skill_md', lambda *args: True) + result = await routes._audit_one_skill(manager, manager.row, 'url', 'worker', {}, + ('teacher-url', 'teacher', {}), 'alice', lambda msg: None) + assert result['result'] == 'pass_after_teacher' + assert [messages for messages, _ in seen] == [ + routes._skill_test_messages(manager.read_skill_md('demo', owner='alice'), 'patched task'), + routes._skill_baseline_messages('patched task'), + routes._skill_test_messages('self rewrite', 'patched task'), + routes._skill_test_messages('teacher rewrite', 'patched task'), + ] + + +@pytest.mark.parametrize('patched', [False, True]) +async def test_approval_continuation_prompt_reaches_agent(approval_route, prompt_execution, monkeypatch, patched): + approve, store, pending, job = approval_route + # prompt_execution supplies fresh registries; restore this pending job. + routes._skill_test_jobs[('alice', 'demo')] = job + _, seen, _ = prompt_execution + if patched: + patch_prompt_builders(monkeypatch) + expected = routes._skill_test_messages('untrusted skill', 'task') + result = await approve(request(body={'approval_id': pending.approval_id, 'decision': 'approve'}), 'demo') + assert result['status'] == 'running' + await asyncio.sleep(0) + assert job['status'] == 'done' + assert seen[0][0][:len(expected)] == expected + transcript = seen[0][0][len(expected)] + assert 'untrusted result' in transcript['content'] + assert transcript['metadata']['trusted'] is False + assert transcript['metadata']['tool_gate_untrusted'] is True + grant = seen[0][1]['exact_approval'] + assert grant.pending is pending and grant.allow_remaining_actions is False + assert store.peek(pending.approval_id) is None + assert seen[1][0] == routes._skill_baseline_messages('task') + + +def test_baseline_task_alias_uses_route_test_task_patch(monkeypatch): + monkeypatch.setattr(routes, '_skill_test_task', lambda skill: 'patched shared task') + assert routes._skill_baseline_task({'name': 'demo'}) == 'patched shared task' + + +async def test_evaluation_can_inject_model_call_and_owner_effort_without_network(): + seen = [] + async def call(url, model, messages, **kwargs): + seen.append(kwargs) + return '{"verdict":"pass"}' + def effort(url, model, owner): + assert (url, model, owner) == ('offline', 'worker', 'alice') + return 'high' + result = await evaluation.evaluate_run('md', 'task', 'done', 'offline', 'worker', None, + owner='alice', _llm_call=call, _effort_for_call=effort) + assert result['verdict'] == 'pass' + assert seen[0]['reasoning_effort'] == 'high' + + +async def test_rewrite_keeps_last_document_and_owner_effort(): + async def call(*args, **kwargs): + assert kwargs['reasoning_effort'] == 'medium' + return 'reasoningold\n---\nname: old\n---\nbody\n---\nname: demo\n---\nfixed' + assert await evaluation.improve_markdown('md', {}, 'transcript', 'url', 'm', {}, + owner='alice', _llm_call=call, _effort_for_call=lambda u, m, o: 'medium') == '---\nname: demo\n---\nfixed' + + +def test_audit_model_resolution_uses_first_owner_scoped_utility_fallback(monkeypatch): + seen = [] + def primary(role, owner=None): + seen.append(('primary', role, owner)) + return None, None, {} + def fallback(owner=None): + seen.append(('fallback', owner)) + return [('first', '/models/worker', {'Authorization': 'owner-key'}), ('second', 'other', {})] + def served(url, headers=None): + assert (url, headers) == ('first', {'Authorization': 'owner-key'}) + return ['worker'] + monkeypatch.setattr('src.endpoint_resolver.resolve_endpoint', primary) + monkeypatch.setattr('src.endpoint_resolver.resolve_utility_fallback_candidates', fallback) + monkeypatch.setattr('src.llm_core.list_model_ids', served) + monkeypatch.setattr('src.settings.get_setting', lambda key, default=None: default) + assert model_selection.resolve_audit_models(owner='alice') == ('first', 'worker', {'Authorization': 'owner-key'}, None) + assert seen == [('primary', 'utility', 'alice'), ('fallback', 'alice')] + + +async def test_audit_stream_failure_closes_and_restores_inherited_authority(monkeypatch): + authority = RequestAuthority.empty(owner='alice') + closed = [] + async def stream(*args, **kwargs): + assert kwargs['owner'] == 'alice' + assert kwargs['request_authority'] is authority + assert kwargs['suppress_skills'] is True + try: + yield 'event: error\ndata: {"error":"offline"}\n\n' + finally: + closed.append(asyncio.current_task()) + monkeypatch.setattr('src.agent_loop.stream_agent_loop', stream) + with bind_request_authority(authority): + with pytest.raises(agent_runner.SkillAuditUnavailable, match='offline'): + await agent_runner.run_audit_arm([], 'url', 'm', {}, 'alice') + assert closed == [asyncio.current_task()] + + +async def test_manual_exception_closes_stream_and_finishes_without_publishing(monkeypatch): + closed = [] + async def stream(*args, **kwargs): + try: + yield 'data: {"delta":"partial"}\n\n' + raise RuntimeError('stream failed') + finally: + closed.append(asyncio.current_task()) + async def arm(*args, **kwargs): + return 'baseline', {}, None + async def evaluate(*args, **kwargs): + return {'verdict': 'fail', 'summary': 'failed', 'issues': []} + class Manager: + def __init__(self): + self.updates = [] + def set_audit(self, name, verdict, **kwargs): + assert name == 'demo' and verdict == 'fail' and kwargs['owner'] == 'alice' + def update_skill(self, name, fields, owner=None): + self.updates.append((fields, owner)) + job = {'status': 'running', 'log': [], '_run': {}, '_transcript': []} + deps = replace(routes._audit_dependencies(), test_jobs={('alice', 'demo'): job}, + run_arm=arm, evaluate_run=evaluate) + monkeypatch.setattr('src.agent_loop.stream_agent_loop', stream) + manager = Manager() + await agent_runner.run_test_job(('alice', 'demo'), 'demo', 'md', 'task', 'url', 'm', {}, + 'alice', manager, deps=deps) + assert closed == [asyncio.current_task()] + assert job['status'] == 'done' and '_run' not in job and '_transcript' not in job + assert {'type': 'error', 'error': 'stream failed'} in job['log'] + assert manager.updates == [({'confidence': .4}, 'alice')] + + +async def test_batch_exception_records_progress_and_finalizes_once(): + class Manager: + def __init__(self): + self.audit = [] + def load(self, owner=None): + assert owner == 'alice' + return [{'name': name, 'status': 'draft'} for name in ('first', 'second')] + def set_audit(self, *args, **kwargs): + self.audit.append((args, kwargs)) + async def audit(manager, skill, *args, **kwargs): + if skill['name'] == 'first': + raise RuntimeError('failed') + return {'skill': 'second', 'result': 'pass'} + finalized = [] + job = {'status': 'running', 'log': [], 'results': [], 'done': 0, 'task': object()} + deps = replace(routes._audit_dependencies(), audit_jobs={('alice',): job}, audit_one=audit, + finalize_batch=lambda manager, results, owner, log: finalized.append((list(results), owner))) + manager = Manager() + await audit_workflow.run_audit_batch(('alice',), manager, ['first', 'second'], 'url', 'm', {}, None, 'alice', deps=deps) + assert job['status'] == 'done' and job['done'] == 2 and job['current'] is None + assert 'task' not in job and 'finished' in job + assert [r['result'] for r in job['results']] == ['error', 'pass'] + assert manager.audit == [(('first', 'inconclusive'), {'worker_model': 'm', 'owner': 'alice'})] + assert finalized == [(job['results'], 'alice')] + + +@pytest.mark.parametrize('enabled,confidence,verdict,expected', [ + (True, .85, 'pass', 'published'), (True, .84, 'pass', 'draft'), + (False, .95, 'pass', 'draft'), (True, .95, 'fail', 'draft'), + (False, .2, 'unknown', 'published'), +]) +def test_policy_preserves_threshold_and_auto_approval(enabled, confidence, verdict, expected): + class Manager: + def __init__(self): + self.updates = [] + def load(self, owner=None): + return [{'name': 'demo', 'status': 'published'}] + def update_skill(self, name, fields, owner=None): + assert owner == 'alice' + self.updates.append(fields) + deps = replace(routes._audit_dependencies(), auto_publish_policy=lambda owner: (enabled, .85), + duplicate_blocker=lambda *args: None) + manager = Manager() + assert publication.finalize_status(manager, 'demo', 'alice', verdict, confidence, deps=deps) == expected + assert manager.updates == ([] if verdict == 'unknown' else [{'status': expected}]) + + +@pytest.mark.parametrize('enabled', [True, False]) +def test_batch_duplicate_finalization_changes_only_processed_unprotected_skills(enabled): + class Manager: + def __init__(self): + self.updates = [] + def load(self, owner=None): + assert owner == 'alice' + return [{'name': name, 'source': 'learned', 'status': 'draft', + 'audit_verdict': 'pass', 'confidence': .95, 'uses': uses} + for name, uses in [('demo', 10), ('demo-1', 0), ('demo-2', 20), ('demo-3', 30)]] + def update_skill(self, name, fields, owner=None): + self.updates.append((name, fields, owner)) + deps = replace(routes._audit_dependencies(), auto_publish_policy=lambda owner: (enabled, .85)) + manager = Manager() + publication.finalize_batch(manager, [{'skill': 'demo', 'result': 'pass'}, + {'skill': 'demo-1', 'result': 'pass'}, + {'skill': 'demo-3', 'result': 'approval_required'}], + 'alice', lambda message: None, deps=deps) + expected = [('demo', {'status': 'published'}, 'alice')] if enabled else [] + assert manager.updates == expected + [('demo-1', {'status': 'binned'}, 'alice')] + + +@pytest.fixture +def approval_route(monkeypatch): + store = ToolApprovalStore() + monkeypatch.setattr('src.tool_approvals.tool_approval_store', store) + monkeypatch.setattr(routes, 'get_current_user', lambda req: req.state.user) + monkeypatch.setattr(routes, '_skill_test_jobs', {}) + class Manager: + def load(self, owner=None): + return [{'name': 'demo', 'owner': owner}] + pending = store.create(owner='alice', session_id=None, origin_run_id='run', tool_name='bash', + content='printf approved', workspace=None, external_untrusted_context_seen=True, + capabilities=capabilities_for_action('bash', 'printf approved')) + job = {'status': 'awaiting_approval', 'log': [], 'task': 'task', + 'approval': pending.public_payload(), '_transcript': ['untrusted result'], + '_run': {'owner': 'alice', 'md': 'untrusted skill', 'url': 'url', 'model': 'm'}} + routes._skill_test_jobs[('alice', 'demo')] = job + return endpoint('POST', '/{skill_id}/test-approval', Manager()), store, pending, job + + +async def test_denied_skill_approval_retires_state_without_spawning(approval_route, monkeypatch): + approve, store, pending, job = approval_route + async def run(*args, **kwargs): + pytest.fail('denial spawned a test') + monkeypatch.setattr(routes, '_run_skill_test_job', run) + result = await approve(request(body={'approval_id': pending.approval_id, 'decision': 'deny'}), 'demo') + assert result == {'ok': True, 'status': 'done', 'decision': 'deny'} + assert store.peek(pending.approval_id) is None + assert job['verdict']['verdict'] == 'inconclusive' + assert not {'approval', '_run', '_transcript'} & job.keys() + + +@pytest.mark.parametrize('case,status', [('other_owner', 409), ('mismatch', 409), ('consumed', 409), ('invalid_decision', 400)]) +async def test_invalid_skill_approval_cannot_resume(approval_route, monkeypatch, case, status): + approve, store, pending, job = approval_route + async def run(*args, **kwargs): + pytest.fail('invalid approval resumed a test') + monkeypatch.setattr(routes, '_run_skill_test_job', run) + owner = 'bob' if case == 'other_owner' else 'alice' + if case == 'other_owner': + routes._skill_test_jobs[('bob', 'demo')] = job + if case == 'consumed': + store.consume(pending.approval_id, decision='deny', owner='alice', session_id=None) + body = {'approval_id': 'wrong' if case == 'mismatch' else pending.approval_id, + 'decision': 'approve_task' if case == 'invalid_decision' else 'approve'} + with pytest.raises(HTTPException) as error: + await approve(request(owner, body), 'demo') + assert error.value.status_code == status + assert job['status'] == 'awaiting_approval' + + +async def test_allowed_skill_approval_is_single_action_and_cannot_be_reused(approval_route, monkeypatch): + approve, store, pending, job = approval_route + resumed = [] + async def run(*args, **kwargs): + resumed.append((args, kwargs)) + monkeypatch.setattr(routes, '_run_skill_test_job', run) + body = {'approval_id': pending.approval_id, 'decision': 'approve'} + assert await approve(request(body=body), 'demo') == {'ok': True, 'status': 'running', 'decision': 'approve'} + await asyncio.sleep(0) + assert len(resumed) == 1 and resumed[0][0][7] == 'alice' + grant = resumed[0][1]['exact_approval'] + assert grant.pending is pending and grant.allow_remaining_actions is False + assert grant.grants_chat_session is False + assert store.peek(pending.approval_id) is None + messages = resumed[0][1]['messages'] + tainted = [m for m in messages if m.get('metadata', {}).get('tool_gate_untrusted')] + assert len(tainted) == 2 + assert any('untrusted result' in m['content'] for m in tainted) + with pytest.raises(HTTPException) as error: + await approve(request(body=body), 'demo') + assert error.value.status_code == 409 + assert len(resumed) == 1 + + +@pytest.mark.parametrize('scope,selected,skip,names,expected_scope', [ + ('all', None, False, ['draft', 'published', 'checked'], 'all'), + ('drafts', None, False, ['draft'], 'unchecked'), + ('all', ['published', 'draft', 'draft', 'builtin', 'missing'], False, ['published', 'draft'], 'selected'), + ('all', None, True, ['draft', 'published'], 'all'), +]) +def test_manual_audit_selection_preserves_published_and_builtin_rules(scope, selected, skip, names, expected_scope): + class Manager: + def load(self, owner=None): + assert owner == 'alice' + return [{'name': 'draft', 'status': 'draft'}, {'name': 'published', 'status': 'published'}, + {'name': 'checked', 'audit_verdict': 'pass'}, {'name': 'builtin', 'source': 'builtin'}] + assert job_coordinator.select_audit_names(Manager(), 'alice', scope, selected, skip) == (names, expected_scope)