mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-05 14:32:20 +02:00
docs(discovery): publish provisional feature baseline
This commit is contained in:
Alexandre Teixeira
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Focused negative tests for validate_discovery_evidence.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MODULE_PATH = Path(__file__).with_name("validate_discovery_evidence.py")
|
||||
SPEC = importlib.util.spec_from_file_location("validate_discovery_evidence", MODULE_PATH)
|
||||
assert SPEC and SPEC.loader
|
||||
validator = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = validator
|
||||
SPEC.loader.exec_module(validator)
|
||||
|
||||
|
||||
class FakeJavascriptParser:
|
||||
supported = True
|
||||
reason = "test parser"
|
||||
|
||||
def __init__(self, symbols: list[validator.Located] | None = None) -> None:
|
||||
self.symbols = symbols or []
|
||||
|
||||
def parse(self, path: Path) -> list[validator.Located]:
|
||||
return self.symbols
|
||||
|
||||
|
||||
class UnsupportedJavascriptParser:
|
||||
supported = False
|
||||
reason = "no repository-local parser"
|
||||
|
||||
|
||||
class EvidenceNegativeTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temp.name)
|
||||
(self.root / "sample.py").write_text(
|
||||
"from fastapi import APIRouter\n"
|
||||
"router = APIRouter(prefix='/api')\n"
|
||||
"\n"
|
||||
"class ChatHandler:\n"
|
||||
" def preprocess_message(self):\n"
|
||||
" return True\n"
|
||||
"\n"
|
||||
"@router.post('/chat')\n"
|
||||
"def chat_stream():\n"
|
||||
" return True\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(self.root / "sample.js").write_text(
|
||||
"export const present = () => true;\n", encoding="utf-8"
|
||||
)
|
||||
(self.root / "sample.sh").write_text(
|
||||
"#!/usr/bin/env bash\nreal_function() {\n return 0\n}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
self.backup = self.root / "copied-backups"
|
||||
self.backup.mkdir()
|
||||
for path in self.root.glob("sample.*"):
|
||||
shutil.copy2(path, self.backup / path.name)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
for backup in self.backup.iterdir():
|
||||
target = self.root / backup.name
|
||||
shutil.copy2(backup, target)
|
||||
self.assertEqual(target.read_bytes(), backup.read_bytes())
|
||||
self.temp.cleanup()
|
||||
|
||||
def validate(
|
||||
self,
|
||||
evidence: dict[str, str],
|
||||
javascript_parser: object | None = None,
|
||||
) -> validator.Validation:
|
||||
return validator.validate_evidence(
|
||||
self.root,
|
||||
"TEST-001",
|
||||
0,
|
||||
evidence,
|
||||
javascript_parser or FakeJavascriptParser(),
|
||||
)
|
||||
|
||||
def test_missing_python_symbol(self) -> None:
|
||||
result = self.validate(
|
||||
{
|
||||
"path": "sample.py",
|
||||
"kind": "python-function",
|
||||
"locator": "fabricated",
|
||||
"line_range": "L1-L1",
|
||||
"explanation": "negative fixture",
|
||||
}
|
||||
)
|
||||
self.assertEqual(result.result, "invalid-locator")
|
||||
|
||||
def test_incorrect_qualified_method(self) -> None:
|
||||
result = self.validate(
|
||||
{
|
||||
"path": "sample.py",
|
||||
"kind": "python-method",
|
||||
"locator": "WrongHandler.preprocess_message",
|
||||
"line_range": "L5-L6",
|
||||
"explanation": "negative fixture",
|
||||
}
|
||||
)
|
||||
self.assertEqual(result.result, "invalid-locator")
|
||||
|
||||
def test_symbol_outside_cited_range(self) -> None:
|
||||
result = self.validate(
|
||||
{
|
||||
"path": "sample.py",
|
||||
"kind": "python-method",
|
||||
"locator": "ChatHandler.preprocess_message",
|
||||
"line_range": "L1-L2",
|
||||
"explanation": "negative fixture",
|
||||
}
|
||||
)
|
||||
self.assertEqual(result.result, "locator-outside-range")
|
||||
|
||||
def test_fabricated_test_function(self) -> None:
|
||||
result = self.validate(
|
||||
{
|
||||
"path": "sample.py",
|
||||
"kind": "test-function",
|
||||
"locator": "test_fabricated",
|
||||
"line_range": "L1-L2",
|
||||
"explanation": "negative fixture",
|
||||
}
|
||||
)
|
||||
self.assertEqual(result.result, "invalid-locator")
|
||||
|
||||
def test_nonexistent_javascript_symbol_with_parser(self) -> None:
|
||||
result = self.validate(
|
||||
{
|
||||
"path": "sample.js",
|
||||
"kind": "javascript-function",
|
||||
"locator": "missing",
|
||||
"line_range": "L1-L1",
|
||||
"explanation": "negative fixture",
|
||||
},
|
||||
FakeJavascriptParser(
|
||||
[validator.Located("present", "javascript-export", 1, 1)]
|
||||
),
|
||||
)
|
||||
self.assertEqual(result.result, "invalid-locator")
|
||||
|
||||
def test_unsupported_javascript_parser(self) -> None:
|
||||
result = self.validate(
|
||||
{
|
||||
"path": "sample.js",
|
||||
"kind": "javascript-function",
|
||||
"locator": "present",
|
||||
"line_range": "L1-L1",
|
||||
"explanation": "negative fixture",
|
||||
},
|
||||
UnsupportedJavascriptParser(),
|
||||
)
|
||||
self.assertEqual(result.result, "unsupported")
|
||||
|
||||
def test_route_path_mismatch(self) -> None:
|
||||
result = self.validate(
|
||||
{
|
||||
"path": "sample.py",
|
||||
"kind": "python-route",
|
||||
"locator": "POST /api/wrong -> chat_stream",
|
||||
"line_range": "L9-L10",
|
||||
"explanation": "negative fixture",
|
||||
}
|
||||
)
|
||||
self.assertEqual(result.result, "invalid-locator")
|
||||
self.assertIn("path", result.problem or "")
|
||||
|
||||
def test_http_method_mismatch(self) -> None:
|
||||
result = self.validate(
|
||||
{
|
||||
"path": "sample.py",
|
||||
"kind": "python-route",
|
||||
"locator": "GET /api/chat -> chat_stream",
|
||||
"line_range": "L9-L10",
|
||||
"explanation": "negative fixture",
|
||||
}
|
||||
)
|
||||
self.assertEqual(result.result, "invalid-locator")
|
||||
self.assertIn("method", result.problem or "")
|
||||
|
||||
def test_shell_function_mismatch(self) -> None:
|
||||
result = self.validate(
|
||||
{
|
||||
"path": "sample.sh",
|
||||
"kind": "shell-function",
|
||||
"locator": "fabricated",
|
||||
"line_range": "L1-L4",
|
||||
"explanation": "negative fixture",
|
||||
}
|
||||
)
|
||||
self.assertEqual(result.result, "invalid-locator")
|
||||
|
||||
def test_invalid_file_level_evidence(self) -> None:
|
||||
result = self.validate(
|
||||
{
|
||||
"path": "missing.file",
|
||||
"kind": "file",
|
||||
"explanation": "negative fixture",
|
||||
}
|
||||
)
|
||||
self.assertEqual(result.result, "invalid-path")
|
||||
|
||||
def test_file_level_evidence_rejects_fake_symbol(self) -> None:
|
||||
result = self.validate(
|
||||
{
|
||||
"path": "sample.sh",
|
||||
"kind": "file",
|
||||
"locator": "whole-script",
|
||||
"explanation": "negative fixture",
|
||||
}
|
||||
)
|
||||
self.assertEqual(result.result, "invalid-locator")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+592
@@ -0,0 +1,592 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CATALOG_PATH = ROOT / "feature-catalog.json"
|
||||
CATALOG_MD_PATH = ROOT / "feature-catalog.md"
|
||||
DOMAINS_DIR = ROOT / "domains"
|
||||
REVIEWS_DIR = ROOT / "reviews"
|
||||
|
||||
EXPECTED_COMMIT = "d8a2059df8e53bc7275c45339849d14c8651e73c"
|
||||
EXPECTED_FEATURES = 79
|
||||
EXPECTED_DOMAINS = 16
|
||||
|
||||
VALID_STATUSES = {
|
||||
"verified",
|
||||
"partial",
|
||||
"disabled",
|
||||
"experimental",
|
||||
"legacy",
|
||||
"dead-code-candidate",
|
||||
"unverified",
|
||||
}
|
||||
|
||||
VALID_MATURITY = {"E0", "E1", "E2", "E3", "E4"}
|
||||
|
||||
VALID_RUNTIME = {
|
||||
"not-required",
|
||||
"pending",
|
||||
"blocked",
|
||||
"completed",
|
||||
}
|
||||
|
||||
REQUIRED_FIELDS = {
|
||||
"id",
|
||||
"domain",
|
||||
"name",
|
||||
"purpose",
|
||||
"status",
|
||||
"evidence_maturity",
|
||||
"verified_at_commit",
|
||||
"evidence",
|
||||
"runtime_validation",
|
||||
}
|
||||
|
||||
FEATURE_ID_RE = re.compile(r"^[A-Z][A-Z0-9]*-\d{3}$")
|
||||
|
||||
DOMAIN_HEADING_RE = re.compile(
|
||||
r"^##\s+`?([A-Z][A-Z0-9]*-\d{3})`?"
|
||||
r"\s+(?:—|-)\s+(.+?)\s*$"
|
||||
)
|
||||
|
||||
DOMAIN_FIELD_RE = re.compile(
|
||||
r"^-\s+\*\*"
|
||||
r"(Domain|Status|Evidence Maturity|Commit Verified)"
|
||||
r"\*\*:\s*(.*?)\s*$"
|
||||
)
|
||||
|
||||
DOMAIN_RUNTIME_RE = re.compile(
|
||||
r"^-\s+\*\*Runtime Validation\*\*:\s*(.*?)\s*$"
|
||||
)
|
||||
|
||||
REVIEW_HEADING_RE = re.compile(
|
||||
r"^###\s+(?:\d+\.\s+)?"
|
||||
r"([A-Z][A-Z0-9]*-\d{3})"
|
||||
r"\s+(?:—|-)\s+.+$"
|
||||
)
|
||||
|
||||
REVIEW_RESULT_RE = re.compile(
|
||||
r"^-\s+\*\*Resulting Status(?:\s+and|/)\s+Maturity\*\*:"
|
||||
r"\s*`([^`]+)`\s*/\s*`([^`]+)`\s*$"
|
||||
)
|
||||
|
||||
|
||||
def clean(value: str) -> str:
|
||||
value = value.strip()
|
||||
|
||||
if (
|
||||
len(value) >= 2
|
||||
and value.startswith("`")
|
||||
and value.endswith("`")
|
||||
):
|
||||
return value[1:-1].strip()
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def load_catalog(errors: list[str]) -> list[dict[str, Any]]:
|
||||
try:
|
||||
data = json.loads(
|
||||
CATALOG_PATH.read_text(encoding="utf-8")
|
||||
)
|
||||
except Exception as exc:
|
||||
errors.append(f"Unable to read catalog: {exc}")
|
||||
return []
|
||||
|
||||
if isinstance(data, list):
|
||||
features = data
|
||||
elif isinstance(data, dict) and isinstance(data.get("features"), list):
|
||||
features = data["features"]
|
||||
else:
|
||||
errors.append(
|
||||
"Catalog must be an array or contain a features array"
|
||||
)
|
||||
return []
|
||||
|
||||
if not all(isinstance(feature, dict) for feature in features):
|
||||
errors.append("Every catalog feature must be an object")
|
||||
return []
|
||||
|
||||
return features
|
||||
|
||||
|
||||
def validate_catalog(
|
||||
features: list[dict[str, Any]],
|
||||
errors: list[str],
|
||||
) -> None:
|
||||
if len(features) != EXPECTED_FEATURES:
|
||||
errors.append(
|
||||
f"Expected {EXPECTED_FEATURES} features, "
|
||||
f"found {len(features)}"
|
||||
)
|
||||
|
||||
ids = [feature.get("id") for feature in features]
|
||||
|
||||
duplicates = sorted(
|
||||
feature_id
|
||||
for feature_id, count in Counter(ids).items()
|
||||
if feature_id and count > 1
|
||||
)
|
||||
|
||||
if duplicates:
|
||||
errors.append(
|
||||
"Duplicate feature IDs: " + ", ".join(duplicates)
|
||||
)
|
||||
|
||||
for index, feature in enumerate(features):
|
||||
feature_id = feature.get("id")
|
||||
label = (
|
||||
feature_id
|
||||
if isinstance(feature_id, str)
|
||||
else f"<index:{index}>"
|
||||
)
|
||||
|
||||
missing = sorted(
|
||||
field
|
||||
for field in REQUIRED_FIELDS
|
||||
if feature.get(field) in (None, "", [])
|
||||
)
|
||||
|
||||
if missing:
|
||||
errors.append(
|
||||
f"{label}: missing fields: {', '.join(missing)}"
|
||||
)
|
||||
|
||||
if (
|
||||
not isinstance(feature_id, str)
|
||||
or not FEATURE_ID_RE.fullmatch(feature_id)
|
||||
):
|
||||
errors.append(f"{label}: invalid feature ID")
|
||||
|
||||
if feature.get("status") not in VALID_STATUSES:
|
||||
errors.append(
|
||||
f"{label}: invalid status "
|
||||
f"{feature.get('status')!r}"
|
||||
)
|
||||
|
||||
if feature.get("evidence_maturity") not in VALID_MATURITY:
|
||||
errors.append(
|
||||
f"{label}: invalid maturity "
|
||||
f"{feature.get('evidence_maturity')!r}"
|
||||
)
|
||||
|
||||
if feature.get("verified_at_commit") != EXPECTED_COMMIT:
|
||||
errors.append(
|
||||
f"{label}: incorrect verified_at_commit"
|
||||
)
|
||||
|
||||
runtime = feature.get("runtime_validation")
|
||||
|
||||
if not isinstance(runtime, dict):
|
||||
errors.append(
|
||||
f"{label}: runtime_validation must be an object"
|
||||
)
|
||||
continue
|
||||
|
||||
required = runtime.get("required")
|
||||
runtime_status = runtime.get("status")
|
||||
reason = runtime.get("reason")
|
||||
|
||||
if not isinstance(required, bool):
|
||||
errors.append(
|
||||
f"{label}: runtime required must be boolean"
|
||||
)
|
||||
|
||||
if runtime_status not in VALID_RUNTIME:
|
||||
errors.append(
|
||||
f"{label}: invalid runtime status "
|
||||
f"{runtime_status!r}"
|
||||
)
|
||||
|
||||
if not isinstance(reason, str) or not reason.strip():
|
||||
errors.append(
|
||||
f"{label}: runtime reason is blank"
|
||||
)
|
||||
|
||||
if required is False and runtime_status != "not-required":
|
||||
errors.append(
|
||||
f"{label}: required=false requires not-required"
|
||||
)
|
||||
|
||||
if required is True and runtime_status == "not-required":
|
||||
errors.append(
|
||||
f"{label}: required=true cannot be not-required"
|
||||
)
|
||||
|
||||
|
||||
def parse_domain(
|
||||
path: Path,
|
||||
errors: list[str],
|
||||
) -> dict[str, dict[str, str]]:
|
||||
records: dict[str, dict[str, str]] = {}
|
||||
current_id: str | None = None
|
||||
|
||||
for line_number, line in enumerate(
|
||||
path.read_text(encoding="utf-8").splitlines(),
|
||||
start=1,
|
||||
):
|
||||
heading = DOMAIN_HEADING_RE.match(line)
|
||||
|
||||
if heading:
|
||||
current_id = heading.group(1)
|
||||
|
||||
if current_id in records:
|
||||
errors.append(
|
||||
f"{path.relative_to(ROOT)}:{line_number}: "
|
||||
f"duplicate heading {current_id}"
|
||||
)
|
||||
|
||||
records[current_id] = {
|
||||
"Name": heading.group(2).strip(),
|
||||
}
|
||||
continue
|
||||
|
||||
field = DOMAIN_FIELD_RE.match(line)
|
||||
|
||||
if field and current_id:
|
||||
value = clean(field.group(2))
|
||||
|
||||
if not value:
|
||||
errors.append(
|
||||
f"{path.relative_to(ROOT)}:{line_number}: "
|
||||
f"blank {field.group(1)}"
|
||||
)
|
||||
|
||||
records[current_id][field.group(1)] = value
|
||||
continue
|
||||
|
||||
runtime = DOMAIN_RUNTIME_RE.match(line)
|
||||
|
||||
if runtime and current_id:
|
||||
value = clean(runtime.group(1))
|
||||
runtime_status = re.split(
|
||||
r"\s+(?:—|-)\s+",
|
||||
value,
|
||||
maxsplit=1,
|
||||
)[0]
|
||||
records[current_id]["Runtime Validation"] = (
|
||||
runtime_status.strip("` ")
|
||||
)
|
||||
|
||||
return records
|
||||
|
||||
|
||||
def validate_domains(
|
||||
features: list[dict[str, Any]],
|
||||
errors: list[str],
|
||||
) -> None:
|
||||
catalog = {
|
||||
feature["id"]: feature
|
||||
for feature in features
|
||||
if feature.get("id")
|
||||
}
|
||||
|
||||
expected_by_domain: defaultdict[str, set[str]] = defaultdict(set)
|
||||
|
||||
for feature in features:
|
||||
expected_by_domain[feature["domain"]].add(feature["id"])
|
||||
|
||||
paths = sorted(DOMAINS_DIR.glob("*.md"))
|
||||
|
||||
if len(paths) != EXPECTED_DOMAINS:
|
||||
errors.append(
|
||||
f"Expected {EXPECTED_DOMAINS} domain files, "
|
||||
f"found {len(paths)}"
|
||||
)
|
||||
|
||||
all_found: set[str] = set()
|
||||
|
||||
for path in paths:
|
||||
domain = path.stem
|
||||
records = parse_domain(path, errors)
|
||||
found = set(records)
|
||||
expected = expected_by_domain.get(domain, set())
|
||||
all_found.update(found)
|
||||
|
||||
if found != expected:
|
||||
missing = sorted(expected - found)
|
||||
extra = sorted(found - expected)
|
||||
|
||||
errors.append(
|
||||
f"{domain}: missing={missing}, unexpected={extra}"
|
||||
)
|
||||
|
||||
for feature_id, record in records.items():
|
||||
feature = catalog.get(feature_id)
|
||||
|
||||
if feature is None:
|
||||
continue
|
||||
|
||||
expected_values = {
|
||||
"Name": feature["name"],
|
||||
"Domain": feature["domain"],
|
||||
"Status": feature["status"],
|
||||
"Evidence Maturity": feature["evidence_maturity"],
|
||||
"Commit Verified": feature["verified_at_commit"],
|
||||
"Runtime Validation": (
|
||||
feature["runtime_validation"]["status"]
|
||||
),
|
||||
}
|
||||
|
||||
for field, expected_value in expected_values.items():
|
||||
actual = record.get(field)
|
||||
|
||||
if actual != expected_value:
|
||||
errors.append(
|
||||
f"{path.relative_to(ROOT)}: "
|
||||
f"{feature_id} {field}: "
|
||||
f"{actual!r} != {expected_value!r}"
|
||||
)
|
||||
|
||||
if all_found != set(catalog):
|
||||
errors.append(
|
||||
"Domain Markdown IDs do not match catalog JSON"
|
||||
)
|
||||
|
||||
|
||||
def validate_catalog_markdown(
|
||||
features: list[dict[str, Any]],
|
||||
errors: list[str],
|
||||
) -> None:
|
||||
expected = {
|
||||
feature["id"]: feature
|
||||
for feature in features
|
||||
}
|
||||
found: dict[str, list[str]] = {}
|
||||
|
||||
for line in CATALOG_MD_PATH.read_text(
|
||||
encoding="utf-8"
|
||||
).splitlines():
|
||||
if not line.startswith("|"):
|
||||
continue
|
||||
|
||||
cells = [
|
||||
cell.strip()
|
||||
for cell in line.strip().strip("|").split("|")
|
||||
]
|
||||
|
||||
if len(cells) < 6:
|
||||
continue
|
||||
|
||||
feature_id = clean(cells[0])
|
||||
|
||||
if FEATURE_ID_RE.fullmatch(feature_id):
|
||||
found[feature_id] = cells
|
||||
|
||||
if set(found) != set(expected):
|
||||
errors.append(
|
||||
"feature-catalog.md IDs do not match JSON"
|
||||
)
|
||||
|
||||
for feature_id, cells in found.items():
|
||||
feature = expected[feature_id]
|
||||
runtime = feature["runtime_validation"]
|
||||
runtime_display = (
|
||||
runtime["status"]
|
||||
if runtime["required"]
|
||||
else "not required"
|
||||
)
|
||||
|
||||
actual = {
|
||||
"name": cells[1].replace("\\|", "|"),
|
||||
"domain": clean(cells[2]),
|
||||
"status": clean(cells[3]),
|
||||
"maturity": clean(cells[4]),
|
||||
"runtime": clean(cells[5]),
|
||||
}
|
||||
|
||||
wanted = {
|
||||
"name": feature["name"],
|
||||
"domain": feature["domain"],
|
||||
"status": feature["status"],
|
||||
"maturity": feature["evidence_maturity"],
|
||||
"runtime": runtime_display,
|
||||
}
|
||||
|
||||
for field, expected_value in wanted.items():
|
||||
if actual[field] != expected_value:
|
||||
errors.append(
|
||||
f"feature-catalog.md: {feature_id} "
|
||||
f"{field}: {actual[field]!r} "
|
||||
f"!= {expected_value!r}"
|
||||
)
|
||||
|
||||
|
||||
def validate_reviews(
|
||||
features: list[dict[str, Any]],
|
||||
errors: list[str],
|
||||
) -> int:
|
||||
catalog = {
|
||||
feature["id"]: feature
|
||||
for feature in features
|
||||
}
|
||||
checked = 0
|
||||
|
||||
for path in sorted(
|
||||
REVIEWS_DIR.glob("evidence-sample-*.md")
|
||||
):
|
||||
current_id: str | None = None
|
||||
results: set[str] = set()
|
||||
|
||||
for line_number, line in enumerate(
|
||||
path.read_text(encoding="utf-8").splitlines(),
|
||||
start=1,
|
||||
):
|
||||
heading = REVIEW_HEADING_RE.match(line)
|
||||
|
||||
if heading:
|
||||
current_id = heading.group(1)
|
||||
continue
|
||||
|
||||
result = REVIEW_RESULT_RE.match(line)
|
||||
|
||||
if not result or current_id is None:
|
||||
continue
|
||||
|
||||
status, maturity = result.groups()
|
||||
results.add(current_id)
|
||||
checked += 1
|
||||
|
||||
feature = catalog.get(current_id)
|
||||
|
||||
if feature is None:
|
||||
errors.append(
|
||||
f"{path.relative_to(ROOT)}:{line_number}: "
|
||||
f"unknown feature {current_id}"
|
||||
)
|
||||
continue
|
||||
|
||||
if feature["status"] != status:
|
||||
errors.append(
|
||||
f"{current_id}: review status {status!r} "
|
||||
f"!= catalog {feature['status']!r}"
|
||||
)
|
||||
|
||||
if feature["evidence_maturity"] != maturity:
|
||||
errors.append(
|
||||
f"{current_id}: review maturity {maturity!r} "
|
||||
f"!= catalog "
|
||||
f"{feature['evidence_maturity']!r}"
|
||||
)
|
||||
|
||||
if path.name == "evidence-sample-01.md" and len(results) != 12:
|
||||
errors.append(
|
||||
f"{path.relative_to(ROOT)}: expected 12 "
|
||||
f"review results, found {len(results)}"
|
||||
)
|
||||
|
||||
return checked
|
||||
|
||||
|
||||
def validate_whitespace(errors: list[str]) -> None:
|
||||
for path in sorted(ROOT.rglob("*")):
|
||||
if not path.is_file():
|
||||
continue
|
||||
|
||||
if path.suffix not in {".md", ".json", ".py", ".txt"}:
|
||||
continue
|
||||
|
||||
for line_number, line in enumerate(
|
||||
path.read_text(
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
).splitlines(),
|
||||
start=1,
|
||||
):
|
||||
if line != line.rstrip(" \t"):
|
||||
errors.append(
|
||||
f"{path.relative_to(ROOT)}:{line_number}: "
|
||||
"trailing whitespace"
|
||||
)
|
||||
|
||||
if any(ROOT.rglob("*.pyc")):
|
||||
errors.append("Generated .pyc files exist")
|
||||
|
||||
if any(
|
||||
path.is_dir()
|
||||
for path in ROOT.rglob("__pycache__")
|
||||
):
|
||||
errors.append("__pycache__ exists")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
errors: list[str] = []
|
||||
features = load_catalog(errors)
|
||||
reviewed = 0
|
||||
|
||||
if features:
|
||||
validate_catalog(features, errors)
|
||||
validate_domains(features, errors)
|
||||
validate_catalog_markdown(features, errors)
|
||||
reviewed = validate_reviews(features, errors)
|
||||
|
||||
validate_whitespace(errors)
|
||||
|
||||
print(f"Catalog Features: {len(features)}")
|
||||
print(
|
||||
"Unique Feature IDs:",
|
||||
len({feature.get("id") for feature in features}),
|
||||
)
|
||||
print(
|
||||
"Domain Files:",
|
||||
len(list(DOMAINS_DIR.glob("*.md"))),
|
||||
)
|
||||
|
||||
if features:
|
||||
print(
|
||||
"Statuses:",
|
||||
dict(
|
||||
Counter(
|
||||
feature.get("status")
|
||||
for feature in features
|
||||
)
|
||||
),
|
||||
)
|
||||
print(
|
||||
"Evidence Maturity:",
|
||||
dict(
|
||||
Counter(
|
||||
feature.get("evidence_maturity")
|
||||
for feature in features
|
||||
)
|
||||
),
|
||||
)
|
||||
print(
|
||||
"Runtime Validation:",
|
||||
dict(
|
||||
Counter(
|
||||
(
|
||||
feature.get(
|
||||
"runtime_validation",
|
||||
{},
|
||||
).get("required"),
|
||||
feature.get(
|
||||
"runtime_validation",
|
||||
{},
|
||||
).get("status"),
|
||||
)
|
||||
for feature in features
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
print(f"Review Results Checked: {reviewed}")
|
||||
print(f"Consistency Errors: {len(errors)}")
|
||||
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}")
|
||||
|
||||
return 1 if errors else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate the Odysseus public discovery documentation package."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = ROOT.parents[1]
|
||||
CATALOG_PATH = ROOT / "feature-catalog.json"
|
||||
DOMAINS_DIR = ROOT / "domains"
|
||||
|
||||
EXPECTED_COMMIT = "d8a2059df8e53bc7275c45339849d14c8651e73c"
|
||||
ALLOWED_STATUSES = {
|
||||
"verified", "partial", "disabled", "experimental", "legacy",
|
||||
"dead-code-candidate", "unverified"
|
||||
}
|
||||
|
||||
LINE_RANGE_RE = re.compile(r"^L([1-9]\d*)-L([1-9]\d*)$")
|
||||
FEATURE_HEADING_RE = re.compile(
|
||||
r"^##\s+`?([A-Z][A-Z0-9]*-\d{3})`?\s+(?:—|-)\s+.+$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
LINK_RE = re.compile(r"(?<!!)\[[^\]]*\]\(([^)]+)\)")
|
||||
FORBIDDEN_TERMS_RE = re.compile(
|
||||
r"(roadforge|kanban|matrix|owner link|editor link|github support|"
|
||||
r"private maintainer|OD-AUD-|[A-Z]{2,10}-AUD-\d+|TASK-\d+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Validate Odysseus discovery docs.")
|
||||
parser.add_argument(
|
||||
"--repo-root",
|
||||
type=Path,
|
||||
default=REPO_ROOT,
|
||||
help="Path to Odysseus repository root.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
def validate() -> int:
|
||||
args = parse_args()
|
||||
repo_root = args.repo_root.resolve()
|
||||
errors: list[str] = []
|
||||
|
||||
# 1. Validate Catalog JSON existence and content
|
||||
if not CATALOG_PATH.is_file():
|
||||
errors.append(f"Missing catalog file: {CATALOG_PATH}")
|
||||
print(f"Errors: {len(errors)}")
|
||||
for e in errors:
|
||||
print(f"ERROR: {e}")
|
||||
return 1
|
||||
|
||||
try:
|
||||
catalog = json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
|
||||
except Exception as e:
|
||||
errors.append(f"Failed to parse catalog JSON: {e}")
|
||||
print(f"Errors: {len(errors)}")
|
||||
for err in errors:
|
||||
print(f"ERROR: {err}")
|
||||
return 1
|
||||
|
||||
if not isinstance(catalog, list):
|
||||
errors.append("feature-catalog.json must be a JSON array")
|
||||
return 1
|
||||
|
||||
feature_ids = [item.get("id") for item in catalog if isinstance(item, dict)]
|
||||
if len(feature_ids) != 79:
|
||||
errors.append(f"Expected 79 unique feature IDs, found {len(feature_ids)}")
|
||||
if len(feature_ids) != len(set(feature_ids)):
|
||||
errors.append("Duplicate feature IDs found in catalog JSON")
|
||||
|
||||
# Domain catalog counts
|
||||
catalog_domain_counts: dict[str, int] = {}
|
||||
for item in catalog:
|
||||
if not isinstance(item, dict):
|
||||
errors.append("Catalog item is not an object")
|
||||
continue
|
||||
fid = item.get("id", "<missing>")
|
||||
status = item.get("status")
|
||||
domain = item.get("domain", "").lower()
|
||||
catalog_domain_counts[domain] = catalog_domain_counts.get(domain, 0) + 1
|
||||
|
||||
if status not in ALLOWED_STATUSES:
|
||||
errors.append(f"{fid}: Invalid status '{status}'")
|
||||
|
||||
evidence_list = item.get("evidence")
|
||||
if not isinstance(evidence_list, list) or not evidence_list:
|
||||
errors.append(f"{fid}: Missing or empty evidence list")
|
||||
continue
|
||||
|
||||
for ev in evidence_list:
|
||||
if not isinstance(ev, dict):
|
||||
errors.append(f"{fid}: Evidence item is not an object")
|
||||
continue
|
||||
path_str = ev.get("path")
|
||||
lr_str = str(ev.get("line_range", ""))
|
||||
if not path_str or Path(path_str).is_absolute() or ".." in Path(path_str).parts:
|
||||
errors.append(f"{fid}: Unsafe or invalid path '{path_str}'")
|
||||
continue
|
||||
|
||||
# Check path exists in repo
|
||||
target_path = repo_root / path_str
|
||||
if not target_path.is_file():
|
||||
errors.append(f"{fid}: Referenced path '{path_str}' does not exist on disk")
|
||||
continue
|
||||
|
||||
# Check line range format & bounds
|
||||
m = LINE_RANGE_RE.fullmatch(lr_str)
|
||||
if not m:
|
||||
errors.append(f"{fid}: Invalid line range format '{lr_str}' for path '{path_str}'")
|
||||
continue
|
||||
|
||||
start, end = int(m.group(1)), int(m.group(2))
|
||||
lines_cnt = len(target_path.read_text(encoding="utf-8", errors="ignore").splitlines())
|
||||
if start > end or end > lines_cnt or start < 1:
|
||||
errors.append(
|
||||
f"{fid}: Line range '{lr_str}' exceeds file length ({lines_cnt} lines) in '{path_str}'"
|
||||
)
|
||||
|
||||
# 2. Check Domain Markdown files
|
||||
md_feature_ids: list[str] = []
|
||||
domain_files = sorted(DOMAINS_DIR.glob("*.md"))
|
||||
for df in domain_files:
|
||||
domain_name = df.stem.lower()
|
||||
content = df.read_text(encoding="utf-8")
|
||||
found_ids = FEATURE_HEADING_RE.findall(content)
|
||||
md_feature_ids.extend(found_ids)
|
||||
if len(found_ids) != catalog_domain_counts.get(domain_name, 0):
|
||||
errors.append(
|
||||
f"Domain '{domain_name}' count mismatch: catalog has {catalog_domain_counts.get(domain_name, 0)}, Markdown has {len(found_ids)}"
|
||||
)
|
||||
|
||||
if sorted(md_feature_ids) != sorted(feature_ids):
|
||||
errors.append("Markdown domain feature IDs do not match catalog JSON feature IDs")
|
||||
|
||||
# 3. Check for forbidden/private terms, sensitive credentials, and broken links across all docs
|
||||
for md_file in ROOT.rglob("*.md"):
|
||||
rel_md = md_file.relative_to(ROOT)
|
||||
content = md_file.read_text(encoding="utf-8")
|
||||
|
||||
# Forbidden terms scan
|
||||
forbidden_matches = FORBIDDEN_TERMS_RE.findall(content)
|
||||
if forbidden_matches:
|
||||
errors.append(
|
||||
f"{rel_md}: Found forbidden/internal terms: {set(forbidden_matches)}"
|
||||
)
|
||||
|
||||
# Broken local link check
|
||||
for target in LINK_RE.findall(content):
|
||||
target = target.strip().strip("<>")
|
||||
if not target or target.startswith(("#", "http://", "https://", "mailto:")):
|
||||
continue
|
||||
target_path = target.split("#", 1)[0]
|
||||
resolved = (md_file.parent / target_path).resolve()
|
||||
if not resolved.exists():
|
||||
errors.append(f"{rel_md}: Broken local link '{target}'")
|
||||
|
||||
print(f"Catalog Features: {len(catalog)}")
|
||||
print(f"Domain Files: {len(domain_files)}")
|
||||
print(f"Validation Errors: {len(errors)}")
|
||||
for err in errors:
|
||||
print(f"ERROR: {err}")
|
||||
|
||||
return 1 if errors else 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(validate())
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user