odysseus/tests/smoke/areas.py

"""The area registry and the result table for the release smoke suite.

Pure stdlib on purpose: this module is the one part of the suite that has
to be readable and testable without a running instance, because it is
what decides whether the suite's output is honest.

Two lists matter here and they are both deliberate:

``COVERED`` names every feature area the suite drives, and the test
module that drives it. A row appears in the table whether or not its
module ran, so an area cannot quietly vanish from the report by having
its file deleted or renamed - it shows up as ``NOT RUN`` instead.

``DECLARED_GAPS`` names the areas the suite does *not* cover, with the
reason. They are printed alongside the results rather than left out,
because a smoke report that lists only what it checked reads as
coverage of everything it does not mention.
"""
from __future__ import annotations

import textwrap
from dataclasses import dataclass

# Result labels. ASCII only - no Unicode status glyphs anywhere in the
# table (repo convention: no emoji in UI or code).
PASS = "PASS"
FAIL = "FAIL"
SKIP = "SKIP"
NOT_RUN = "NOT RUN"
NOT_COVERED = "NOT COVERED"

# Precedence when one area's module produces several outcomes: a single
# failure decides the row, then a skip, then pass.
_PRECEDENCE = (FAIL, SKIP, PASS)

# Table geometry. Wide enough for the longest gap reason to read as a
# sentence, narrow enough to survive a normal terminal.
TABLE_WIDTH = 100
MIN_DETAIL_WIDTH = 30


@dataclass(frozen=True)
class Area:
    """One advertised feature area and the module that exercises it."""

    key: str
    label: str
    module: str


@dataclass(frozen=True)
class Gap:
    """An area this suite does not cover, and why it does not."""

    label: str
    reason: str


# Order is the order the table prints in: the chat surface first, then
# the feature areas README.md advertises, then the setup surface.
COVERED = (
    Area("chat", "Chat", "test_chat_smoke.py"),
    Area("compare", "Compare", "test_compare_smoke.py"),
    Area("notes", "Notes", "test_notes_smoke.py"),
    Area("calendar", "Calendar", "test_calendar_smoke.py"),
    Area("tasks", "Tasks (scheduled)", "test_tasks_smoke.py"),
    Area("documents", "Documents (editor)", "test_documents_smoke.py"),
    Area("documents_rag", "Documents (RAG)", "test_documents_rag_smoke.py"),
    Area("email", "Email", "test_email_smoke.py"),
    Area("memory", "Memory", "test_memory_smoke.py"),
    Area("uploads", "Uploads", "test_uploads_smoke.py"),
    Area("cookbook", "Cookbook", "test_cookbook_smoke.py"),
    Area("settings", "Settings", "test_settings_smoke.py"),
)

DECLARED_GAPS = (
    Gap(
        "Deep Research",
        "needs live web egress; the crawler has no deterministic stub and adding "
        "one would be an application change",
    ),
    Gap(
        "Web Search",
        "needs a reachable SearXNG or an external provider, so the result is not "
        "reproducible from a clean checkout",
    ),
    Gap(
        "Email over IMAP/SMTP",
        "covered through the existing ODYSSEUS_EMAIL_FIXTURE path only; no local "
        "mail server, so real account sync and send are untested",
    ),
    Gap(
        "Cookbook download and serve",
        "needs tmux, a GPU runtime and a multi-GB model download; only hardware "
        "fit and state sync are checked",
    ),
    Gap(
        "Gallery and photo editor",
        "already the one area with Playwright specs under tests/e2e/photo-editor/",
    ),
    Gap(
        "Agent tool loop",
        "measured by the checkpoint benchmark, which is the safety net that does "
        "cover the agent runtime",
    ),
    Gap(
        "MCP servers",
        "the built-in servers are stdio subprocesses whose readiness is not part "
        "of the app's own readiness contract",
    ),
    Gap(
        "Rendering and layout",
        "pinned by the computed-style snapshot in "
        "tests/test_css_computed_style_snapshot.py",
    ),
)

_MODULE_TO_KEY = {area.module: area.key for area in COVERED}


def area_for_module(module_name: str) -> str | None:
    """Map a test module filename to its area key, or None."""
    return _MODULE_TO_KEY.get(module_name)


def resolve(outcomes: list[str]) -> str:
    """Collapse one module's outcomes into the row's single result."""
    if not outcomes:
        return NOT_RUN
    for label in _PRECEDENCE:
        if label in outcomes:
            return label
    return NOT_RUN


def render_table(results, *, header="", areas=COVERED, gaps=DECLARED_GAPS,
                 width=TABLE_WIDTH) -> str:
    """Render the per-area table.

    ``results`` maps an area key to a mapping with ``result`` and,
    optionally, ``checks`` and ``detail``. Unknown keys are ignored and
    missing keys render as ``NOT RUN`` - the registry, not the run,
    decides which rows exist.
    """
    rows = []
    for area in areas:
        entry = results.get(area.key) or {}
        result = entry.get("result") or NOT_RUN
        checks = entry.get("checks")
        detail = entry.get("detail") or ""
        if result == NOT_RUN and not detail:
            detail = "no test ran for this area"
        rows.append((area.label, result,
                     "" if checks is None else str(checks), detail))

    labels = [row[0] for row in rows] + [gap.label for gap in gaps] + ["AREA"]
    label_width = max(len(label) for label in labels)
    result_width = max([len(row[1]) for row in rows] + [len(NOT_COVERED), len("RESULT")])
    checks_width = max([len(row[2]) for row in rows] + [len("CHECKS")])
    # Indent + label + gap + result + gap + checks + gap, then the detail.
    detail_indent = 2 + label_width + 2 + result_width + 2 + checks_width + 2
    detail_width = max(width - detail_indent, MIN_DETAIL_WIDTH)

    def row_lines(label, result, checks, detail):
        first = (f"  {label.ljust(label_width)}  {result.ljust(result_width)}  "
                 f"{checks.rjust(checks_width)}  ")
        wrapped = textwrap.wrap(detail, detail_width) or [""]
        out = [(first + wrapped[0]).rstrip()]
        out += [(" " * detail_indent + line).rstrip() for line in wrapped[1:]]
        return out

    lines = []
    if header:
        lines.extend([header, ""])
    lines.append(
        f"  {'AREA'.ljust(label_width)}  {'RESULT'.ljust(result_width)}  "
        f"{'CHECKS'.rjust(checks_width)}  DETAIL"
    )
    for row in rows:
        lines.extend(row_lines(*row))

    if gaps:
        lines.extend(["", "  Not covered, deliberately:"])
        for gap in gaps:
            lines.extend(row_lines(gap.label, NOT_COVERED, "", gap.reason))

    failed = [row for row in rows if row[1] == FAIL]
    skipped = [row for row in rows if row[1] == SKIP]
    not_run = [row for row in rows if row[1] == NOT_RUN]
    passed = len(rows) - len(failed) - len(skipped) - len(not_run)
    lines.extend(["", (
        f"  {passed} pass, {len(failed)} fail, {len(skipped)} skip, "
        f"{len(not_run)} not run, {len(gaps)} declared gaps"
    )])
    return "\n".join(lines)