mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-07 15:32:21 +02:00
chore(tools): add a read-only ref-to-ref parity audit
`lab` and the public `dev` line share only the repository's first commit as a merge base, so `git log lab..dev` lists two thousand commits that are almost all already present on both sides under different SHAs. Nothing in that output says which public fixes never reached `lab`, which is the only question that matters before `lab` becomes a release. `scripts/ref_parity_audit.py` samples the most distinctive added lines from each commit in the range and searches the other tree for them with `git grep -F`, then reports the file-level presence diff. The two complement each other: a commit whose probes are all found while one of the files it added is missing from the target is a fix whose production change was reproduced without its test, which the line sampling alone cannot see. Probes are stripped of indentation and searched for anywhere in the tree, so a port that moved or was re-indented still reads as present. Verdicts are absent / partial / present / no-probe, and the report says outright that the commit verdicts are a heuristic while the two file lists are exact. Read-only by construction: `git log`, `show`, `diff`, `grep`, `ls-tree` and `merge-base` only, no remote access, and it does not import the app package.
This commit is contained in:
Léo
@@ -0,0 +1,353 @@
|
||||
"""Behaviour of scripts/ref_parity_audit.py.
|
||||
|
||||
The audit answers one question: which commits on a source ref left no trace in a
|
||||
target ref? The interesting cases are the ones a plain `git log A..B` gets wrong
|
||||
-- a change that was reproduced on the target under a different SHA must read as
|
||||
present, and a change whose lines were merely re-indented must not read as
|
||||
missing. Those are exercised against a real throwaway repository rather than a
|
||||
fake, because the verdicts come from git's own `grep` and `diff`.
|
||||
"""
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.cli_loader import load_script
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def audit():
|
||||
return load_script("ref_parity_audit.py")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# probe selection
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_short_and_thin_lines_are_not_probes(audit):
|
||||
assert not audit.is_probe_candidate("pass")
|
||||
assert not audit.is_probe_candidate("})")
|
||||
# Long enough, but nothing named in it: a separator comment is in every file.
|
||||
assert not audit.is_probe_candidate("# " + "-" * 60)
|
||||
assert audit.is_probe_candidate("_hosts_cache_time = now # remember the empty answer")
|
||||
|
||||
|
||||
def test_probes_are_ranked_by_distinctiveness_and_capped(audit):
|
||||
lines = [
|
||||
("src/a.py", " x = 1"),
|
||||
("src/a.py", " parsed_args = json.loads(args) if args else []"),
|
||||
("src/a.py", " if not isinstance(parsed_args, list): raise HTTPException(400)"),
|
||||
("src/a.py", " return parsed_args"),
|
||||
]
|
||||
|
||||
probes = audit.pick_probes(lines, limit=2)
|
||||
|
||||
assert probes == [
|
||||
"if not isinstance(parsed_args, list): raise HTTPException(400)",
|
||||
"parsed_args = json.loads(args) if args else []",
|
||||
]
|
||||
|
||||
|
||||
def test_probes_are_stripped_so_a_reindented_port_still_counts(audit):
|
||||
probes = audit.pick_probes([("src/a.py", "\t\tself.cache_time = now # keep the empty answer")], limit=4)
|
||||
|
||||
assert probes == ["self.cache_time = now # keep the empty answer"]
|
||||
|
||||
|
||||
def test_identical_added_lines_yield_one_probe(audit):
|
||||
lines = [
|
||||
("src/a.py", " raise HTTPException(400, 'args must be a JSON array')"),
|
||||
("src/b.py", " raise HTTPException(400, 'args must be a JSON array')"),
|
||||
]
|
||||
|
||||
assert len(audit.pick_probes(lines, limit=4)) == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# path exclusion and diff parsing
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path, excluded",
|
||||
[
|
||||
("static/lib/marked.js", True),
|
||||
("static/js/chat.min.js", True),
|
||||
("package-lock.json", True),
|
||||
("assets/logo.png", True),
|
||||
("src/model_discovery.py", False),
|
||||
("static/js/chat.js", False),
|
||||
],
|
||||
)
|
||||
def test_default_excludes_cover_vendored_and_binary_paths(audit, path, excluded):
|
||||
assert audit.is_excluded(path, audit.DEFAULT_EXCLUDES) is excluded
|
||||
|
||||
|
||||
def test_added_lines_skips_headers_deletions_and_excluded_paths(audit):
|
||||
patch = "\n".join(
|
||||
[
|
||||
"diff --git a/src/a.py b/src/a.py",
|
||||
"--- a/src/a.py",
|
||||
"+++ b/src/a.py",
|
||||
"@@ -1 +1 @@",
|
||||
"-old_line_that_is_long_enough()",
|
||||
"+new_line_that_is_long_enough()",
|
||||
"diff --git a/static/lib/vendor.js b/static/lib/vendor.js",
|
||||
"--- a/static/lib/vendor.js",
|
||||
"+++ b/static/lib/vendor.js",
|
||||
"@@ -1 +1 @@",
|
||||
"+vendored_line_that_is_long_enough()",
|
||||
"diff --git a/src/gone.py b/src/gone.py",
|
||||
"--- a/src/gone.py",
|
||||
"+++ /dev/null",
|
||||
]
|
||||
)
|
||||
|
||||
assert audit.added_lines(patch, audit.DEFAULT_EXCLUDES) == [
|
||||
("src/a.py", "new_line_that_is_long_enough()")
|
||||
]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# end-to-end against a real repository
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
MARKER_ABSENT = "def confine_agent_to_workspace(root_dir, session_store):"
|
||||
MARKER_PORTED = "def cache_the_empty_tailscale_answer(now, hosts_cache_time):"
|
||||
MARKER_KEPT = "def shared_helper_present_on_both_lines(value):"
|
||||
|
||||
|
||||
def _git(repo, *args):
|
||||
subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(repo),
|
||||
"-c",
|
||||
"user.name=Audit Fixture",
|
||||
"-c",
|
||||
"user.email=fixture@example.invalid",
|
||||
"-c",
|
||||
"commit.gpgsign=false",
|
||||
*args,
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def two_line_repo(tmp_path):
|
||||
"""A repository shaped like the real problem: two branches off one root.
|
||||
|
||||
`source` carries three commits. `target` independently reproduces one of
|
||||
them (re-indented, so only a stripped probe finds it), never sees another,
|
||||
and carries a file of its own plus a vendored file that must stay out of
|
||||
the presence diff.
|
||||
"""
|
||||
if not shutil.which("git"):
|
||||
pytest.skip("git is not available")
|
||||
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
_git(repo, "init", "-q", "-b", "root")
|
||||
(repo / "base.py").write_text(MARKER_KEPT + "\n return value\n", encoding="utf-8")
|
||||
_git(repo, "add", "base.py")
|
||||
_git(repo, "commit", "-q", "-m", "chore: base")
|
||||
|
||||
_git(repo, "checkout", "-q", "-b", "source")
|
||||
(repo / "confinement.py").write_text(MARKER_ABSENT + "\n return root_dir\n", encoding="utf-8")
|
||||
_git(repo, "add", "confinement.py")
|
||||
_git(repo, "commit", "-q", "-m", "fix: confine the agent to its workspace")
|
||||
|
||||
(repo / "discovery.py").write_text(MARKER_PORTED + "\n return now\n", encoding="utf-8")
|
||||
_git(repo, "add", "discovery.py")
|
||||
_git(repo, "commit", "-q", "-m", "fix: cache an empty lookup")
|
||||
|
||||
(repo / "base.py").write_text(MARKER_KEPT + "\n", encoding="utf-8")
|
||||
_git(repo, "add", "base.py")
|
||||
_git(repo, "commit", "-q", "-m", "refactor: drop the base body")
|
||||
|
||||
_git(repo, "checkout", "-q", "-b", "target", "root")
|
||||
# The same fix, written by hand at a different indentation and in another file.
|
||||
(repo / "net.py").write_text("class Net:\n " + MARKER_PORTED + "\n return now\n", encoding="utf-8")
|
||||
(repo / "target_only.py").write_text("# only on the target line\n", encoding="utf-8")
|
||||
(repo / "static").mkdir()
|
||||
(repo / "static" / "lib").mkdir()
|
||||
(repo / "static" / "lib" / "vendor.min.js").write_text("var a=1;\n", encoding="utf-8")
|
||||
_git(repo, "add", "-A")
|
||||
_git(repo, "commit", "-q", "-m", "feat: reproduce the lookup cache by hand")
|
||||
|
||||
return repo
|
||||
|
||||
|
||||
def _verdicts(report):
|
||||
return {item.commit.subject: item.verdict for item in report.verdicts}
|
||||
|
||||
|
||||
def test_absent_ported_and_deletion_only_commits_are_told_apart(audit, two_line_repo):
|
||||
report = audit.audit(source="source", target="target", repo=two_line_repo)
|
||||
|
||||
assert _verdicts(report) == {
|
||||
"fix: confine the agent to its workspace": audit.VERDICT_ABSENT,
|
||||
"fix: cache an empty lookup": audit.VERDICT_PRESENT,
|
||||
"refactor: drop the base body": audit.VERDICT_NO_PROBE,
|
||||
}
|
||||
|
||||
|
||||
def test_the_absent_commit_reports_the_probes_it_looked_for(audit, two_line_repo):
|
||||
report = audit.audit(source="source", target="target", repo=two_line_repo)
|
||||
absent = report.by_verdict(audit.VERDICT_ABSENT)
|
||||
|
||||
assert [item.commit.subject for item in absent] == ["fix: confine the agent to its workspace"]
|
||||
assert absent[0].probes == (MARKER_ABSENT,)
|
||||
assert absent[0].found == ()
|
||||
assert absent[0].paths == ("confinement.py",)
|
||||
|
||||
|
||||
def test_file_presence_diff_is_exact_and_skips_vendored_paths(audit, two_line_repo):
|
||||
report = audit.audit(source="source", target="target", repo=two_line_repo)
|
||||
|
||||
assert report.source_only_files == ("confinement.py", "discovery.py")
|
||||
assert report.target_only_files == ("net.py", "target_only.py")
|
||||
|
||||
|
||||
def test_probes_are_only_drawn_from_included_paths(audit, two_line_repo):
|
||||
"""Excluding discovery.py leaves its commit with nothing to probe."""
|
||||
report = audit.audit(
|
||||
source="source",
|
||||
target="target",
|
||||
repo=two_line_repo,
|
||||
excludes=list(audit.DEFAULT_EXCLUDES) + ["discovery.py"],
|
||||
)
|
||||
|
||||
assert _verdicts(report)["fix: cache an empty lookup"] == audit.VERDICT_NO_PROBE
|
||||
|
||||
|
||||
def test_a_probe_found_only_partly_reads_as_partial(audit, two_line_repo):
|
||||
# base.py's marker is on both branches; confinement.py's is not. A commit
|
||||
# adding both is the partial case.
|
||||
_git(two_line_repo, "checkout", "-q", "source")
|
||||
(two_line_repo / "mixed.py").write_text(
|
||||
MARKER_KEPT + "\n" + MARKER_ABSENT + "\n", encoding="utf-8"
|
||||
)
|
||||
_git(two_line_repo, "add", "mixed.py")
|
||||
_git(two_line_repo, "commit", "-q", "-m", "fix: a change that half landed")
|
||||
|
||||
report = audit.audit(source="source", target="target", repo=two_line_repo, probe_limit=4)
|
||||
|
||||
assert _verdicts(report)["fix: a change that half landed"] == audit.VERDICT_PARTIAL
|
||||
|
||||
|
||||
def test_first_parent_traversal_diffs_a_merge_against_its_first_parent(audit, two_line_repo):
|
||||
"""A squash-free merge carries its content in the merge commit itself."""
|
||||
_git(two_line_repo, "checkout", "-q", "-b", "side", "root")
|
||||
(two_line_repo / "from_side.py").write_text(MARKER_ABSENT + "\n", encoding="utf-8")
|
||||
_git(two_line_repo, "add", "from_side.py")
|
||||
_git(two_line_repo, "commit", "-q", "-m", "fix: arrived on a side branch")
|
||||
_git(two_line_repo, "checkout", "-q", "source")
|
||||
_git(two_line_repo, "merge", "-q", "--no-ff", "-m", "Merge commit from fork", "side")
|
||||
|
||||
report = audit.audit(
|
||||
source="source", target="target", repo=two_line_repo, traversal="first-parent"
|
||||
)
|
||||
merge = [v for v in report.verdicts if v.commit.subject == "Merge commit from fork"]
|
||||
|
||||
assert len(merge) == 1
|
||||
assert merge[0].commit.parent_count == 2
|
||||
assert merge[0].verdict == audit.VERDICT_ABSENT
|
||||
assert MARKER_ABSENT in merge[0].probes
|
||||
|
||||
|
||||
def test_linear_traversal_reports_the_side_branch_commit_not_the_merge(audit, two_line_repo):
|
||||
_git(two_line_repo, "checkout", "-q", "-b", "side", "root")
|
||||
(two_line_repo / "from_side.py").write_text(MARKER_ABSENT + "\n", encoding="utf-8")
|
||||
_git(two_line_repo, "add", "from_side.py")
|
||||
_git(two_line_repo, "commit", "-q", "-m", "fix: arrived on a side branch")
|
||||
_git(two_line_repo, "checkout", "-q", "source")
|
||||
_git(two_line_repo, "merge", "-q", "--no-ff", "-m", "Merge commit from fork", "side")
|
||||
|
||||
subjects = [
|
||||
v.commit.subject
|
||||
for v in audit.audit(source="source", target="target", repo=two_line_repo).verdicts
|
||||
]
|
||||
|
||||
assert "fix: arrived on a side branch" in subjects
|
||||
assert "Merge commit from fork" not in subjects
|
||||
|
||||
|
||||
def test_date_window_bounds_the_commit_range(audit, two_line_repo):
|
||||
empty = audit.audit(source="source", target="target", repo=two_line_repo, until="2000-01-01")
|
||||
full = audit.audit(source="source", target="target", repo=two_line_repo, since="2000-01-01")
|
||||
|
||||
assert empty.verdicts == []
|
||||
assert len(full.verdicts) == 3
|
||||
# The presence diff is not date-filtered: it compares the two trees.
|
||||
assert empty.source_only_files == full.source_only_files
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# rendering and cli
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_markdown_report_leads_with_the_absent_commits(audit, two_line_repo):
|
||||
text = audit.render_markdown(audit.audit(source="source", target="target", repo=two_line_repo))
|
||||
|
||||
assert "No trace in the target: **1**" in text
|
||||
assert "Fully present: **1**" in text
|
||||
assert "fix: confine the agent to its workspace" in text
|
||||
assert "`confinement.py`" in text
|
||||
|
||||
|
||||
def test_json_output_is_written_to_the_requested_path(audit, two_line_repo, tmp_path, capsys):
|
||||
out = tmp_path / "report.json"
|
||||
|
||||
code = audit.main(
|
||||
[
|
||||
"--source",
|
||||
"source",
|
||||
"--target",
|
||||
"target",
|
||||
"--repo",
|
||||
str(two_line_repo),
|
||||
"--format",
|
||||
"json",
|
||||
"--output",
|
||||
str(out),
|
||||
"--quiet",
|
||||
]
|
||||
)
|
||||
|
||||
assert code == 0
|
||||
assert capsys.readouterr().out == ""
|
||||
payload = json.loads(out.read_text(encoding="utf-8"))
|
||||
assert payload["totals"] == {"absent": 1, "no-probe": 1, "partial": 0, "present": 1}
|
||||
assert payload["source_only_files"] == ["confinement.py", "discovery.py"]
|
||||
|
||||
|
||||
def test_an_unknown_ref_exits_two_with_a_message(audit, two_line_repo, capsys):
|
||||
code = audit.main(
|
||||
[
|
||||
"--source",
|
||||
"no-such-ref",
|
||||
"--target",
|
||||
"target",
|
||||
"--repo",
|
||||
str(two_line_repo),
|
||||
"--quiet",
|
||||
]
|
||||
)
|
||||
|
||||
assert code == 2
|
||||
assert "error:" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_probe_count_must_be_positive(audit):
|
||||
with pytest.raises(SystemExit):
|
||||
audit.build_parser().parse_args(
|
||||
["--source", "a", "--target", "b", "--probes", "0"]
|
||||
)
|
||||
Reference in New Issue
Block a user