mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-06 15:02:20 +02:00
merge: reconcile PR 40 with current lab
Integrate lab fff55a78 into PR #40 (cc25d5ba). Lab's modular email backend/frontend, modular settings, split stylesheets (static/style.css stays deleted), procfs compatibility, and request-scoped TurnContract authority win; PR #40's routing classifiers, editor/email/task features, and style.css changes are ported into lab's module and stylesheet homes. Integration fixes: - settings/api.js imports ui.js under its canonical versioned URL - browser observations keep legacy CAPTCHA/access-block evidence - artifact turns do not re-trigger broad-web research recovery - env reference documents PR test-tool variables; page regenerated PR #40 defects surfaced by lab gates and fixed here: - web_fetch generic schema drops top-level anyOf (OpenAI contract); the compact preview contract still requires url or urls - get_weather registered as a brokered network read - new lazy editor modules precached for offline use - SearXNG pin mirrored into GPU standalone compose files - image model picker again skips offline endpoints Tests updated where PR #40 changed behaviour on purpose, and PR tests moved onto lab's document_source helpers.
This commit is contained in:
Alexandre Teixeira
@@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Computed-style snapshot harness for the shipped app CSS cascade.
|
||||
|
||||
The app CSS is an ordered multi-file cascade whose rendered result depends
|
||||
on source order: hundreds of selectors are declared more than once and
|
||||
``!important`` is used throughout. Any restructuring - extracting a block into
|
||||
its own file, reordering ``<link>`` tags, moving an ``@media`` rule - can
|
||||
silently change which declaration wins, and nothing else in the suite would
|
||||
notice.
|
||||
|
||||
This module captures ``getComputedStyle`` for a fixed inventory of elements
|
||||
across pages, viewports, themes and density modes, hashes the result, and
|
||||
compares it against a committed baseline. It moves no CSS. It only makes a move
|
||||
falsifiable.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/css_snapshot.py --check # compare to the baseline
|
||||
python scripts/css_snapshot.py --write-baseline # re-record it
|
||||
python scripts/css_snapshot.py --dump before.json # raw values, for diffing
|
||||
|
||||
With no ``--origin`` the script serves the repository over loopback on an
|
||||
ephemeral port for the duration of the run, so it works standalone. Under
|
||||
pytest the session static server is reused instead.
|
||||
|
||||
To see *which property* moved rather than just which element::
|
||||
|
||||
python scripts/css_snapshot.py --dump after.json
|
||||
git stash && python scripts/css_snapshot.py --dump before.json && git stash pop
|
||||
diff <(python -m json.tool before.json) <(python -m json.tool after.json)
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socketserver
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SNAPSHOT_DIR = ROOT / "tests" / "css_snapshot"
|
||||
INVENTORY_PATH = SNAPSHOT_DIR / "inventory.json"
|
||||
BASELINE_PATH = SNAPSHOT_DIR / "baseline.json"
|
||||
CAPTURE_SCRIPT = SNAPSHOT_DIR / "capture.mjs"
|
||||
|
||||
# A capture is ~70 page loads; on a warm checkout it runs in well under a
|
||||
# minute, but a cold `npx playwright install` machine can be slow to start
|
||||
# Chromium the first time.
|
||||
CAPTURE_TIMEOUT_SECONDS = 900
|
||||
|
||||
# Hash prefix length. 16 hex characters is 64 bits - far past any accidental
|
||||
# collision risk for a few thousand entries, and short enough that the baseline
|
||||
# stays readable in a diff.
|
||||
HASH_LENGTH = 16
|
||||
|
||||
|
||||
def load_inventory(path=INVENTORY_PATH):
|
||||
"""Load the checked-in element inventory."""
|
||||
return json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def load_baseline(path=BASELINE_PATH):
|
||||
"""Load the committed baseline digest."""
|
||||
return json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _canonical(value):
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
|
||||
|
||||
def _hash(value):
|
||||
return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest()[:HASH_LENGTH]
|
||||
|
||||
|
||||
def node_available(node="node"):
|
||||
"""True when the node binary is on PATH."""
|
||||
return shutil.which(node) is not None
|
||||
|
||||
|
||||
def playwright_available(node="node", cwd=ROOT):
|
||||
"""True when node can resolve the playwright package from the repo root.
|
||||
|
||||
Playwright is a devDependency installed by ``npm ci``; a clean checkout
|
||||
that has not run it cannot drive a browser at all.
|
||||
"""
|
||||
if not node_available(node):
|
||||
return False
|
||||
result = subprocess.run(
|
||||
[node, "-e", "require.resolve('playwright')"],
|
||||
cwd=str(cwd), capture_output=True, text=True, check=False,
|
||||
)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def capture(origin, inventory=None, *, swap_rule=None, variants=None,
|
||||
node="node", cwd=ROOT, timeout=CAPTURE_TIMEOUT_SECONDS):
|
||||
"""Drive the browser capture and return ``{"snapshot": ..., "missing": ...}``.
|
||||
|
||||
``swap_rule`` swaps the first two top-level declarations of one selector
|
||||
before the stylesheet reaches the browser. It exists for the harness
|
||||
self-test: a snapshot that does not move when two conflicting rules trade
|
||||
places is not evidence of anything.
|
||||
|
||||
``variants`` restricts the run to the named variants, for a faster
|
||||
focused capture.
|
||||
"""
|
||||
inventory = inventory or load_inventory()
|
||||
selected = inventory["variants"]
|
||||
if variants:
|
||||
wanted = set(variants)
|
||||
selected = [v for v in selected if v["name"] in wanted]
|
||||
unknown = wanted - {v["name"] for v in inventory["variants"]}
|
||||
if unknown:
|
||||
raise ValueError(f"unknown variants: {sorted(unknown)}")
|
||||
job = {
|
||||
"origin": origin.rstrip("/"),
|
||||
"properties": inventory["properties"],
|
||||
"variants": selected,
|
||||
"pages": inventory["pages"],
|
||||
"swapRule": swap_rule,
|
||||
}
|
||||
result = subprocess.run(
|
||||
[node, str(CAPTURE_SCRIPT)],
|
||||
input=json.dumps(job), cwd=str(cwd),
|
||||
capture_output=True, text=True, check=False, timeout=timeout,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"css snapshot capture failed:\n{result.stderr.strip()}")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def summarize(snapshot):
|
||||
"""Reduce a raw capture to the committed digest shape.
|
||||
|
||||
Two orthogonal projections are stored rather than one hash per
|
||||
(element, variant) pair: hashing every pair would commit ~5,000 lines that
|
||||
nobody reads, while a single global digest would only ever say "something
|
||||
moved". Per-element and per-variant hashes localise a failure from both
|
||||
directions - which element drifted, and in which variant - for a file small
|
||||
enough to review.
|
||||
"""
|
||||
elements = {}
|
||||
variants = {}
|
||||
for page, per_variant in snapshot.items():
|
||||
element_values = {}
|
||||
variants[page] = {}
|
||||
for variant, measured in per_variant.items():
|
||||
variants[page][variant] = _hash(measured)
|
||||
for key, values in measured.items():
|
||||
element_values.setdefault(key, {})[variant] = values
|
||||
elements[page] = {key: _hash(values) for key, values in element_values.items()}
|
||||
return {
|
||||
"digest": _hash(snapshot),
|
||||
"elements": elements,
|
||||
"variants": variants,
|
||||
}
|
||||
|
||||
|
||||
def compare(baseline, current):
|
||||
"""Return the drift between a committed baseline and a fresh summary."""
|
||||
drift = {"digest_changed": baseline.get("digest") != current["digest"],
|
||||
"elements": [], "variants": []}
|
||||
for section in ("elements", "variants"):
|
||||
old = baseline.get(section, {})
|
||||
new = current.get(section, {})
|
||||
for page in sorted(set(old) | set(new)):
|
||||
old_page = old.get(page, {})
|
||||
new_page = new.get(page, {})
|
||||
for key in sorted(set(old_page) | set(new_page)):
|
||||
if old_page.get(key) != new_page.get(key):
|
||||
drift[section].append(f"{page}/{key}")
|
||||
return drift
|
||||
|
||||
|
||||
def serve_repository(root=ROOT):
|
||||
"""Serve the repository over loopback on an ephemeral port.
|
||||
|
||||
Mirrors the browser-test static server in ``tests/conftest.py`` so the CLI
|
||||
can run outside pytest. Returns ``(origin, shutdown)``.
|
||||
"""
|
||||
root = Path(root).resolve()
|
||||
|
||||
class Handler(http.server.SimpleHTTPRequestHandler):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, directory=str(root), **kwargs)
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
pass
|
||||
|
||||
def guess_type(self, path):
|
||||
if path.endswith(".js") or path.endswith(".mjs"):
|
||||
return "application/javascript"
|
||||
if path.endswith(".css"):
|
||||
return "text/css"
|
||||
return super().guess_type(path)
|
||||
|
||||
class Server(socketserver.TCPServer):
|
||||
allow_reuse_address = True
|
||||
|
||||
server = Server(("127.0.0.1", 0), Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
def shutdown():
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
return f"http://127.0.0.1:{server.server_address[1]}", shutdown
|
||||
|
||||
|
||||
def _describe(drift, limit=25):
|
||||
lines = []
|
||||
for section in ("elements", "variants"):
|
||||
items = drift[section]
|
||||
if not items:
|
||||
continue
|
||||
shown = items[:limit]
|
||||
suffix = f" (+{len(items) - limit} more)" if len(items) > limit else ""
|
||||
lines.append(f" {section} that moved ({len(items)}): {', '.join(shown)}{suffix}")
|
||||
return "\n".join(lines) or " (no per-element drift; the digest itself changed)"
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--origin", help="static server origin to capture against; "
|
||||
"one is started on an ephemeral port when omitted")
|
||||
parser.add_argument("--write-baseline", action="store_true",
|
||||
help=f"re-record {BASELINE_PATH.relative_to(ROOT)}")
|
||||
parser.add_argument("--check", action="store_true",
|
||||
help="compare against the committed baseline (default)")
|
||||
parser.add_argument("--dump", metavar="PATH",
|
||||
help="write the raw computed values, for property-level diffing")
|
||||
parser.add_argument("--swap-rule", metavar="SELECTOR",
|
||||
help="swap the first two top-level declarations of SELECTOR "
|
||||
"before capturing (harness self-test)")
|
||||
parser.add_argument("--variants", help="comma-separated variant names to restrict the run to")
|
||||
parser.add_argument("--node", default="node", help="node binary to use")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not playwright_available(args.node):
|
||||
parser.error("node with the playwright package is required; run `npm ci` first")
|
||||
|
||||
variants = [v.strip() for v in args.variants.split(",")] if args.variants else None
|
||||
shutdown = None
|
||||
origin = args.origin or os.environ.get("ODYSSEUS_TEST_STATIC_ORIGIN")
|
||||
if not origin:
|
||||
origin, shutdown = serve_repository()
|
||||
try:
|
||||
captured = capture(origin, swap_rule=args.swap_rule, variants=variants, node=args.node)
|
||||
finally:
|
||||
if shutdown:
|
||||
shutdown()
|
||||
|
||||
if captured["missing"]:
|
||||
print("inventory entries that matched no element:", file=sys.stderr)
|
||||
for scope, keys in sorted(captured["missing"].items()):
|
||||
print(f" {scope}: {', '.join(keys)}", file=sys.stderr)
|
||||
|
||||
summary = summarize(captured["snapshot"])
|
||||
|
||||
if args.dump:
|
||||
Path(args.dump).write_text(json.dumps(captured["snapshot"], indent=1, sort_keys=True) + "\n",
|
||||
encoding="utf-8")
|
||||
print(f"raw values written to {args.dump}")
|
||||
|
||||
if args.write_baseline:
|
||||
if variants or args.swap_rule:
|
||||
parser.error("--write-baseline needs a full, unmutated capture: "
|
||||
"drop --variants and --swap-rule")
|
||||
BASELINE_PATH.write_text(json.dumps(summary, indent=1, sort_keys=True) + "\n",
|
||||
encoding="utf-8")
|
||||
print(f"baseline written: digest {summary['digest']}")
|
||||
return 0
|
||||
|
||||
baseline = load_baseline()
|
||||
drift = compare(baseline, summary)
|
||||
if not drift["digest_changed"] and not drift["elements"] and not drift["variants"]:
|
||||
print(f"computed styles match the baseline (digest {summary['digest']})")
|
||||
return 0
|
||||
print(f"computed styles moved: baseline {baseline.get('digest')} -> {summary['digest']}")
|
||||
print(_describe(drift))
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
File diff suppressed because it is too large
Load Diff
Executable
+853
@@ -0,0 +1,853 @@
|
||||
#!/usr/bin/env python3
|
||||
"""odysseus-dev — boot the checkout you are standing in, isolated from every other one.
|
||||
|
||||
`start-macos.sh` is the single-instance launcher: it owns the Homebrew
|
||||
deps, the venv, and the production-shaped boot. It deliberately shares
|
||||
whatever is already listening — an open ChromaDB port is a resource it
|
||||
adopts. That is right for one instance and wrong for N worktrees, where
|
||||
adopting a port means writing into another checkout's vector store.
|
||||
|
||||
This tool is the sibling that owns isolation instead:
|
||||
|
||||
- ports are derived from the worktree path, so two checkouts never
|
||||
pick the same ones and the same checkout always picks its own;
|
||||
- a ChromaDB we did not start is never adopted — we start our own on
|
||||
our own port against our own data dir, or fall closed to keyword
|
||||
mode and say so;
|
||||
- the data dir, the database and the browser-MCP cache all live under
|
||||
`.odysseus-dev/`, so a dev boot leaves `data/` — what a normal launch
|
||||
of this checkout owns — untouched;
|
||||
- readiness is `/api/ready` (database, writable data dir, storage
|
||||
metadata), never a TCP accept and never `/api/health`, which is
|
||||
liveness only;
|
||||
- the app runs detached with durable logs and a recorded stop handle,
|
||||
so closing the terminal does not decide the instance's lifetime.
|
||||
|
||||
odysseus dev up # boot this worktree, print URL + stop handle
|
||||
odysseus dev up --from-pr 42 # fetch PR 42 into a worktree and boot that
|
||||
odysseus dev status # what is running here (JSON)
|
||||
odysseus dev down # stop what `up` started here
|
||||
odysseus dev ports # the derived port set (JSON)
|
||||
odysseus dev env # shell exports for running tests in this worktree
|
||||
|
||||
Every subcommand acts on the checkout containing the current working
|
||||
directory, so a single copy on $PATH serves every worktree.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "_lib"))
|
||||
from cli import quiet_logs, emit, fail, common_parser, run # noqa: E402
|
||||
|
||||
quiet_logs()
|
||||
|
||||
import hashlib # noqa: E402
|
||||
import json # noqa: E402
|
||||
import signal # noqa: E402
|
||||
import socket # noqa: E402
|
||||
import subprocess # noqa: E402
|
||||
import time # noqa: E402
|
||||
import urllib.error # noqa: E402
|
||||
import urllib.request # noqa: E402
|
||||
from pathlib import Path # noqa: E402
|
||||
|
||||
# Everything this tool writes lives under one directory inside the
|
||||
# worktree, next to but never inside `data/` — a dev boot must not be
|
||||
# able to corrupt the data dir a normal `start-macos.sh` run owns.
|
||||
DEV_DIR_NAME = ".odysseus-dev"
|
||||
STATE_FILE_NAME = "run.json"
|
||||
|
||||
# Files that identify a checkout root, so `odysseus dev` from any
|
||||
# subdirectory finds the worktree it belongs to.
|
||||
ROOT_MARKERS = ("app.py", "setup.py", "requirements.txt")
|
||||
|
||||
# Port block derivation. Three consecutive ports per worktree (app,
|
||||
# ChromaDB, test static server) starting at 7200; the last block ends at
|
||||
# 7799. The range is chosen to exclude every port the project already
|
||||
# means something by, so a derived port can never collide with a normal
|
||||
# launch on the same machine.
|
||||
PORT_BLOCK_BASE = 7200
|
||||
PORT_BLOCK_COUNT = 200
|
||||
PORTS_PER_BLOCK = 3
|
||||
|
||||
# Ports this tool refuses to use even when asked explicitly, with the
|
||||
# reason each one is spoken for.
|
||||
RESERVED_PORTS = {
|
||||
7000: "the historical app default (and macOS AirPlay Receiver)",
|
||||
7011: "the app's own default bind and the compose APP_PORT",
|
||||
7860: "start-macos.sh's default, i.e. a normal launch of this app",
|
||||
8100: "the default CHROMADB_PORT, i.e. someone else's vector store",
|
||||
}
|
||||
|
||||
READY_PATH = "/api/ready"
|
||||
HEALTH_PATH = "/api/health"
|
||||
LOGIN_PATH = "/api/auth/login"
|
||||
SESSION_COOKIE = "odysseus_session"
|
||||
DEFAULT_READY_TIMEOUT = 180
|
||||
STOP_GRACE_SECONDS = 10
|
||||
|
||||
# `/api/ready` is not in app.py's AUTH_EXEMPT_EXACT set, so readiness is
|
||||
# only observable with a session. The launcher therefore owns the dev
|
||||
# admin account: it generates the password once, hands it to setup.py,
|
||||
# keeps it here, and prints it — otherwise a generated password scrolls
|
||||
# past on first boot and the instance is unusable afterwards.
|
||||
CREDENTIALS_FILE_NAME = "admin.json"
|
||||
VENV_FILE_NAME = "venv-path"
|
||||
DEV_ADMIN_USER = "admin"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Locating the worktree
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def find_repo_root(start):
|
||||
"""Return the checkout root at or above `start`, or None.
|
||||
|
||||
Resolved from the working directory rather than from this file, so a
|
||||
symlink on $PATH still boots the worktree the user is standing in.
|
||||
"""
|
||||
current = Path(start).resolve()
|
||||
for candidate in [current, *current.parents]:
|
||||
if all((candidate / marker).exists() for marker in ROOT_MARKERS):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def dev_dir(root):
|
||||
return Path(root) / DEV_DIR_NAME
|
||||
|
||||
|
||||
def state_path(root):
|
||||
return dev_dir(root) / STATE_FILE_NAME
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Ports
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def derive_ports(root):
|
||||
"""Map a worktree path to its own block of three ports.
|
||||
|
||||
Deterministic: the same checkout gets the same ports on every run, so
|
||||
a bookmarked URL keeps working, and two checkouts only collide if
|
||||
their paths hash into the same block — which `up` detects and refuses
|
||||
rather than papers over.
|
||||
"""
|
||||
digest = hashlib.blake2s(str(Path(root).resolve()).encode("utf-8"), digest_size=8).digest()
|
||||
block = int.from_bytes(digest, "big") % PORT_BLOCK_COUNT
|
||||
base = PORT_BLOCK_BASE + block * PORTS_PER_BLOCK
|
||||
return {"app": base, "chroma": base + 1, "test_static": base + 2}
|
||||
|
||||
|
||||
def reserved_reason(port):
|
||||
"""Return why `port` is off limits, or None if it is usable."""
|
||||
return RESERVED_PORTS.get(int(port))
|
||||
|
||||
|
||||
def port_bound(port, host="127.0.0.1", timeout=0.4):
|
||||
"""True if something already accepts connections on host:port."""
|
||||
try:
|
||||
with socket.create_connection((host, int(port)), timeout=timeout):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def unused_port():
|
||||
"""Ask the OS for a free port and release it immediately.
|
||||
|
||||
Used only to point CHROMADB_PORT at something that will refuse the
|
||||
connection, which is how the app falls back to keyword mode.
|
||||
"""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return sock.getsockname()[1]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Refusing to boot where a real instance lives
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def service_unit_dirs():
|
||||
home = Path.home()
|
||||
if sys.platform == "darwin":
|
||||
return [
|
||||
home / "Library" / "LaunchAgents",
|
||||
Path("/Library/LaunchAgents"),
|
||||
Path("/Library/LaunchDaemons"),
|
||||
]
|
||||
return [
|
||||
home / ".config" / "systemd" / "user",
|
||||
Path("/etc/systemd/system"),
|
||||
Path("/usr/lib/systemd/system"),
|
||||
]
|
||||
|
||||
|
||||
def managed_by_service(root, unit_dirs=None):
|
||||
"""Return the unit file naming a path inside `root`, or None.
|
||||
|
||||
A checkout wired into launchd or systemd is somebody's running
|
||||
instance: booting a second process out of it would share its source
|
||||
tree and, on the first mistake, its data. We refuse rather than trust
|
||||
the user to remember which directory this is. The match is on the
|
||||
path, so a unit pointing anywhere inside the checkout counts.
|
||||
"""
|
||||
needle = str(Path(root).resolve())
|
||||
for directory in unit_dirs if unit_dirs is not None else service_unit_dirs():
|
||||
try:
|
||||
entries = sorted(Path(directory).iterdir())
|
||||
except OSError:
|
||||
continue
|
||||
for entry in entries:
|
||||
if entry.suffix not in (".plist", ".service"):
|
||||
continue
|
||||
try:
|
||||
text = entry.read_text(errors="ignore")
|
||||
except OSError:
|
||||
continue
|
||||
if needle in text:
|
||||
return str(entry)
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Process ownership
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def pid_command(pid):
|
||||
"""Return the full command line of `pid`, or "" if it is not ours to see."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["ps", "-o", "command=", "-p", str(int(pid))],
|
||||
capture_output=True, text=True, timeout=5, check=False,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError, ValueError):
|
||||
return ""
|
||||
return result.stdout.strip() if result.returncode == 0 else ""
|
||||
|
||||
|
||||
def pid_is_ours(pid, fingerprints):
|
||||
"""True only when `pid` is alive AND its command line still shows every
|
||||
fingerprint we recorded when we started it.
|
||||
|
||||
A pid alone proves nothing — the number is reused. Everything that
|
||||
kills or adopts a process goes through here.
|
||||
"""
|
||||
if not pid:
|
||||
return False
|
||||
command = pid_command(pid)
|
||||
if not command:
|
||||
return False
|
||||
return all(str(mark) in command for mark in fingerprints)
|
||||
|
||||
|
||||
def pid_alive(pid):
|
||||
try:
|
||||
os.kill(int(pid), 0)
|
||||
except (OSError, TypeError, ValueError):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def credentials(root):
|
||||
"""Return this worktree's dev admin account, generating it once.
|
||||
|
||||
Stored outside the data dir so `down`, a wiped database, or a fresh
|
||||
`up` all keep the same login.
|
||||
"""
|
||||
path = dev_dir(root) / CREDENTIALS_FILE_NAME
|
||||
try:
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
import secrets
|
||||
|
||||
account = {"username": DEV_ADMIN_USER, "password": secrets.token_urlsafe(18)}
|
||||
dev_dir(root).mkdir(parents=True, exist_ok=True)
|
||||
with open(os.open(path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), "w",
|
||||
encoding="utf-8") as handle:
|
||||
json.dump(account, handle, indent=2)
|
||||
return account
|
||||
|
||||
|
||||
def read_state(root):
|
||||
try:
|
||||
with open(state_path(root), encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def write_state(root, state):
|
||||
dev_dir(root).mkdir(parents=True, exist_ok=True)
|
||||
with open(state_path(root), "w", encoding="utf-8") as handle:
|
||||
json.dump(state, handle, indent=2)
|
||||
|
||||
|
||||
def running_app(state):
|
||||
"""Return the recorded app entry if that exact process is still alive."""
|
||||
app = (state or {}).get("app") or {}
|
||||
if pid_is_ours(app.get("pid"), app.get("fingerprints") or []):
|
||||
return app
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# ChromaDB
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def start_chroma(venv_python, port, chroma_path, log_path):
|
||||
"""Start our own ChromaDB, or explain why we are going without one.
|
||||
|
||||
Returns (entry_or_None, note). Adopting a foreign server is not one of
|
||||
the outcomes: the caller has already established that the port is free.
|
||||
"""
|
||||
binary = Path(venv_python).parent / "chroma"
|
||||
if not binary.exists():
|
||||
return None, (
|
||||
"keyword-only mode: no `chroma` binary in this venv "
|
||||
"(requirements.txt pins chromadb-client, the HTTP client). "
|
||||
f"Install the server with `{venv_python} -m pip install chromadb` to enable vectors."
|
||||
)
|
||||
chroma_path.mkdir(parents=True, exist_ok=True)
|
||||
command = [
|
||||
str(binary), "run",
|
||||
"--host", "127.0.0.1",
|
||||
"--port", str(port),
|
||||
"--path", str(chroma_path),
|
||||
]
|
||||
with open(log_path, "ab") as log:
|
||||
process = subprocess.Popen(
|
||||
command, stdout=log, stderr=subprocess.STDOUT,
|
||||
start_new_session=True, cwd=str(chroma_path.parent),
|
||||
)
|
||||
entry = {
|
||||
"pid": process.pid,
|
||||
"port": port,
|
||||
"path": str(chroma_path),
|
||||
"log": str(log_path),
|
||||
# The data path is the identity: it is unique to this worktree
|
||||
# and appears in the command line whichever way ps resolves the
|
||||
# console script.
|
||||
"fingerprints": ["chroma", str(chroma_path)],
|
||||
}
|
||||
return entry, f"own server on 127.0.0.1:{port} against {chroma_path}"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Readiness
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def http_json(port, path, payload=None, cookie=None, timeout=5.0):
|
||||
"""One request against the local instance. Returns (status, body)."""
|
||||
url = f"http://127.0.0.1:{port}{path}"
|
||||
data = json.dumps(payload).encode("utf-8") if payload is not None else None
|
||||
headers = {"Content-Type": "application/json"} if data else {}
|
||||
if cookie:
|
||||
headers["Cookie"] = f"{SESSION_COOKIE}={cookie}"
|
||||
request = urllib.request.Request(url, data=data, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return response.status, _decode(response.read()), response
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, _decode(exc.read()), exc
|
||||
except (OSError, ValueError) as exc:
|
||||
return 0, {"error": str(exc)}, None
|
||||
|
||||
|
||||
def _decode(raw):
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return {}
|
||||
|
||||
|
||||
def login(port, account):
|
||||
"""Return a session cookie for the dev admin, or None."""
|
||||
status, _, response = http_json(port, LOGIN_PATH, payload={
|
||||
"username": account["username"], "password": account["password"],
|
||||
})
|
||||
if status != 200 or response is None:
|
||||
return None
|
||||
for header in response.headers.get_all("Set-Cookie") or []:
|
||||
if header.startswith(f"{SESSION_COOKIE}="):
|
||||
return header.split(";", 1)[0].split("=", 1)[1]
|
||||
return None
|
||||
|
||||
|
||||
def probe_ready(port, cookie=None, timeout=5.0):
|
||||
"""GET /api/ready once. Returns (ready, payload).
|
||||
|
||||
/api/health only proves the process is alive. /api/ready is the one
|
||||
that checks the database, a writable data dir and storage metadata,
|
||||
and it answers 503 until all three hold — which is why a TCP accept
|
||||
is not what this tool waits for.
|
||||
"""
|
||||
status, body, _ = http_json(port, READY_PATH, cookie=cookie, timeout=timeout)
|
||||
body = dict(body or {})
|
||||
body.setdefault("status", status)
|
||||
return bool(body.get("ready")), body
|
||||
|
||||
|
||||
def wait_ready(port, process, timeout, log_path, account):
|
||||
"""Wait for liveness, authenticate, then wait for real readiness."""
|
||||
deadline = time.monotonic() + timeout
|
||||
|
||||
def alive():
|
||||
if process is not None and process.poll() is not None:
|
||||
fail(
|
||||
f"the app exited with code {process.returncode} before becoming ready.\n"
|
||||
f" last lines of {log_path}:\n{tail(log_path, 20)}"
|
||||
)
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
alive()
|
||||
if http_json(port, HEALTH_PATH, timeout=2.0)[0] == 200:
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
# One login, not one per poll: the login route is rate limited.
|
||||
cookie, last = None, {}
|
||||
while time.monotonic() < deadline and cookie is None:
|
||||
alive()
|
||||
cookie = login(port, account)
|
||||
if cookie is None:
|
||||
time.sleep(3)
|
||||
if cookie is None:
|
||||
fail(
|
||||
f"could not log in as {account['username']} to read {READY_PATH}.\n"
|
||||
f" The recorded credentials may not match this data dir. Remove "
|
||||
f"{Path(log_path).parent.parent / CREDENTIALS_FILE_NAME} and the data dir "
|
||||
f"to start clean.\n"
|
||||
f" The app is running; stop it with `odysseus dev down`."
|
||||
)
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
alive()
|
||||
ready, last = probe_ready(port, cookie=cookie)
|
||||
if ready:
|
||||
return last
|
||||
time.sleep(1)
|
||||
fail(
|
||||
f"{READY_PATH} did not report ready within {timeout}s.\n"
|
||||
f" last response: {json.dumps(last, default=str)[:400]}\n"
|
||||
f" the app is still running; logs: {log_path}\n"
|
||||
f" stop it with `odysseus dev down`"
|
||||
)
|
||||
|
||||
|
||||
def tail(path, lines):
|
||||
try:
|
||||
with open(path, encoding="utf-8", errors="replace") as handle:
|
||||
return "".join(f" {line}" for line in handle.readlines()[-lines:])
|
||||
except OSError:
|
||||
return " (no log)"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# git helpers for --from-pr
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def git(root, *args, check=True):
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), *args],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
if check and result.returncode != 0:
|
||||
fail(f"git {' '.join(args)} failed: {result.stderr.strip()}")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def worktree_for_pr(root, number, remote):
|
||||
"""Fetch a pull request head into its own worktree and return its path.
|
||||
|
||||
`pull/<n>/head` is served by the repository the PR targets, so this
|
||||
works for forks without knowing anything about the fork layout.
|
||||
"""
|
||||
target = Path(root).resolve().parent / f"{Path(root).resolve().name}-pr{number}"
|
||||
if target.exists():
|
||||
sys.stdout.write(f" worktree for PR {number} already exists at {target}\n")
|
||||
return target
|
||||
git(root, "fetch", remote, f"pull/{number}/head")
|
||||
head = git(root, "rev-parse", "FETCH_HEAD")
|
||||
git(root, "worktree", "add", "--detach", str(target), head)
|
||||
sys.stdout.write(f" PR {number} checked out at {target} ({head[:8]})\n")
|
||||
return target
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Commands
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def resolve_root(args):
|
||||
root = find_repo_root(Path.cwd())
|
||||
if root is None:
|
||||
fail(
|
||||
"not inside an Odysseus checkout "
|
||||
f"(looked for {', '.join(ROOT_MARKERS)} from {Path.cwd()} upwards)",
|
||||
code=2,
|
||||
)
|
||||
return root
|
||||
|
||||
|
||||
def resolve_ports(root, args):
|
||||
ports = derive_ports(root)
|
||||
for name, override in (("app", getattr(args, "port", None)),
|
||||
("chroma", getattr(args, "chroma_port", None))):
|
||||
if override:
|
||||
ports[name] = int(override)
|
||||
for name, port in ports.items():
|
||||
reason = reserved_reason(port)
|
||||
if reason:
|
||||
fail(f"port {port} is {reason}; refusing to use it as the {name} port")
|
||||
return ports
|
||||
|
||||
|
||||
def remembered_venv(root):
|
||||
"""The venv a previous `up` borrowed for this worktree, if any."""
|
||||
try:
|
||||
return Path(dev_dir(root).joinpath(VENV_FILE_NAME).read_text(encoding="utf-8").strip())
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def remember_venv(root, venv_root):
|
||||
dev_dir(root).mkdir(parents=True, exist_ok=True)
|
||||
dev_dir(root).joinpath(VENV_FILE_NAME).write_text(str(venv_root), encoding="utf-8")
|
||||
|
||||
|
||||
def resolve_venv(root, args):
|
||||
"""Pick the interpreter to run the app with. This tool never builds a
|
||||
venv — `--venv` pointing at a sibling worktree's environment is what
|
||||
makes booting a PR take seconds rather than minutes, and the choice is
|
||||
remembered so the next `up` in that worktree does not need the flag."""
|
||||
candidates = [
|
||||
Path(args.venv).expanduser().resolve() if args.venv else None,
|
||||
Path(root) / "venv",
|
||||
remembered_venv(root),
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate and (candidate / "bin" / "python").exists():
|
||||
return candidate / "bin" / "python"
|
||||
if args.venv:
|
||||
fail(f"no interpreter at {Path(args.venv).expanduser().resolve() / 'bin' / 'python'}")
|
||||
fail(
|
||||
f"no venv at {Path(root) / 'venv'}.\n"
|
||||
f" build one with ./start-macos.sh, or reuse another worktree's "
|
||||
f"with --venv /path/to/worktree/venv"
|
||||
)
|
||||
|
||||
|
||||
def refuse_if_taken(root, args, ports, state):
|
||||
"""Stop before anything is started if this worktree cannot own the boot."""
|
||||
unit = managed_by_service(root)
|
||||
if unit:
|
||||
fail(
|
||||
f"{root} is run as a service by {unit}.\n"
|
||||
f" That is a real instance, not a scratch worktree. Boot a separate "
|
||||
f"checkout instead:\n"
|
||||
f" git worktree add ../odysseus-dev <ref> && cd ../odysseus-dev"
|
||||
)
|
||||
if port_bound(ports["app"]):
|
||||
fail(
|
||||
f"port {ports['app']} is already in use by a process we do not own.\n"
|
||||
f" This worktree derives that port from its path, so something else "
|
||||
f"took it.\n"
|
||||
f" Re-run with --port <n> to pick another."
|
||||
)
|
||||
|
||||
|
||||
def resolve_chroma(args, state, ports, venv_python, data_dir, log_dir):
|
||||
"""Decide what this worktree talks to for vectors.
|
||||
|
||||
Returns (state_entry, port, note). The one outcome this never
|
||||
produces is a port somebody else is serving: the whole tool exists
|
||||
because `start-macos.sh` treats that as a resource to adopt.
|
||||
"""
|
||||
if args.no_chroma:
|
||||
# Point at a port nothing is listening on rather than at the
|
||||
# derived one, which may be exactly the foreign server we are
|
||||
# refusing to touch. Connection refused is what makes the app
|
||||
# fall back to keyword search.
|
||||
return None, unused_port(), "disabled by --no-chroma"
|
||||
|
||||
if port_bound(ports["chroma"]):
|
||||
ours = (state or {}).get("chroma") or {}
|
||||
if pid_is_ours(ours.get("pid"), ours.get("fingerprints") or []):
|
||||
return ours, ours["port"], f"reusing the server we started earlier on {ours['port']}"
|
||||
fail(
|
||||
f"port {ports['chroma']} is serving a ChromaDB this worktree did not start.\n"
|
||||
f" Adopting it would read and write another checkout's vectors, so we "
|
||||
f"will not.\n"
|
||||
f" Re-run with --chroma-port <n>, or with --no-chroma to run in "
|
||||
f"keyword-only mode."
|
||||
)
|
||||
|
||||
entry, note = start_chroma(
|
||||
venv_python, ports["chroma"], data_dir / "chroma", log_dir / "chroma.log"
|
||||
)
|
||||
# If we could not start one, point the app at a port nothing is on
|
||||
# rather than at our derived one: otherwise a ChromaDB that binds
|
||||
# that port later would be adopted by a running app, which is the
|
||||
# exact failure this tool exists to prevent.
|
||||
return entry, (ports["chroma"] if entry else unused_port()), note
|
||||
|
||||
|
||||
def boot_environment(account, ports, chroma_port, data_dir):
|
||||
"""The environment that makes the child process this worktree's own."""
|
||||
env = dict(os.environ)
|
||||
env.update({
|
||||
"ODYSSEUS_ADMIN_USER": account["username"],
|
||||
"ODYSSEUS_ADMIN_PASSWORD": account["password"],
|
||||
"APP_PORT": str(ports["app"]),
|
||||
"APP_BIND": "127.0.0.1",
|
||||
"ODYSSEUS_DATA_DIR": str(data_dir),
|
||||
"DATABASE_URL": f"sqlite:///{data_dir / 'app.db'}",
|
||||
# src/builtin_mcp.py derives this cache from a literal "data"
|
||||
# under the app root rather than from DATA_DIR, so without an
|
||||
# explicit value a dev boot would write into the checkout's
|
||||
# data/ after all. Pointing it at our own dir keeps the
|
||||
# isolation claim true.
|
||||
"ODYSSEUS_BROWSER_MCP_CACHE": str(data_dir / "playwright-mcp-cache"),
|
||||
"CHROMADB_HOST": "127.0.0.1",
|
||||
"CHROMADB_PORT": str(chroma_port),
|
||||
"ODYSSEUS_TEST_STATIC_PORT": str(ports["test_static"]),
|
||||
"ODYSSEUS_NO_OPEN": "1",
|
||||
"ODYSSEUS_SKIP_RUN_HINT": "1",
|
||||
"ODYSSEUS_SKIP_ADMIN_PROMPT": "1",
|
||||
})
|
||||
return env
|
||||
|
||||
|
||||
def run_setup(root, venv_python, env, data_dir, log_dir):
|
||||
"""Create the data dir, database and admin account. Idempotent."""
|
||||
sys.stdout.write(f" preparing {data_dir} (setup.py is idempotent)\n")
|
||||
setup = subprocess.run(
|
||||
[str(venv_python), "setup.py"], cwd=str(root), env=env,
|
||||
capture_output=True, text=True, stdin=subprocess.DEVNULL, check=False,
|
||||
)
|
||||
log = log_dir / "setup.log"
|
||||
with open(log, "w", encoding="utf-8") as handle:
|
||||
handle.write(setup.stdout + setup.stderr)
|
||||
if setup.returncode != 0:
|
||||
fail(f"setup.py failed; see {log}\n{tail(log, 15)}")
|
||||
|
||||
|
||||
def borrow_venv_for_pr(root, args):
|
||||
"""A fresh PR worktree has no venv; the one we came from will do."""
|
||||
if args.venv or (root / "venv" / "bin" / "python").exists():
|
||||
return
|
||||
source_venv = find_repo_root(Path.cwd()) / "venv"
|
||||
if (source_venv / "bin" / "python").exists():
|
||||
args.venv = str(source_venv)
|
||||
sys.stdout.write(f" reusing {source_venv} (the PR worktree has none)\n")
|
||||
|
||||
|
||||
def cmd_up(args):
|
||||
root = resolve_root(args)
|
||||
if args.from_pr:
|
||||
root = worktree_for_pr(root, args.from_pr, args.remote)
|
||||
borrow_venv_for_pr(root, args)
|
||||
|
||||
state = read_state(root)
|
||||
already = running_app(state)
|
||||
if already:
|
||||
sys.stdout.write(
|
||||
f"already up: http://127.0.0.1:{already['port']} (pid {already['pid']})\n"
|
||||
f"stop it with `odysseus dev down`, or re-run after that to restart.\n"
|
||||
)
|
||||
return
|
||||
|
||||
ports = resolve_ports(root, args)
|
||||
refuse_if_taken(root, args, ports, state)
|
||||
venv_python = resolve_venv(root, args)
|
||||
remember_venv(root, venv_python.parent.parent)
|
||||
|
||||
data_dir = dev_dir(root) / "data"
|
||||
log_dir = dev_dir(root) / "logs"
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
app_log = log_dir / "app.log"
|
||||
|
||||
chroma_entry, chroma_port, chroma_note = resolve_chroma(
|
||||
args, state, ports, venv_python, data_dir, log_dir
|
||||
)
|
||||
account = credentials(root)
|
||||
env = boot_environment(account, ports, chroma_port, data_dir)
|
||||
run_setup(root, venv_python, env, data_dir, log_dir)
|
||||
|
||||
command = [
|
||||
str(venv_python), "-m", "uvicorn", "app:app",
|
||||
"--host", "127.0.0.1", "--port", str(ports["app"]),
|
||||
]
|
||||
if args.foreground:
|
||||
sys.stdout.write(f" starting in the foreground on http://127.0.0.1:{ports['app']}\n")
|
||||
os.execve(str(venv_python), command, env)
|
||||
|
||||
with open(app_log, "ab") as log:
|
||||
process = subprocess.Popen(
|
||||
command, cwd=str(root), env=env, stdout=log, stderr=subprocess.STDOUT,
|
||||
stdin=subprocess.DEVNULL, start_new_session=True,
|
||||
)
|
||||
|
||||
state = {
|
||||
"root": str(root),
|
||||
"started_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"commit": git(root, "rev-parse", "--short", "HEAD", check=False),
|
||||
"branch": git(root, "rev-parse", "--abbrev-ref", "HEAD", check=False),
|
||||
"venv": str(Path(venv_python).parent.parent),
|
||||
"data_dir": str(data_dir),
|
||||
"ports": ports,
|
||||
"app": {
|
||||
"pid": process.pid,
|
||||
"port": ports["app"],
|
||||
"log": str(app_log),
|
||||
# The interpreter path is not one of these on purpose: macOS
|
||||
# reports the framework binary a venv symlinks to, not the
|
||||
# venv path we launched. The port is derived per worktree, so
|
||||
# it is the part that actually identifies this instance.
|
||||
"fingerprints": ["uvicorn", "app:app", f"--port {ports['app']}"],
|
||||
},
|
||||
"chroma": chroma_entry,
|
||||
}
|
||||
write_state(root, state)
|
||||
|
||||
sys.stdout.write(f" waiting for {READY_PATH} (up to {args.timeout}s)\n")
|
||||
report = wait_ready(ports["app"], process, args.timeout, app_log, account)
|
||||
|
||||
sys.stdout.write(
|
||||
f"\nOdysseus is up — this worktree only.\n\n"
|
||||
f" URL http://127.0.0.1:{ports['app']}\n"
|
||||
f" Login {account['username']} / {account['password']}\n"
|
||||
f" Worktree {root} ({state['branch']} @ {state['commit']})\n"
|
||||
f" Data dir {data_dir}\n"
|
||||
f" ChromaDB {chroma_note}\n"
|
||||
f" Test port {ports['test_static']} (ODYSSEUS_TEST_STATIC_PORT; see `odysseus dev env`)\n"
|
||||
f" Logs {app_log}\n"
|
||||
f" Ready {json.dumps({k: v.get('ok') for k, v in report.get('checks', {}).items()})}\n"
|
||||
f" Stop with odysseus dev down\n"
|
||||
)
|
||||
|
||||
|
||||
def cmd_down(args):
|
||||
root = resolve_root(args)
|
||||
state = read_state(root)
|
||||
stopped, unclaimed = [], []
|
||||
for name in ("app", "chroma"):
|
||||
entry = (state or {}).get(name) or {}
|
||||
pid = entry.get("pid")
|
||||
if not pid_is_ours(pid, entry.get("fingerprints") or []):
|
||||
if pid_alive(pid):
|
||||
# Alive but no longer recognisable: signalling it would be
|
||||
# signalling a stranger. Say so and keep the record.
|
||||
unclaimed.append(f"{name} (pid {pid})")
|
||||
continue
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
deadline = time.monotonic() + STOP_GRACE_SECONDS
|
||||
while time.monotonic() < deadline and pid_is_ours(pid, entry.get("fingerprints") or []):
|
||||
time.sleep(0.2)
|
||||
if pid_is_ours(pid, entry.get("fingerprints") or []):
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
stopped.append(f"{name} (pid {pid})")
|
||||
if stopped:
|
||||
sys.stdout.write(f"stopped {', '.join(stopped)}.\n")
|
||||
elif not unclaimed:
|
||||
sys.stdout.write("nothing this worktree started is still running.\n")
|
||||
if unclaimed:
|
||||
sys.stdout.write(
|
||||
f"left alone: {', '.join(unclaimed)} — still alive but no longer matching "
|
||||
f"what we recorded. Check it before killing it; {state_path(root)} is kept.\n"
|
||||
)
|
||||
return
|
||||
try:
|
||||
state_path(root).unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def cmd_status(args):
|
||||
root = resolve_root(args)
|
||||
state = read_state(root)
|
||||
app = running_app(state)
|
||||
chroma = (state or {}).get("chroma") or {}
|
||||
ready = False
|
||||
if app:
|
||||
ready = probe_ready(app["port"], cookie=login(app["port"], credentials(root)))[0]
|
||||
emit({
|
||||
"root": str(root),
|
||||
"running": bool(app),
|
||||
"url": f"http://127.0.0.1:{app['port']}" if app else None,
|
||||
"ready": ready,
|
||||
"chroma_running": pid_is_ours(chroma.get("pid"), chroma.get("fingerprints") or []),
|
||||
"ports": (state or {}).get("ports") or derive_ports(root),
|
||||
"state_file": str(state_path(root)),
|
||||
}, args)
|
||||
|
||||
|
||||
def cmd_ports(args):
|
||||
root = resolve_root(args)
|
||||
ports = derive_ports(root)
|
||||
emit({
|
||||
"root": str(root),
|
||||
"ports": ports,
|
||||
"in_use": {name: port_bound(port) for name, port in ports.items()},
|
||||
}, args)
|
||||
|
||||
|
||||
def cmd_env(args):
|
||||
"""Print the isolated environment as shell exports, so a test run in
|
||||
this worktree uses the same ports and data dir the app does."""
|
||||
root = resolve_root(args)
|
||||
ports = derive_ports(root)
|
||||
data = dev_dir(root) / "data"
|
||||
for key, value in (
|
||||
("APP_PORT", ports["app"]),
|
||||
("ODYSSEUS_TEST_STATIC_PORT", ports["test_static"]),
|
||||
("CHROMADB_PORT", ports["chroma"]),
|
||||
("ODYSSEUS_DATA_DIR", data),
|
||||
("DATABASE_URL", f"sqlite:///{data / 'app.db'}"),
|
||||
):
|
||||
sys.stdout.write(f"export {key}={value}\n")
|
||||
|
||||
|
||||
def build_parser():
|
||||
parser = common_parser("odysseus-dev", "Boot this worktree in isolation.")
|
||||
common = parser._common_parents[0]
|
||||
sub = parser.add_subparsers(dest="cmd")
|
||||
|
||||
up = sub.add_parser("up", parents=[common], help="boot this worktree")
|
||||
up.add_argument("--port", type=int, help="override the derived app port")
|
||||
up.add_argument("--chroma-port", type=int, help="override the derived ChromaDB port")
|
||||
up.add_argument("--no-chroma", action="store_true",
|
||||
help="run without vectors (keyword mode) instead of starting a server")
|
||||
up.add_argument("--venv", help="use this venv instead of ./venv (e.g. a sibling worktree's)")
|
||||
up.add_argument("--from-pr", type=int, metavar="N",
|
||||
help="fetch pull request N into its own worktree and boot that")
|
||||
up.add_argument("--remote", default="origin", help="remote to fetch the PR from")
|
||||
up.add_argument("--foreground", action="store_true",
|
||||
help="run uvicorn in this terminal instead of detaching")
|
||||
up.add_argument("--timeout", type=int, default=DEFAULT_READY_TIMEOUT,
|
||||
help=f"seconds to wait for {READY_PATH} (default: {DEFAULT_READY_TIMEOUT})")
|
||||
up.set_defaults(func=cmd_up)
|
||||
|
||||
down = sub.add_parser("down", parents=[common], help="stop what `up` started here")
|
||||
down.set_defaults(func=cmd_down)
|
||||
|
||||
status = sub.add_parser("status", parents=[common], help="what is running in this worktree")
|
||||
status.set_defaults(func=cmd_status)
|
||||
|
||||
ports = sub.add_parser("ports", parents=[common], help="the derived port set")
|
||||
ports.set_defaults(func=cmd_ports)
|
||||
|
||||
env = sub.add_parser("env", parents=[common], help="shell exports for this worktree")
|
||||
env.set_defaults(func=cmd_env)
|
||||
|
||||
parser.set_defaults(func=lambda args: parser.print_help())
|
||||
return parser
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run(build_parser()))
|
||||
@@ -2,7 +2,7 @@
|
||||
"""odysseus-mail — Unix-style command-line wrapper around the email
|
||||
backend that powers the web UI.
|
||||
|
||||
Calls the same helpers `routes/email_helpers.py` exports, so a request
|
||||
Calls the same helpers `routes/email/email_helpers.py` exports, so a request
|
||||
issued from the shell hits IMAP/SMTP through the same connection pool
|
||||
and the same parsing pipeline as the HTTP routes. State is shared via
|
||||
`data/app.db` and `data/.app_key` (passwords decrypt automatically).
|
||||
|
||||
Executable
+209
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""odysseus-smoke — boot this worktree and drive every advertised feature area once.
|
||||
|
||||
The decomposition work has two safety nets and neither one covers the
|
||||
product: the checkpoint benchmark measures the agent runtime, and the
|
||||
computed-style snapshot pins the CSS. Nothing checked that Notes,
|
||||
Calendar, Documents, Email, Memory, Cookbook or Settings still worked
|
||||
after a route package moved or a 17,000-line module was split. This is
|
||||
that check, and it is deliberately shallow: one scenario per area,
|
||||
asserting a user-visible outcome rather than an HTTP 200.
|
||||
|
||||
It owns no instance logic. `odysseus dev` already isolates the ports,
|
||||
the data dir and ChromaDB per worktree, so this boots through it, hands
|
||||
the details to pytest in the environment, and stops what it started.
|
||||
|
||||
odysseus smoke # boot, run every area, stop again
|
||||
odysseus smoke --keep-up # leave the instance running afterwards
|
||||
odysseus smoke --no-boot # drive whatever is already up here
|
||||
odysseus smoke --restart # stop a running instance and boot fresh
|
||||
odysseus smoke --areas # print the coverage table without running
|
||||
odysseus smoke -- -k notes # everything after -- goes to pytest
|
||||
|
||||
The report is a per-area table, printed by the suite itself, listing the
|
||||
areas it does not cover next to the ones it does. An area with no
|
||||
scenario shows up as NOT RUN rather than going missing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "_lib"))
|
||||
from cli import quiet_logs, fail, common_parser, run # noqa: E402
|
||||
|
||||
quiet_logs()
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent
|
||||
REPO_ROOT = SCRIPTS_DIR.parent
|
||||
|
||||
# The launcher this tool delegates every instance decision to.
|
||||
DEV_SCRIPT = "odysseus-dev"
|
||||
|
||||
# What pytest is pointed at, relative to the checkout root.
|
||||
SMOKE_SUITE = "tests/smoke"
|
||||
|
||||
# Email is the one area with no reachable real backend, and the repo
|
||||
# already has a deterministic path for it. Turning it on is the reason
|
||||
# this tool owns the boot rather than leaving it to the caller: the flag
|
||||
# is read inside the app's process, so it has to be in the environment
|
||||
# the app is started with.
|
||||
EMAIL_FIXTURE_ENV = "ODYSSEUS_EMAIL_FIXTURE"
|
||||
|
||||
|
||||
def load_dev():
|
||||
"""Import `scripts/odysseus-dev` as a module.
|
||||
|
||||
Same loader the CLI tests use. Delegating by import rather than by
|
||||
parsing `odysseus dev env` output means the port derivation and the
|
||||
credential handling have exactly one implementation.
|
||||
"""
|
||||
path = SCRIPTS_DIR / DEV_SCRIPT
|
||||
if not path.exists():
|
||||
fail(f"{path} is missing; this tool boots through it.", code=2)
|
||||
loader = importlib.machinery.SourceFileLoader("odysseus_dev_cli", str(path))
|
||||
spec = importlib.util.spec_from_loader(loader.name, loader)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def suite_environment(dev, root, ports, account):
|
||||
"""The environment the smoke suite reads its target instance from.
|
||||
|
||||
Deliberately the same values `odysseus dev env` prints, plus the dev
|
||||
admin account, so a manual `pytest tests/smoke` under
|
||||
`eval $(odysseus dev env)` behaves the way this tool does.
|
||||
"""
|
||||
data_dir = dev.dev_dir(root) / "data"
|
||||
env = dict(os.environ)
|
||||
env.update({
|
||||
"APP_PORT": str(ports["app"]),
|
||||
"CHROMADB_PORT": str(ports["chroma"]),
|
||||
"ODYSSEUS_TEST_STATIC_PORT": str(ports["test_static"]),
|
||||
"ODYSSEUS_DATA_DIR": str(data_dir),
|
||||
"DATABASE_URL": f"sqlite:///{data_dir / 'app.db'}",
|
||||
"ODYSSEUS_ADMIN_USER": account["username"],
|
||||
"ODYSSEUS_ADMIN_PASSWORD": account["password"],
|
||||
})
|
||||
return env
|
||||
|
||||
|
||||
def boot(dev, root, args):
|
||||
"""Start the instance, or adopt one already running in this worktree.
|
||||
|
||||
Returns (started_by_us, note). A reused instance is never restarted
|
||||
without being asked: it may be someone's debugging session, and the
|
||||
one thing it can cost us is the email fixture flag, which the suite
|
||||
reports as a skip rather than a pass.
|
||||
"""
|
||||
already = dev.running_app(dev.read_state(root))
|
||||
if already and args.restart:
|
||||
subprocess.run([sys.executable, str(SCRIPTS_DIR / DEV_SCRIPT), "down"],
|
||||
cwd=str(root), check=False)
|
||||
already = None
|
||||
if already:
|
||||
return False, (
|
||||
f"reusing the instance already up on port {already['port']} "
|
||||
f"(pid {already['pid']}). If it was not booted with "
|
||||
f"{EMAIL_FIXTURE_ENV}=1 the Email area will report a skip; "
|
||||
f"re-run with --restart for a clean boot."
|
||||
)
|
||||
if args.no_boot:
|
||||
fail(
|
||||
"nothing is running in this worktree and --no-boot was passed.\n"
|
||||
" boot it with `odysseus dev up`, or drop --no-boot.",
|
||||
)
|
||||
|
||||
command = [sys.executable, str(SCRIPTS_DIR / DEV_SCRIPT), "up"]
|
||||
if args.venv:
|
||||
command += ["--venv", args.venv]
|
||||
env = dict(os.environ)
|
||||
env[EMAIL_FIXTURE_ENV] = "1"
|
||||
result = subprocess.run(command, cwd=str(root), env=env, check=False)
|
||||
if result.returncode != 0:
|
||||
fail(f"`odysseus dev up` exited {result.returncode}; not running the suite.")
|
||||
return True, ""
|
||||
|
||||
|
||||
def venv_python(dev, root, args):
|
||||
"""The interpreter to run pytest with: the one the app runs under."""
|
||||
recorded = (dev.read_state(root) or {}).get("venv")
|
||||
for candidate in (Path(args.venv).expanduser() if args.venv else None,
|
||||
Path(recorded) if recorded else None,
|
||||
Path(root) / "venv"):
|
||||
if candidate and (candidate / "bin" / "python").exists():
|
||||
return candidate / "bin" / "python"
|
||||
fail(
|
||||
f"no interpreter found for the suite (looked at {Path(root) / 'venv'}).\n"
|
||||
f" build one with ./start-macos.sh, or pass --venv."
|
||||
)
|
||||
|
||||
|
||||
def cmd_run(args):
|
||||
dev = load_dev()
|
||||
root = dev.find_repo_root(Path.cwd())
|
||||
if root is None:
|
||||
fail(f"not inside an Odysseus checkout (looked upwards from {Path.cwd()})", code=2)
|
||||
|
||||
if args.areas:
|
||||
sys.path.insert(0, str(root))
|
||||
from tests.smoke import areas
|
||||
sys.stdout.write(areas.render_table({}, header="Odysseus release smoke - coverage") + "\n")
|
||||
return 0
|
||||
|
||||
ports = dev.derive_ports(root)
|
||||
account = dev.credentials(root)
|
||||
started_by_us, note = boot(dev, root, args)
|
||||
if note:
|
||||
sys.stdout.write(f" {note}\n")
|
||||
|
||||
python = venv_python(dev, root, args)
|
||||
env = suite_environment(dev, root, ports, account)
|
||||
command = [str(python), "-m", "pytest", SMOKE_SUITE, "-q"] + list(args.pytest_args)
|
||||
sys.stdout.write(f"\n running {SMOKE_SUITE} against http://127.0.0.1:{ports['app']}\n\n")
|
||||
# Flush before handing the terminal to pytest, or our own lines land
|
||||
# after its output and the report reads out of order.
|
||||
sys.stdout.flush()
|
||||
result = subprocess.run(command, cwd=str(root), env=env, check=False)
|
||||
|
||||
if started_by_us and not args.keep_up:
|
||||
subprocess.run([sys.executable, str(SCRIPTS_DIR / DEV_SCRIPT), "down"],
|
||||
cwd=str(root), check=False)
|
||||
elif started_by_us:
|
||||
sys.stdout.write(
|
||||
f"\n left running: http://127.0.0.1:{ports['app']} "
|
||||
f"({account['username']} / {account['password']})\n"
|
||||
f" stop it with `odysseus dev down`\n"
|
||||
)
|
||||
# `cli.run` discards a returned value but lets SystemExit through, and
|
||||
# a smoke run's exit code is the whole point of having one command.
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(result.returncode)
|
||||
return 0
|
||||
|
||||
|
||||
def build_parser():
|
||||
parser = common_parser("odysseus-smoke",
|
||||
"Boot this worktree and run the release smoke suite.")
|
||||
parser.add_argument("--keep-up", action="store_true",
|
||||
help="leave the instance running after the suite finishes")
|
||||
parser.add_argument("--no-boot", action="store_true",
|
||||
help="require an instance already up in this worktree")
|
||||
parser.add_argument("--restart", action="store_true",
|
||||
help="stop a running instance and boot a fresh one")
|
||||
parser.add_argument("--venv", help="use this venv instead of ./venv")
|
||||
parser.add_argument("--areas", action="store_true",
|
||||
help="print the coverage table and exit without booting")
|
||||
parser.add_argument("pytest_args", nargs="*", metavar="-- PYTEST ARGS",
|
||||
help="arguments forwarded to pytest after a literal --")
|
||||
parser.set_defaults(func=cmd_run)
|
||||
return parser
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run(build_parser()))
|
||||
Executable
+586
@@ -0,0 +1,586 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only audit of what one git ref carries that another does not.
|
||||
|
||||
Two long-lived lines that are not merged into each other drift silently. A fix
|
||||
landed on one of them leaves no mark on the other, and nothing in git tells you
|
||||
so: the two histories share only a distant merge base, so `git log A..B` lists
|
||||
thousands of commits whose content is in fact already present on both sides
|
||||
under different SHAs.
|
||||
|
||||
This script answers the question that actually matters at release time -- which
|
||||
commits on the source ref left *no trace at all* in the target ref -- by
|
||||
sampling distinctive added lines from each commit and searching the target tree
|
||||
for them. It also reports the file-level presence diff, which catches the case
|
||||
the line sampling cannot: a fix whose production change was reproduced on the
|
||||
target but whose test file was never brought over.
|
||||
|
||||
It is read-only. It runs `git log`, `git show`, `git diff`, `git grep`,
|
||||
`git ls-tree` and `git merge-base`, writes nothing to the repository, touches no
|
||||
remote, and does not import the Odysseus application package.
|
||||
|
||||
Usage:
|
||||
|
||||
scripts/ref_parity_audit.py --source public/dev --target lab --since 2026-08-10
|
||||
|
||||
Read `docs/ref-parity-audit.md` before acting on the output: the line sampling
|
||||
is a heuristic and the report labels which of its verdicts are exact.
|
||||
"""
|
||||
import argparse
|
||||
import fnmatch
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Sequence
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
# Paths whose contents are never worth probing: vendored third-party code,
|
||||
# committed build output, lockfiles and binaries. A distinctive line does not
|
||||
# exist in a minified bundle, and a lockfile churns on every dependency bump.
|
||||
DEFAULT_EXCLUDES = (
|
||||
"static/lib/*",
|
||||
"static/js/editor/build/*",
|
||||
"*.min.js",
|
||||
"*.min.css",
|
||||
"*.map",
|
||||
"package-lock.json",
|
||||
"*.lock",
|
||||
"*.png",
|
||||
"*.jpg",
|
||||
"*.jpeg",
|
||||
"*.gif",
|
||||
"*.ico",
|
||||
"*.webp",
|
||||
"*.svg",
|
||||
"*.pdf",
|
||||
"*.woff",
|
||||
"*.woff2",
|
||||
"*.ttf",
|
||||
"*.otf",
|
||||
"*.mp3",
|
||||
"*.mp4",
|
||||
"*.wav",
|
||||
"*.zip",
|
||||
"*.gz",
|
||||
)
|
||||
|
||||
# A probe has to be long enough and carry enough named things to be unlikely to
|
||||
# appear by coincidence. `return hosts` is in a hundred files; a line naming two
|
||||
# identifiers over 24 characters is usually unique to the change that added it.
|
||||
MIN_PROBE_LENGTH = 24
|
||||
MIN_PROBE_IDENTIFIERS = 2
|
||||
IDENTIFIER_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]{2,}")
|
||||
|
||||
FIELD_SEP = "\x1f"
|
||||
|
||||
VERDICT_ABSENT = "absent"
|
||||
VERDICT_PARTIAL = "partial"
|
||||
VERDICT_PRESENT = "present"
|
||||
VERDICT_NO_PROBE = "no-probe"
|
||||
|
||||
|
||||
class GitError(RuntimeError):
|
||||
"""A git invocation failed in a way the audit cannot work around."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Commit:
|
||||
sha: str
|
||||
author: str
|
||||
date: str
|
||||
subject: str
|
||||
parent_count: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommitVerdict:
|
||||
commit: Commit
|
||||
probes: tuple[str, ...]
|
||||
found: tuple[str, ...]
|
||||
paths: tuple[str, ...]
|
||||
|
||||
@property
|
||||
def verdict(self) -> str:
|
||||
if not self.probes:
|
||||
return VERDICT_NO_PROBE
|
||||
if not self.found:
|
||||
return VERDICT_ABSENT
|
||||
if len(self.found) < len(self.probes):
|
||||
return VERDICT_PARTIAL
|
||||
return VERDICT_PRESENT
|
||||
|
||||
|
||||
@dataclass
|
||||
class Report:
|
||||
source: str
|
||||
source_sha: str
|
||||
target: str
|
||||
target_sha: str
|
||||
merge_base: str
|
||||
since: str | None
|
||||
until: str | None
|
||||
traversal: str
|
||||
probe_limit: int
|
||||
verdicts: list[CommitVerdict] = field(default_factory=list)
|
||||
source_only_files: tuple[str, ...] = ()
|
||||
target_only_files: tuple[str, ...] = ()
|
||||
|
||||
def by_verdict(self, verdict: str) -> list[CommitVerdict]:
|
||||
return [v for v in self.verdicts if v.verdict == verdict]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# git plumbing
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def run_git(args: Sequence[str], repo: Path) -> str:
|
||||
"""Run a read-only git command and return stdout, raising on failure."""
|
||||
proc = subprocess.run(
|
||||
["git", "-C", str(repo), *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise GitError(f"git {' '.join(args)} failed: {proc.stderr.strip()}")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def resolve_ref(ref: str, repo: Path) -> str:
|
||||
return run_git(["rev-parse", "--short=8", ref], repo).strip()
|
||||
|
||||
|
||||
def merge_base(source: str, target: str, repo: Path) -> str:
|
||||
try:
|
||||
return run_git(["merge-base", source, target], repo).strip()[:8]
|
||||
except GitError:
|
||||
# Unrelated histories have no merge base. That is a finding, not a crash.
|
||||
return ""
|
||||
|
||||
|
||||
def list_commits(
|
||||
source: str,
|
||||
target: str,
|
||||
repo: Path,
|
||||
since: str | None = None,
|
||||
until: str | None = None,
|
||||
traversal: str = "linear",
|
||||
) -> list[Commit]:
|
||||
"""List commits reachable from `source` but not from `target`.
|
||||
|
||||
`linear` drops merge commits and reports the individual authored commits,
|
||||
which is what finds a fix that arrived on a side branch. `first-parent`
|
||||
reports one entry per merge into the source branch, which reads as one row
|
||||
per merged pull request.
|
||||
"""
|
||||
args = [
|
||||
"log",
|
||||
"--date=short",
|
||||
f"--format=%H{FIELD_SEP}%an{FIELD_SEP}%cd{FIELD_SEP}%p{FIELD_SEP}%s",
|
||||
]
|
||||
args.append("--no-merges" if traversal == "linear" else "--first-parent")
|
||||
if since:
|
||||
args.append(f"--since={since}")
|
||||
if until:
|
||||
args.append(f"--until={until}")
|
||||
args.append(f"{target}..{source}")
|
||||
|
||||
commits = []
|
||||
for line in run_git(args, repo).splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
sha, author, date, parents, subject = line.split(FIELD_SEP, 4)
|
||||
commits.append(
|
||||
Commit(
|
||||
sha=sha,
|
||||
author=author,
|
||||
date=date,
|
||||
subject=subject,
|
||||
parent_count=len(parents.split()) if parents.strip() else 0,
|
||||
)
|
||||
)
|
||||
return commits
|
||||
|
||||
|
||||
def commit_diff(commit: Commit, repo: Path) -> str:
|
||||
"""Return the commit's patch with no context lines.
|
||||
|
||||
A merge is diffed against its first parent so the whole merged content is
|
||||
visible; `git show` would otherwise print only the conflicting hunks.
|
||||
"""
|
||||
if commit.parent_count > 1:
|
||||
return run_git(
|
||||
["diff", "--no-color", "--no-renames", "-U0", f"{commit.sha}^1", commit.sha],
|
||||
repo,
|
||||
)
|
||||
return run_git(
|
||||
["show", "--no-color", "--no-renames", "-U0", "--format=", commit.sha], repo
|
||||
)
|
||||
|
||||
|
||||
def probe_present(probe: str, ref: str, repo: Path) -> bool:
|
||||
"""Is this exact text anywhere in the ref's tree?
|
||||
|
||||
The whole tree is searched on purpose. The question is whether the change
|
||||
left a trace at all, not whether it landed in the same file -- a ported fix
|
||||
routinely moves, and the exclusion list only governs where probes come
|
||||
from.
|
||||
"""
|
||||
proc = subprocess.run(
|
||||
["git", "-C", str(repo), "grep", "--fixed-strings", "--quiet", "-e", probe, ref],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode not in (0, 1):
|
||||
raise GitError(f"git grep failed for {ref}: {proc.stderr.strip()}")
|
||||
return proc.returncode == 0
|
||||
|
||||
|
||||
def list_tree(ref: str, repo: Path) -> list[str]:
|
||||
raw = run_git(["ls-tree", "-r", "-z", "--name-only", ref], repo)
|
||||
return [path for path in raw.split("\0") if path]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# probe selection (pure)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def is_excluded(path: str, patterns: Iterable[str]) -> bool:
|
||||
name = path.rsplit("/", 1)[-1]
|
||||
return any(
|
||||
fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(name, pattern)
|
||||
for pattern in patterns
|
||||
)
|
||||
|
||||
|
||||
def added_lines(patch: str, excludes: Iterable[str]) -> list[tuple[str, str]]:
|
||||
"""Extract `(path, added line)` pairs from a unified diff."""
|
||||
results = []
|
||||
path = None
|
||||
skip = False
|
||||
for line in patch.splitlines():
|
||||
if line.startswith("+++ "):
|
||||
target = line[4:].strip()
|
||||
path = None if target == "/dev/null" else target[2:] if target.startswith("b/") else target
|
||||
skip = path is None or is_excluded(path, excludes)
|
||||
elif line.startswith("--- ") or line.startswith("diff --git "):
|
||||
continue
|
||||
elif line.startswith("+") and path and not skip:
|
||||
results.append((path, line[1:]))
|
||||
return results
|
||||
|
||||
|
||||
def probe_score(text: str) -> int:
|
||||
"""Rank a candidate probe: distinct named things first, then length."""
|
||||
identifiers = set(IDENTIFIER_RE.findall(text))
|
||||
return len(identifiers) * 1000 + min(len(text), 400)
|
||||
|
||||
|
||||
def is_probe_candidate(text: str) -> bool:
|
||||
stripped = text.strip()
|
||||
if len(stripped) < MIN_PROBE_LENGTH:
|
||||
return False
|
||||
if "\0" in stripped:
|
||||
return False
|
||||
return len(set(IDENTIFIER_RE.findall(stripped))) >= MIN_PROBE_IDENTIFIERS
|
||||
|
||||
|
||||
def pick_probes(lines: Sequence[tuple[str, str]], limit: int) -> list[str]:
|
||||
"""Pick up to `limit` distinctive stripped lines, highest-scoring first.
|
||||
|
||||
Leading and trailing whitespace is dropped so a re-indented port still
|
||||
counts as present. Ties break on first appearance, keeping the output
|
||||
stable across runs.
|
||||
"""
|
||||
seen: dict[str, int] = {}
|
||||
for index, (_path, text) in enumerate(lines):
|
||||
stripped = text.strip()
|
||||
if not is_probe_candidate(stripped) or stripped in seen:
|
||||
continue
|
||||
seen[stripped] = index
|
||||
ranked = sorted(seen, key=lambda text: (-probe_score(text), seen[text]))
|
||||
return ranked[:limit]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# audit
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def audit(
|
||||
source: str,
|
||||
target: str,
|
||||
repo: Path,
|
||||
since: str | None = None,
|
||||
until: str | None = None,
|
||||
traversal: str = "linear",
|
||||
probe_limit: int = 4,
|
||||
excludes: Sequence[str] = DEFAULT_EXCLUDES,
|
||||
progress: bool = False,
|
||||
) -> Report:
|
||||
report = Report(
|
||||
source=source,
|
||||
source_sha=resolve_ref(source, repo),
|
||||
target=target,
|
||||
target_sha=resolve_ref(target, repo),
|
||||
merge_base=merge_base(source, target, repo),
|
||||
since=since,
|
||||
until=until,
|
||||
traversal=traversal,
|
||||
probe_limit=probe_limit,
|
||||
)
|
||||
|
||||
commits = list_commits(source, target, repo, since, until, traversal)
|
||||
for index, commit in enumerate(commits, start=1):
|
||||
if progress:
|
||||
print(
|
||||
f"\r[{index}/{len(commits)}] {commit.sha[:8]}",
|
||||
end="",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
lines = added_lines(commit_diff(commit, repo), excludes)
|
||||
probes = pick_probes(lines, probe_limit)
|
||||
found = tuple(p for p in probes if probe_present(p, target, repo))
|
||||
report.verdicts.append(
|
||||
CommitVerdict(
|
||||
commit=commit,
|
||||
probes=tuple(probes),
|
||||
found=found,
|
||||
paths=tuple(dict.fromkeys(path for path, _ in lines)),
|
||||
)
|
||||
)
|
||||
if progress:
|
||||
print("", file=sys.stderr)
|
||||
|
||||
source_files = {p for p in list_tree(source, repo) if not is_excluded(p, excludes)}
|
||||
target_files = {p for p in list_tree(target, repo) if not is_excluded(p, excludes)}
|
||||
report.source_only_files = tuple(sorted(source_files - target_files))
|
||||
report.target_only_files = tuple(sorted(target_files - source_files))
|
||||
return report
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# rendering
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _commit_table(verdicts: Sequence[CommitVerdict]) -> list[str]:
|
||||
rows = [
|
||||
"| Commit | Committed | Author | Probes found | Subject |",
|
||||
"|---|---|---|---|---|",
|
||||
]
|
||||
for item in verdicts:
|
||||
rows.append(
|
||||
f"| `{item.commit.sha[:8]}` | {item.commit.date} | {item.commit.author} "
|
||||
f"| {len(item.found)}/{len(item.probes)} | {item.commit.subject} |"
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _file_list(paths: Sequence[str], top: int) -> list[str]:
|
||||
lines = [f"- `{path}`" for path in paths[:top]]
|
||||
if len(paths) > top:
|
||||
lines.append(f"- … and {len(paths) - top} more")
|
||||
return lines
|
||||
|
||||
|
||||
def render_markdown(report: Report, top: int = 50) -> str:
|
||||
absent = report.by_verdict(VERDICT_ABSENT)
|
||||
partial = report.by_verdict(VERDICT_PARTIAL)
|
||||
present = report.by_verdict(VERDICT_PRESENT)
|
||||
no_probe = report.by_verdict(VERDICT_NO_PROBE)
|
||||
|
||||
window = []
|
||||
if report.since:
|
||||
window.append(f"since {report.since}")
|
||||
if report.until:
|
||||
window.append(f"until {report.until}")
|
||||
|
||||
out = [
|
||||
"# Ref parity audit",
|
||||
"",
|
||||
f"Source `{report.source}` @ `{report.source_sha}` → "
|
||||
f"target `{report.target}` @ `{report.target_sha}`.",
|
||||
f"Merge base `{report.merge_base or 'none (unrelated histories)'}`.",
|
||||
f"{len(report.verdicts)} commits on the source and not the target "
|
||||
f"({report.traversal} traversal"
|
||||
+ (", " + ", ".join(window) if window else "")
|
||||
+ f"), up to {report.probe_limit} probes each.",
|
||||
"",
|
||||
f"No trace in the target: **{len(absent)}**. "
|
||||
f"Partly present: **{len(partial)}**. "
|
||||
f"Fully present: **{len(present)}**. "
|
||||
f"Unprobeable: **{len(no_probe)}**.",
|
||||
"",
|
||||
"## Commits with no trace in the target",
|
||||
"",
|
||||
]
|
||||
out += _commit_table(absent) if absent else ["None."]
|
||||
|
||||
out += [
|
||||
"",
|
||||
"## Commits only partly present",
|
||||
"",
|
||||
"A partial verdict is inconclusive, not a finding: a line can move or be "
|
||||
"rewritten by a refactor on the target and still be the same change. Read the "
|
||||
"diff before porting anything from this table.",
|
||||
"",
|
||||
]
|
||||
out += _commit_table(partial) if partial else ["None."]
|
||||
|
||||
out += ["", "## Commits with no usable probe", ""]
|
||||
if no_probe:
|
||||
out += [
|
||||
"Deletion-only commits, and commits touching nothing but excluded paths. "
|
||||
"The audit has no verdict on these.",
|
||||
"",
|
||||
] + _commit_table(no_probe)
|
||||
else:
|
||||
out.append("None.")
|
||||
|
||||
out += [
|
||||
"",
|
||||
f"## Files on the source and not the target ({len(report.source_only_files)})",
|
||||
"",
|
||||
"Exact, not sampled. A file here whose commit is reported fully present is "
|
||||
"usually a fix that was reproduced without its test.",
|
||||
"",
|
||||
]
|
||||
out += _file_list(report.source_only_files, top) if report.source_only_files else ["None."]
|
||||
|
||||
out += [
|
||||
"",
|
||||
f"## Files on the target and not the source ({len(report.target_only_files)})",
|
||||
"",
|
||||
]
|
||||
out += _file_list(report.target_only_files, top) if report.target_only_files else ["None."]
|
||||
out.append("")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def render_json(report: Report) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"source": {"ref": report.source, "sha": report.source_sha},
|
||||
"target": {"ref": report.target, "sha": report.target_sha},
|
||||
"merge_base": report.merge_base,
|
||||
"since": report.since,
|
||||
"until": report.until,
|
||||
"traversal": report.traversal,
|
||||
"probe_limit": report.probe_limit,
|
||||
"totals": {
|
||||
verdict: len(report.by_verdict(verdict))
|
||||
for verdict in (
|
||||
VERDICT_ABSENT,
|
||||
VERDICT_PARTIAL,
|
||||
VERDICT_PRESENT,
|
||||
VERDICT_NO_PROBE,
|
||||
)
|
||||
},
|
||||
"commits": [
|
||||
{
|
||||
"sha": item.commit.sha,
|
||||
"date": item.commit.date,
|
||||
"author": item.commit.author,
|
||||
"subject": item.commit.subject,
|
||||
"verdict": item.verdict,
|
||||
"probes": list(item.probes),
|
||||
"probes_found": list(item.found),
|
||||
"paths": list(item.paths),
|
||||
}
|
||||
for item in report.verdicts
|
||||
],
|
||||
"source_only_files": list(report.source_only_files),
|
||||
"target_only_files": list(report.target_only_files),
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# cli
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def positive_int(value: str) -> int:
|
||||
parsed = int(value)
|
||||
if parsed < 1:
|
||||
raise argparse.ArgumentTypeError("must be 1 or greater")
|
||||
return parsed
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Read-only audit of which commits on one ref left no trace in another."
|
||||
)
|
||||
parser.add_argument("--source", required=True, help="Ref whose commits are audited")
|
||||
parser.add_argument("--target", required=True, help="Ref searched for traces of them")
|
||||
parser.add_argument("--repo", default=str(REPO_ROOT), help="Repository to run in")
|
||||
parser.add_argument("--since", help="Only commits committed on or after this date")
|
||||
parser.add_argument("--until", help="Only commits committed on or before this date")
|
||||
parser.add_argument(
|
||||
"--traversal",
|
||||
choices=["linear", "first-parent"],
|
||||
default="linear",
|
||||
help="linear: individual commits, no merges. first-parent: one row per merge",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--probes", type=positive_int, default=4, help="Probe lines sampled per commit"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--exclude",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="GLOB",
|
||||
help="Extra path glob whose lines are not used as probes (repeatable)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-default-excludes",
|
||||
action="store_true",
|
||||
help="Drop the built-in vendored/lockfile/binary exclusions",
|
||||
)
|
||||
parser.add_argument("--format", choices=["markdown", "json"], default="markdown")
|
||||
parser.add_argument("--top", type=positive_int, default=50, help="Rows per file list")
|
||||
parser.add_argument("--output", help="Write the report here instead of stdout")
|
||||
parser.add_argument("--quiet", action="store_true", help="No progress output")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
excludes = list(args.exclude)
|
||||
if not args.no_default_excludes:
|
||||
excludes = list(DEFAULT_EXCLUDES) + excludes
|
||||
|
||||
try:
|
||||
report = audit(
|
||||
source=args.source,
|
||||
target=args.target,
|
||||
repo=Path(args.repo),
|
||||
since=args.since,
|
||||
until=args.until,
|
||||
traversal=args.traversal,
|
||||
probe_limit=args.probes,
|
||||
excludes=excludes,
|
||||
progress=not args.quiet and sys.stderr.isatty(),
|
||||
)
|
||||
except GitError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
text = render_json(report) if args.format == "json" else render_markdown(report, args.top)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + "\n", encoding="utf-8")
|
||||
else:
|
||||
print(text)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,10 +1,26 @@
|
||||
/** Card layout and reconciliation against the served assets; no user mutations. */
|
||||
import { chromium } from 'playwright';
|
||||
|
||||
const ORIGIN = 'http://127.0.0.1:7011';
|
||||
|
||||
/** The `<link rel="stylesheet">` tags the app shell ships, in shell order. */
|
||||
async function shellStylesheets() {
|
||||
const response = await fetch(`${ORIGIN}/static/index.html`);
|
||||
if (!response.ok) throw new Error(`static/index.html returned ${response.status}`);
|
||||
const links = (await response.text()).match(/<link\b[^>]*rel=["']stylesheet["'][^>]*>/gi) || [];
|
||||
if (!links.length) throw new Error('no stylesheet links found in static/index.html');
|
||||
return links.join('');
|
||||
}
|
||||
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
try {
|
||||
const page = await browser.newPage({ viewport: { width: 390, height: 844 } });
|
||||
await page.goto('http://127.0.0.1:7011/static/test-fixtures/browser-catalog.html');
|
||||
await page.setContent('<link rel="stylesheet" href="/static/style.css"><main style="padding:16px"><div id="chat-history"><p>Existing conversation</p></div></main>');
|
||||
await page.goto(`${ORIGIN}/static/test-fixtures/browser-catalog.html`);
|
||||
// Read the shell's stylesheets out of index.html rather than naming one
|
||||
// here. style.css is now a set of ordered fragments, and a hardcoded link
|
||||
// to a file that has moved does not fail - it renders unstyled and the
|
||||
// layout checks below pass against nothing.
|
||||
await page.setContent(`${await shellStylesheets()}<main style="padding:16px"><div id="chat-history"><p>Existing conversation</p></div></main>`);
|
||||
const checks = await page.evaluate(async () => {
|
||||
const { renderResearchCards } = await import('/static/js/backgroundToolJobs.js');
|
||||
const box = document.querySelector('#chat-history');
|
||||
|
||||
Reference in New Issue
Block a user