refactor(frontend): establish source ownership and architecture gates (#6701)

scripts/build_frontend.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Publish explicitly owned native ES modules; never run at application startup."""
+
+import argparse
+import json
+import os
+from pathlib import Path, PurePosixPath
+import shutil
+import subprocess
+import tempfile
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def module_path(base: Path, relative: str) -> Path:
+    """Accept canonical JS paths only, contained within the declared tree."""
+    if not isinstance(relative, str):
+        raise ValueError("module paths must be strings")
+    parts = PurePosixPath(relative)
+    if (
+        parts.is_absolute()
+        or parts.as_posix() != relative
+        or ".." in parts.parts
+        or "\\" in relative
+        or "?" in relative
+        or "#" in relative
+        or parts.suffix != ".js"
+    ):
+        raise ValueError(f"invalid module path: {relative!r}")
+    path = base / relative
+    if not path.resolve().is_relative_to(base.resolve()):
+        raise ValueError(f"module path escapes its tree: {relative!r}")
+    if any(p.is_symlink() for p in (path, *path.parents)):
+        raise ValueError(f"symlink in module path: {relative!r}")
+    return path
+
+
+def check_lf_attributes(root: Path, paths: set[Path]) -> None:
+    """Require portable, repository-owned LF attributes for every mapped path.
+
+    Ask Git in an isolated temporary repository so global attributes, local
+    info/attributes and alternate Git directories cannot mask missing rules.
+    Copy only attribute files at ancestors of the manifest's source/output paths.
+    """
+    git = shutil.which("git")
+    if git is None:
+        raise ValueError("Git is required for frontend publication attribute checks")
+    attributes = {root / ".gitattributes"}
+    for path in paths:
+        for parent in path.parents:
+            if parent == root:
+                break
+            attributes.add(parent / ".gitattributes")
+    env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
+    env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull)
+    relative = sorted(path.relative_to(root).as_posix() for path in paths)
+    with tempfile.TemporaryDirectory(prefix="ody-frontend-attributes-") as directory:
+        target = Path(directory)
+        for attribute in attributes:
+            if attribute.is_symlink():
+                raise ValueError(f"symlink attribute file: {attribute.relative_to(root)}")
+            if attribute.is_file():
+                copy = target / attribute.relative_to(root)
+                copy.parent.mkdir(parents=True, exist_ok=True)
+                copy.write_bytes(attribute.read_bytes())
+        command = [git, "-c", f"core.attributesFile={os.devnull}", "-C", str(target)]
+        try:
+            subprocess.run(command + ["init", "--quiet", "--template="], env=env, check=True, capture_output=True, timeout=15)
+            result = subprocess.run(
+                command + ["check-attr", "-z", "--stdin", "text", "eol"],
+                input=("\0".join(relative) + "\0").encode("utf-8"),
+                env=env, check=True, capture_output=True, timeout=15,
+            )
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
+            raise ValueError("Git frontend attribute check failed") from error
+    fields = result.stdout.decode("utf-8").split("\0")[:-1]
+    effective = {name: {} for name in relative}
+    for index in range(0, len(fields), 3):
+        name, attribute, value = fields[index:index + 3]
+        effective[name][attribute] = value
+    invalid = [
+        f"{name} (text={values.get('text')}, eol={values.get('eol')})"
+        for name, values in effective.items()
+        if values != {"text": "set", "eol": "lf"}
+    ]
+    if invalid:
+        raise ValueError("manifest paths require repository-owned 'text eol=lf' Git attributes: " + ", ".join(invalid))
+
+
+def synchronize(root: Path = ROOT, *, check: bool = False) -> list[str]:
+    """Validate the entire plan before writing. Return stale deployment paths."""
+    root = root.resolve()
+    source_root = root / "frontend" / "src"
+    static_root = root / "static"
+    manifest = json.loads((root / "frontend" / "modules.json").read_text(encoding="utf-8"))
+    if not isinstance(manifest, dict) or manifest.get("version") != 1 or not isinstance(manifest.get("modules"), list):
+        raise ValueError("expected frontend module manifest version 1")
+    if not manifest["modules"]:
+        raise ValueError("module manifest must not be empty")
+    sources, outputs, plan = set(), set(), []
+    for entry in manifest["modules"]:
+        source = module_path(source_root, entry["source"])
+        output = module_path(static_root, entry["deployment"])
+        if not entry["deployment"].startswith("js/"):
+            raise ValueError("deployment modules must be under static/js/")
+        if source in sources or output in outputs:
+            raise ValueError("each source and deployment path must be unique")
+        sources.add(source)
+        outputs.add(output)
+        content = source.read_bytes()
+        if output.exists() and not output.is_file():
+            raise ValueError(f"deployment path is not a file: {output}")
+        plan.append((output, content))
+    unlisted = set(source_root.rglob("*.js")) - sources
+    if unlisted:
+        raise ValueError(f"source modules missing from manifest: {sorted(map(str, unlisted))}")
+    check_lf_attributes(root, sources | outputs)
+    stale = []
+    for output, content in plan:
+        if not output.is_file() or output.read_bytes() != content:
+            stale.append(output.relative_to(root).as_posix())
+    if not check:
+        for output, content in plan:
+            if output.relative_to(root).as_posix() in stale:
+                output.parent.mkdir(parents=True, exist_ok=True)
+                output.write_bytes(content)
+    return stale
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--check", action="store_true", help="fail on invalid attributes or stale output; write no deployment files")
+    args = parser.parse_args()
+    try:
+        stale = synchronize(check=args.check)
+    except (OSError, ValueError, KeyError, TypeError) as error:
+        parser.exit(2, f"frontend: {error}\n")
+    if args.check and stale:
+        print("Stale frontend deployment modules: " + ", ".join(stale))
+        print("Run python scripts/build_frontend.py and include the source and output changes together.")
+        return 1
+    print(f"Frontend {'check passed' if args.check else 'published'} ({len(stale)} changed modules).")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/check_frontend.mjs

@@ -0,0 +1,180 @@
+/** Architecture gates for maintained source. Legacy static modules stay under existing guards. */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parse } from 'acorn';
+
+const layers = ['domain', 'application', 'infrastructure', 'ui'];
+const platformIdentifiers = new Set(['window', 'document', 'globalThis', 'navigator', 'localStorage', 'sessionStorage', 'fetch', 'Notification', 'indexedDB', 'WebSocket', 'XMLHttpRequest', 'Worker', 'setTimeout', 'setInterval', 'clearTimeout', 'clearInterval', 'requestAnimationFrame', 'cancelAnimationFrame', 'performance', 'eval', 'Function']);
+
+function ownership(source) {
+  const parts = source.split('/');
+  if (parts[0] === 'app' && parts.length >= 2) return { feature: 'app', layer: 'index' };
+  if (parts[0] === 'shared') {
+    if (parts.length === 2) return { feature: 'shared', layer: 'domain' };
+    const layer = { domain: 'domain', platform: 'infrastructure', ui: 'ui' }[parts[1]];
+    if (layer) return { feature: 'shared', layer };
+  }
+  if (parts[0] === 'features' && parts.length >= 3) {
+    if (['app', 'shared'].includes(parts[1])) throw new Error(`reserved feature name: ${parts[1]}`);
+    if (parts.length === 3 && parts[2] === 'index.js') return { feature: parts[1], layer: 'index' };
+    if (layers.includes(parts[2]) && parts.length >= 4) return { feature: parts[1], layer: parts[2] };
+  }
+  throw new Error(`unsupported source ownership: ${source}`);
+}
+
+export function allowsDependency(from, to) {
+  const a = ownership(from), b = ownership(to);
+  if (b.feature === 'app') return a.feature === 'app';
+  if (a.feature === 'app') return b.feature === 'shared' || b.layer === 'index';
+  if (b.feature === 'shared') {
+    const shared = { domain: ['domain'], application: ['domain'], infrastructure: ['domain', 'infrastructure'], ui: ['domain', 'infrastructure', 'ui'], index: layers };
+    return shared[a.layer].includes(b.layer);
+  }
+  if (a.feature === 'shared') return false;
+  if (a.feature !== b.feature) return ['application', 'index'].includes(a.layer) && b.layer === 'index';
+  const internal = { domain: ['domain'], application: ['domain', 'application'], infrastructure: ['domain', 'infrastructure'], ui: ['domain', 'application', 'ui'], index: layers };
+  return internal[a.layer].includes(b.layer);
+}
+
+export function imports(source, { domain = false } = {}) {
+  const ast = parse(source, { ecmaVersion: 'latest', sourceType: 'module', locations: true });
+  const found = [];
+  function visit(node, parent, field) {
+    if (!node || typeof node !== 'object') return;
+    // Domain modules use explicit values/ports, including an injected clock.
+    // Keep platform vocabulary out even when a local binding shadows a global.
+    const propertyName = parent && !parent.computed && (
+      (parent.type === 'MemberExpression' && field === 'property') ||
+      (['Property', 'MethodDefinition'].includes(parent.type) && field === 'key' && !parent.shorthand)
+    );
+    if (domain && !propertyName && node.type === 'Identifier' && platformIdentifiers.has(node.name)) {
+      throw new Error(`platform identifier in domain at line ${node.loc.start.line}: ${node.name}`);
+    }
+    if (domain && ((node.type === 'MemberExpression' && node.object.name === 'Date' && (node.property.name === 'now' || node.property.value === 'now')) || (['CallExpression', 'NewExpression'].includes(node.type) && node.callee.name === 'Date' && !node.arguments.length))) {
+      throw new Error(`ambient clock in domain at line ${node.loc.start.line}; inject time`);
+    }
+    if (['ImportDeclaration', 'ExportNamedDeclaration', 'ExportAllDeclaration', 'ImportExpression'].includes(node.type) && node.source) {
+      if (node.source.type !== 'Literal' || typeof node.source.value !== 'string') {
+        throw new Error(`computed import at line ${node.loc.start.line}; use explicit import thunks`);
+      }
+      found.push({ specifier: node.source.value, line: node.loc.start.line, dynamic: node.type === 'ImportExpression' });
+    }
+    for (const [key, value] of Object.entries(node)) {
+      if (Array.isArray(value)) value.forEach(item => visit(item, node, key));
+      else if (value && typeof value === 'object' && value.type) visit(value, node, key);
+    }
+  }
+  visit(ast);
+  return found;
+}
+
+export function checkArchitecture(root) {
+  const manifest = JSON.parse(fs.readFileSync(path.join(root, 'frontend/modules.json'), 'utf8'));
+  const bySource = new Map(manifest.modules.map(m => [m.source, m]));
+  const byOutput = new Map(manifest.modules.map(m => [`/static/${m.deployment}`, m.source]));
+  const graph = new Map();
+  const errors = [];
+  const sw = fs.readFileSync(path.join(root, 'static/sw.js'), 'utf8');
+  // Parse the actual manifest arrays, not comments or unrelated strings.
+  const swAst = parse(sw, { ecmaVersion: 'latest', sourceType: 'script' });
+  const precached = new Set();
+  for (const statement of swAst.body) {
+    for (const declaration of statement.declarations || []) {
+      if (['PRECACHE', 'PANEL_PRECACHE'].includes(declaration.id.name)) {
+        for (const item of declaration.init.elements) {
+          if (item?.type === 'Literal' && typeof item.value === 'string') precached.add(item.value);
+        }
+      }
+    }
+  }
+  for (const module of manifest.modules) {
+    try {
+      const owner = ownership(module.source);
+      const deployed = `/static/${module.deployment}`;
+      if (!precached.has(deployed)) throw new Error(`deployment URL absent from explicit precache: ${deployed}`);
+      const dependencies = [];
+      const exceptions = module.legacyImports || [];
+      const used = new Set();
+      for (const { specifier, line } of imports(fs.readFileSync(path.join(root, 'frontend/src', module.source), 'utf8'), { domain: owner.layer === 'domain' })) {
+        const fail = message => { throw new Error(`line ${line}: ${message}: ${specifier}`); };
+        const runtime = new URL(specifier, `https://odysseus.invalid${deployed}`);
+        if (!runtime.pathname.startsWith('/static/js/') || !runtime.pathname.endsWith('.js') || runtime.pathname.includes('%')) fail('runtime imports must name canonical static JS paths');
+        let target;
+        if (specifier.startsWith('.')) {
+          if (runtime.search || runtime.hash) fail('maintained modules use one unversioned URL identity');
+          target = path.posix.normalize(path.posix.join(path.posix.dirname(module.source), specifier));
+          if (!bySource.has(target)) fail('relative import must name maintained source');
+          if (runtime.pathname !== `/static/${bySource.get(target).deployment}`) fail('source and deployment relative paths resolve differently');
+        } else if (specifier.startsWith('/static/')) {
+          target = byOutput.get(runtime.pathname);
+          if (!target) {
+            const exception = exceptions.find(e => e.specifier === specifier && e.reason?.trim());
+            if (!exception || !['index', 'infrastructure', 'ui'].includes(owner.layer) || owner.feature === 'shared') fail('legacy import requires an explicit module-scoped rationale at a composition, infrastructure or UI boundary');
+            if (!fs.statSync(path.join(root, runtime.pathname.slice(1))).isFile()) fail('legacy target does not exist');
+            used.add(exception);
+            continue;
+          }
+        } else fail('bare, external and source-tree runtime imports are unsupported');
+        if (runtime.search || runtime.hash) fail('maintained modules use one unversioned URL identity');
+        if (!allowsDependency(module.source, target)) fail(`forbidden dependency on ${target}`);
+        dependencies.push(target);
+      }
+      if (exceptions.some(e => !used.has(e))) throw new Error('unused legacy import exception');
+      graph.set(module.source, dependencies);
+    } catch (error) { errors.push(`${module.source}: ${error.message}`); }
+  }
+  const visiting = new Set(), visited = new Set();
+  function visit(source, trail = []) {
+    if (visiting.has(source)) throw new Error(`maintained-source dependency cycle: ${[...trail, source].join(' -> ')}`);
+    if (visited.has(source)) return;
+    visiting.add(source);
+    for (const target of graph.get(source) || []) visit(target, [...trail, source]);
+    visiting.delete(source);
+    visited.add(source);
+  }
+  try { for (const source of graph.keys()) visit(source); }
+  catch (error) { errors.push(error.message); }
+  if (errors.length) throw new Error(errors.join('\n'));
+  return graph.size;
+}
+
+export function inventory(root) {
+  const staticRoot = path.join(root, 'static');
+  const files = fs.readdirSync(staticRoot, { recursive: true }).map(p => p.split(path.sep).join('/')).filter(p => p.endsWith('.js') && !p.startsWith('lib/') && !p.startsWith('js/editor/build/')).sort();
+  const graph = new Map(), fan = new Map(), missing = [];
+  let staticImports = 0, dynamicLiteralImports = 0;
+  for (const file of files) {
+    const targets = [];
+    for (const { specifier, dynamic } of imports(fs.readFileSync(path.join(staticRoot, file), 'utf8'))) {
+      if (!specifier.startsWith('.') && !specifier.startsWith('/static/')) continue;
+      const target = new URL(specifier, `https://odysseus.invalid/static/${file}`).pathname.slice('/static/'.length);
+      if (dynamic) dynamicLiteralImports++;
+      else { staticImports++; targets.push(target); }
+      fan.set(target, (fan.get(target) || 0) + 1);
+      if (!fs.existsSync(path.join(staticRoot, target))) missing.push({ file, specifier });
+    }
+    graph.set(file, targets);
+  }
+  const indices = new Map(), low = new Map(), stack = [], active = new Set(), staticCycles = [];
+  function visit(v) {
+    indices.set(v, indices.size); low.set(v, indices.get(v)); stack.push(v); active.add(v);
+    for (const w of graph.get(v) || []) {
+      if (!indices.has(w)) { visit(w); low.set(v, Math.min(low.get(v), low.get(w))); }
+      else if (active.has(w)) low.set(v, Math.min(low.get(v), indices.get(w)));
+    }
+    if (low.get(v) === indices.get(v)) {
+      const component = []; let w;
+      do { w = stack.pop(); active.delete(w); component.push(w); } while (w !== v);
+      if (component.length > 1) staticCycles.push(component.sort());
+    }
+  }
+  for (const file of files) if (!indices.has(file)) visit(file);
+  return { files: files.length, staticImports, dynamicLiteralImports, missing, fanIn: [...fan].sort((a, b) => b[1] - a[1]).slice(0, 8), staticCycles };
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+  try { console.log(process.argv.includes('--inventory') ? JSON.stringify(inventory(root), null, 2) : `Frontend architecture passed (${checkArchitecture(root)} maintained modules).`); }
+  catch (error) { console.error(error.message); process.exitCode = 1; }
+}

scripts/verify_frontend_runtime.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Verify the current Python runtime from an isolated distribution without frontend tooling."""
+import os
+from pathlib import Path
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+
+ROOT = Path(__file__).resolve().parents[1]
+
+PROBE = """
+from pathlib import Path
+import shutil
+from starlette.testclient import TestClient
+from app import app
+assert not Path('frontend').exists()
+assert not Path('node_modules').exists()
+assert shutil.which('node') is None
+with TestClient(app) as client:
+    response = client.get('/static/js/actionMenuOrder.js')
+    assert response.status_code == 200, response.status_code
+    assert response.content == Path('static/js/actionMenuOrder.js').read_bytes()
+    assert response.headers['content-type'].startswith('text/javascript'), response.headers
+    assert response.headers['cache-control'] == 'no-cache', response.headers
+    assert "script-src 'self' 'nonce-" in response.headers['content-security-policy'], response.headers
+    assert client.get('/frontend/src/shared/actionMenuOrder.js').status_code == 404
+    assert client.get('/static/%2e%2e/app.py').status_code == 404
+    shell = client.get('/')
+    assert shell.status_code == 200, shell.status_code
+    assert '{{CSP_NONCE}}' not in shell.text
+    assert '/static/app.js' in shell.text
+print('PASS: isolated Python lifespan, shell, deployed bytes, MIME, revalidation, CSP and source-tree 404')
+"""
+
+
+def main() -> int:
+    with tempfile.TemporaryDirectory(prefix="ody-f1-distribution-") as directory, socket.socket() as chroma_guard:
+        target = Path(directory)
+        for name in ("app.py", "HARNESS_VERSION"):
+            shutil.copy2(ROOT / name, target / name)
+        for name in ("core", "routes", "src", "services", "resources", "mcp_servers", "companion", "config", "static", "scripts"):
+            shutil.copytree(ROOT / name, target / name, ignore=shutil.ignore_patterns("__pycache__", "node_modules", ".env", "secrets.env*", "*.db", "*.sqlite*"))
+        # Reserve a non-listening port so initialization cannot reach a real store.
+        chroma_guard.bind(("127.0.0.1", 0))
+        env = dict(os.environ)
+        env.update(
+            ODYSSEUS_DATA_DIR=str(target / "data"),
+            DATABASE_URL="sqlite:///:memory:",
+            AUTH_ENABLED="false",
+            ODYSSEUS_STARTUP_WARMUPS="0",
+            ODYSSEUS_TOOL_INDEX_PREWARM="0",
+            CHROMADB_HOST="127.0.0.1",
+            CHROMADB_PORT=str(chroma_guard.getsockname()[1]),
+            FASTEMBED_CACHE_PATH=str(target / "fastembed"),
+            PATH=str(target / "no-development-tools"),
+        )
+        # A caller's Python search path must not import the original checkout.
+        env.pop("PYTHONPATH", None)
+        result = subprocess.run([sys.executable, "-c", PROBE], cwd=target, env=env, capture_output=True, text=True, timeout=60)
+        print(result.stdout, end="")
+        if result.returncode:
+            print(result.stderr[-8000:], file=sys.stderr)
+        return result.returncode
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())