diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2c5d81ec..7ca73f33 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,7 +97,22 @@ jobs: # (tickets come from the changelog; decisions from `decisions sync`). pql plan import pql decisions sync - - run: make smoke-bundle + # Point the real release app's crash logs at an uploadable workspace dir + # (T-436): if the bundle wedges on boot, the watchdog heartbeat/sample + + # FileLogSink land here and get uploaded below. CLIDE_LOG=debug so the + # file sink captures info/debug, not just the release-default warn. + - name: bundle smoke (logs → artifact) + env: + CLIDE_LOG: debug + CLIDE_LOG_DIR: ${{ github.workspace }}/clide-logs + run: make smoke-bundle + - name: Upload crash logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: bundle-crash-logs + path: ${{ github.workspace }}/clide-logs + if-no-files-found: ignore # The web-WASM Playwright job is withheld: `flutter build web --wasm` cannot # compile the tree since the tree-sitter/PTY dart:ffi pivot (dart:ffi is diff --git a/.github/workflows/windows-soak.yml b/.github/workflows/windows-soak.yml index 5cef084d..20798529 100644 --- a/.github/workflows/windows-soak.yml +++ b/.github/workflows/windows-soak.yml @@ -80,6 +80,13 @@ jobs: - run: flutter pub get - name: ConPTY abrupt-death orphan probe shell: pwsh + # CLIDE_LOG_DIR makes the probe emit FFI breadcrumbs (T-436): when a + # parent is force-killed mid-life, its reader/waiter isolates' last + # crumb ("ReadFile enter" / "WaitForSingleObject enter") is fsynced to + # clide-pty.crumbs.log and uploaded below — naming what the wedged + # isolate was doing at the instant of death. + env: + CLIDE_LOG_DIR: ${{ github.workspace }}/kill-crumbs run: | $iters = "${{ github.event.inputs.kill_iterations }}" if (-not $iters) { $iters = "15" } @@ -99,3 +106,10 @@ jobs: name: conpty-kill-probe path: kill-out if-no-files-found: warn + - name: Upload FFI breadcrumbs (last act of each killed reader/waiter) + if: always() + uses: actions/upload-artifact@v4 + with: + name: conpty-kill-crumbs + path: ${{ github.workspace }}/kill-crumbs + if-no-files-found: ignore diff --git a/.pql/changelog/ticket_history/2026-06.sql b/.pql/changelog/ticket_history/2026-06.sql index c5b3f123..659f192b 100644 --- a/.pql/changelog/ticket_history/2026-06.sql +++ b/.pql/changelog/ticket_history/2026-06.sql @@ -5074,3 +5074,4 @@ INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, chang INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9ER04JVFW8CN3JW1AWYA8', 'status', 'in_progress', 'done', NULL, '2026-06-15 07:29:45', '2026-06-15 07:29:45', '2026-06-15 07:29:45', NULL, 'ab3371153c8037fe287c07c4f34d1575', 2) ON CONFLICT(hash) DO NOTHING; INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9FHC8VX50759X35VNER1R', 'status', 'backlog', 'done', NULL, '2026-06-15 07:56:56', '2026-06-15 07:56:56', '2026-06-15 07:56:56', NULL, 'ff37484d4920968d13bdafa85336cd66', 2) ON CONFLICT(hash) DO NOTHING; INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9FYDEXCM15FXTER032K84', 'status', 'backlog', 'done', NULL, '2026-06-15 08:15:52', '2026-06-15 08:15:52', '2026-06-15 08:15:52', NULL, '59248f388c7cc5fd834eb1b384b6cab1', 2) ON CONFLICT(hash) DO NOTHING; +INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9GAQ2G0KCVMZS67SK3324', 'status', 'backlog', 'in_progress', NULL, '2026-06-15 08:49:49', '2026-06-15 08:49:49', '2026-06-15 08:49:49', NULL, '23fbf6cd85b51eebea4a6f503399df7e', 2) ON CONFLICT(hash) DO NOTHING; diff --git a/.pql/changelog/tickets/2026-06.sql b/.pql/changelog/tickets/2026-06.sql index 6c0fc384..0eb57cbb 100644 --- a/.pql/changelog/tickets/2026-06.sql +++ b/.pql/changelog/tickets/2026-06.sql @@ -5670,3 +5670,4 @@ INSERT INTO tickets (record_id, type, parent_record_id, title, description, stat INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9ER04JVFW8CN3JW1AWYA8', 'task', '06FCENXXZFBZ0HVD1VCW4ZASCC', 'FileLogSink + logDirectory + boot-time verbosity resolver (CLIDE_LOG → env → setting → release/debug default)', NULL, 'done', 'high', NULL, NULL, NULL, '2026-06-15 07:18:58', '2026-06-15 07:29:45', NULL, '2a0f0ae1b6fad09cdbf46e9469be0ec9', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at > tickets.updated_at OR (excluded.updated_at = tickets.updated_at AND excluded.hash > tickets.hash); INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9FHC8VX50759X35VNER1R', 'task', '06FCENXXZFBZ0HVD1VCW4ZASCC', 'FFI breadcrumbs in windows_pty.dart (+native_pty): injectable log callback, before/after each risky syscall with return + GetLastError; reader/waiter isolates flushSync their own append handle', NULL, 'done', 'high', NULL, NULL, NULL, '2026-06-15 07:19:04', '2026-06-15 07:56:56', NULL, '8c16b539c378e46174b52197a1a26bf0', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at > tickets.updated_at OR (excluded.updated_at = tickets.updated_at AND excluded.hash > tickets.hash); INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9FYDEXCM15FXTER032K84', 'task', '06FCENXXZFBZ0HVD1VCW4ZASCC', 'Watchdog heartbeat + resource sampler in a dedicated isolate (heartbeat ~500ms; sample ConPTY child / handle / thread / memory ~2s)', NULL, 'done', 'medium', NULL, NULL, NULL, '2026-06-15 07:19:08', '2026-06-15 08:15:52', NULL, 'c72bedb8cc95ae2232767537f7832087', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at > tickets.updated_at OR (excluded.updated_at = tickets.updated_at AND excluded.hash > tickets.hash); +INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9GAQ2G0KCVMZS67SK3324', 'task', '06FCENXXZFBZ0HVD1VCW4ZASCC', 'Wire FileLogSink into test harness + ci/test.sh (CLIDE_LOG=debug, log dir outside build tree, upload as CI artifact in always() step)', NULL, 'in_progress', 'medium', NULL, NULL, NULL, '2026-06-15 07:19:11', '2026-06-15 08:49:49', NULL, '21a64df388e4f8de2ed217fe93b29207', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at > tickets.updated_at OR (excluded.updated_at = tickets.updated_at AND excluded.hash > tickets.hash); diff --git a/CHANGELOG.md b/CHANGELOG.md index d88d7a85..aee497c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,8 +22,8 @@ heading, and (b) bumping `pubspec.yaml` `version:` in the same commit. persistent per-platform dir (Windows `%LOCALAPPDATA%`, macOS `~/Library/Logs`, Linux `$XDG_STATE_HOME`), fsyncing warn/error + pty/ffi records immediately so a freeze leaves on-disk evidence. `CLIDE_LOG` (dart-define / env) or the - `app.log.level` setting sets verbosity (warn in release, info in debug). - (T-432) + `app.log.level` setting sets verbosity (warn in release, info in debug); + `CLIDE_LOG_DIR` redirects where the logs land. (T-432, T-436) - **PTY FFI breadcrumbs.** Each PTY backend drops a breadcrumb before/after every risky syscall (`CreatePseudoConsole`/`CreateProcessW`/`ReadFile`, `posix_spawn`/`read`); the reader/waiter isolates fsync their OWN file handle diff --git a/lib/src/ipc/paths.dart b/lib/src/ipc/paths.dart index 6759f053..fb4ee8f7 100644 --- a/lib/src/ipc/paths.dart +++ b/lib/src/ipc/paths.dart @@ -65,18 +65,26 @@ String socketDirectory() { /// on logout/reboot — this is a DURABLE location. The whole point of the /// FileLogSink is that a freeze's last breadcrumbs survive the power-cycle, so /// the log dir must outlive a reboot. -String logDirectory() { +/// +/// `CLIDE_LOG_DIR` overrides everything: CI points it at a workspace dir +/// outside the build tree so a wedged run's logs can be uploaded as an +/// artifact (T-436), and tests redirect it to a temp dir. +/// [env] defaults to [Platform.environment]; injectable for tests. +String logDirectory([Map? env]) { + final e = env ?? Platform.environment; + final override = e['CLIDE_LOG_DIR']; + if (override != null && override.isNotEmpty) return override; if (Platform.isWindows) { - final local = Platform.environment['LOCALAPPDATA']; - final base = (local != null && local.isNotEmpty) ? local : '${Platform.environment['USERPROFILE'] ?? r'C:\'}\\AppData\\Local'; + final local = e['LOCALAPPDATA']; + final base = (local != null && local.isNotEmpty) ? local : '${e['USERPROFILE'] ?? r'C:\'}\\AppData\\Local'; return '$base\\clide\\logs'; } if (Platform.isMacOS) { - final home = Platform.environment['HOME'] ?? '/tmp'; + final home = e['HOME'] ?? '/tmp'; return '$home/Library/Logs/clide'; } - final state = Platform.environment['XDG_STATE_HOME']; - final base = (state != null && state.isNotEmpty) ? state : '${Platform.environment['HOME'] ?? '/tmp'}/.local/state'; + final state = e['XDG_STATE_HOME']; + final base = (state != null && state.isNotEmpty) ? state : '${e['HOME'] ?? '/tmp'}/.local/state'; return '$base/clide/logs'; } diff --git a/lib/test_app.dart b/lib/test_app.dart index 76dcf87a..b237e295 100644 --- a/lib/test_app.dart +++ b/lib/test_app.dart @@ -31,6 +31,7 @@ import 'kernel/kernel.dart'; import 'src/pty/ffi/libc.dart' as libc; import 'src/daemon/pane_commands.dart'; import 'src/ipc/envelope.dart'; +import 'src/ipc/paths.dart' show logDirectory; import 'src/panes/event_sink.dart'; import 'src/panes/registry.dart'; import 'src/daemon/dispatcher.dart'; @@ -59,6 +60,7 @@ class _ClideTestAppState extends State { @override void initState() { super.initState(); + _attachCrashLogging(); WidgetsBinding.instance.addPostFrameCallback((_) => _runTests()); Timer(_timeout, () { _say('timeout reached — exiting'); @@ -66,6 +68,28 @@ class _ClideTestAppState extends State { }); } + /// Opt-in crash evidence for testmode (T-436): when `CLIDE_LOG_DIR` is set + /// (CI / a manual Windows repro), tee this harness's logger to a + /// FileLogSink and spawn the watchdog, so a wedged testmode run leaves the + /// same log + watchdog files the real app would. `_say` breadcrumbs each + /// test through the logger, so they land in the file too. Off by default — + /// normal `make run-testmode` keeps the stderr-only path, no isolate. + void _attachCrashLogging() { + final dir = Platform.environment['CLIDE_LOG_DIR']; + if (dir == null || dir.isEmpty) return; + final logDir = logDirectory(); + try { + _logger.addSink(FileLogSink(dir: Directory(logDir)).call); + } catch (_) {} + unawaited(_spawnWatchdog(logDir)); + } + + Future _spawnWatchdog(String logDir) async { + try { + await Isolate.spawn(watchdogEntry, ('$logDir/clide-watchdog.log', 500, 2000)); + } catch (_) {} + } + Future _runTests() async { const workspace = String.fromEnvironment('CLIDE_PROJECT'); const category = String.fromEnvironment('CLIDE_TESTMODE'); diff --git a/test/ipc/paths_test.dart b/test/ipc/paths_test.dart index f56c65bf..8f14db66 100644 --- a/test/ipc/paths_test.dart +++ b/test/ipc/paths_test.dart @@ -62,6 +62,12 @@ void main() { final home = Platform.environment['HOME']!; expect(logDirectory(), '$home/Library/Logs/clide'); }); + + test('CLIDE_LOG_DIR overrides everything (CI artifact / test redirect)', () { + expect(logDirectory({'CLIDE_LOG_DIR': '/tmp/ci-logs'}), '/tmp/ci-logs'); + // Empty override is ignored — falls through to the platform default. + expect(logDirectory({'CLIDE_LOG_DIR': '', 'XDG_STATE_HOME': '/x', 'HOME': '/h'}), isNot('/')); + }); }); group('fnv1a64Hex (T-126 cross-check)', () { diff --git a/tools/windows-verify/conpty_orphan_probe.dart b/tools/windows-verify/conpty_orphan_probe.dart index 1cdbfb8a..854c0742 100644 --- a/tools/windows-verify/conpty_orphan_probe.dart +++ b/tools/windows-verify/conpty_orphan_probe.dart @@ -25,6 +25,7 @@ library; import 'dart:io'; +import 'package:clide/src/pty/pty_log.dart'; import 'package:clide/src/pty/windows_pty.dart'; Future main(List args) async { @@ -34,6 +35,13 @@ Future main(List args) async { } final count = args.isNotEmpty ? (int.tryParse(args.first) ?? 1) : 1; + // When CLIDE_LOG_DIR is set (the soak workflow sets it), emit FFI breadcrumbs + // so that when soak-conpty-kill.ps1 force-kills this process mid-life, the + // reader/waiter isolates' LAST crumb (e.g. "ReadFile enter") is on disk — + // CI then uploads it, naming what the reader was doing when killed (T-436). + final logDir = Platform.environment['CLIDE_LOG_DIR']; + final ptyLog = (logDir == null || logDir.isEmpty) ? PtyLog.none : PtyLog(crumbPath: '$logDir/clide-pty.crumbs.log', verbose: true); + final sessions = []; for (var i = 0; i < count; i++) { final s = WindowsPty.start( @@ -44,6 +52,7 @@ Future main(List args) async { columns: 80, rows: 24, environment: {...Platform.environment, 'TERM': 'xterm-256color'}, + log: ptyLog, ); // Drain output so the reader isolate is actively pumping, closest to a // real live pane. Discard the bytes.