feat(ci): capture crash evidence as artifacts when a run wedges (T-436)
Closes the observability loop: the log + breadcrumb + watchdog files are now collected by CI so a wedged run leaves downloadable evidence instead of nothing. - logDirectory(): CLIDE_LOG_DIR overrides the per-platform default, so CI can point the logs at an uploadable workspace dir (and tests at a temp dir). Now takes an injectable env map; tested. - test_app.dart: when CLIDE_LOG_DIR is set, the testmode harness tees its logger to a FileLogSink + spawns the watchdog (off by default — normal run-testmode keeps the stderr-only path, no isolate). _say breadcrumbs each test into the file. - conpty_orphan_probe.dart: with CLIDE_LOG_DIR set it passes a verbose PtyLog, so when soak-conpty-kill.ps1 force-kills the parent, the reader/waiter isolates' LAST crumb is fsynced to disk — naming what the wedged isolate was doing at the instant of death. - bundle-smoke job: runs the real release app with CLIDE_LOG=debug + CLIDE_LOG_DIR, uploads clide-logs (watchdog heartbeat/sample + FileLogSink) in an always() step. - windows-soak kill-probe job: sets CLIDE_LOG_DIR, uploads the FFI crumbs. Coverage gate 95.08%. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -97,7 +97,22 @@ jobs:
|
||||
# (tickets come from the changelog; decisions from `decisions sync`).
|
||||
pql plan import
|
||||
pql decisions sync
|
||||
- run: make smoke-bundle
|
||||
# Point the real release app's crash logs at an uploadable workspace dir
|
||||
# (T-436): if the bundle wedges on boot, the watchdog heartbeat/sample +
|
||||
# FileLogSink land here and get uploaded below. CLIDE_LOG=debug so the
|
||||
# file sink captures info/debug, not just the release-default warn.
|
||||
- name: bundle smoke (logs → artifact)
|
||||
env:
|
||||
CLIDE_LOG: debug
|
||||
CLIDE_LOG_DIR: ${{ github.workspace }}/clide-logs
|
||||
run: make smoke-bundle
|
||||
- name: Upload crash logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: bundle-crash-logs
|
||||
path: ${{ github.workspace }}/clide-logs
|
||||
if-no-files-found: ignore
|
||||
|
||||
# The web-WASM Playwright job is withheld: `flutter build web --wasm` cannot
|
||||
# compile the tree since the tree-sitter/PTY dart:ffi pivot (dart:ffi is
|
||||
|
||||
@@ -80,6 +80,13 @@ jobs:
|
||||
- run: flutter pub get
|
||||
- name: ConPTY abrupt-death orphan probe
|
||||
shell: pwsh
|
||||
# CLIDE_LOG_DIR makes the probe emit FFI breadcrumbs (T-436): when a
|
||||
# parent is force-killed mid-life, its reader/waiter isolates' last
|
||||
# crumb ("ReadFile enter" / "WaitForSingleObject enter") is fsynced to
|
||||
# clide-pty.crumbs.log and uploaded below — naming what the wedged
|
||||
# isolate was doing at the instant of death.
|
||||
env:
|
||||
CLIDE_LOG_DIR: ${{ github.workspace }}/kill-crumbs
|
||||
run: |
|
||||
$iters = "${{ github.event.inputs.kill_iterations }}"
|
||||
if (-not $iters) { $iters = "15" }
|
||||
@@ -99,3 +106,10 @@ jobs:
|
||||
name: conpty-kill-probe
|
||||
path: kill-out
|
||||
if-no-files-found: warn
|
||||
- name: Upload FFI breadcrumbs (last act of each killed reader/waiter)
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: conpty-kill-crumbs
|
||||
path: ${{ github.workspace }}/kill-crumbs
|
||||
if-no-files-found: ignore
|
||||
|
||||
@@ -5074,3 +5074,4 @@ INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, chang
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9ER04JVFW8CN3JW1AWYA8', 'status', 'in_progress', 'done', NULL, '2026-06-15 07:29:45', '2026-06-15 07:29:45', '2026-06-15 07:29:45', NULL, 'ab3371153c8037fe287c07c4f34d1575', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9FHC8VX50759X35VNER1R', 'status', 'backlog', 'done', NULL, '2026-06-15 07:56:56', '2026-06-15 07:56:56', '2026-06-15 07:56:56', NULL, 'ff37484d4920968d13bdafa85336cd66', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9FYDEXCM15FXTER032K84', 'status', 'backlog', 'done', NULL, '2026-06-15 08:15:52', '2026-06-15 08:15:52', '2026-06-15 08:15:52', NULL, '59248f388c7cc5fd834eb1b384b6cab1', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9GAQ2G0KCVMZS67SK3324', 'status', 'backlog', 'in_progress', NULL, '2026-06-15 08:49:49', '2026-06-15 08:49:49', '2026-06-15 08:49:49', NULL, '23fbf6cd85b51eebea4a6f503399df7e', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
|
||||
@@ -5670,3 +5670,4 @@ INSERT INTO tickets (record_id, type, parent_record_id, title, description, stat
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9ER04JVFW8CN3JW1AWYA8', 'task', '06FCENXXZFBZ0HVD1VCW4ZASCC', 'FileLogSink + logDirectory + boot-time verbosity resolver (CLIDE_LOG → env → setting → release/debug default)', NULL, 'done', 'high', NULL, NULL, NULL, '2026-06-15 07:18:58', '2026-06-15 07:29:45', NULL, '2a0f0ae1b6fad09cdbf46e9469be0ec9', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at > tickets.updated_at OR (excluded.updated_at = tickets.updated_at AND excluded.hash > tickets.hash);
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9FHC8VX50759X35VNER1R', 'task', '06FCENXXZFBZ0HVD1VCW4ZASCC', 'FFI breadcrumbs in windows_pty.dart (+native_pty): injectable log callback, before/after each risky syscall with return + GetLastError; reader/waiter isolates flushSync their own append handle', NULL, 'done', 'high', NULL, NULL, NULL, '2026-06-15 07:19:04', '2026-06-15 07:56:56', NULL, '8c16b539c378e46174b52197a1a26bf0', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at > tickets.updated_at OR (excluded.updated_at = tickets.updated_at AND excluded.hash > tickets.hash);
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9FYDEXCM15FXTER032K84', 'task', '06FCENXXZFBZ0HVD1VCW4ZASCC', 'Watchdog heartbeat + resource sampler in a dedicated isolate (heartbeat ~500ms; sample ConPTY child / handle / thread / memory ~2s)', NULL, 'done', 'medium', NULL, NULL, NULL, '2026-06-15 07:19:08', '2026-06-15 08:15:52', NULL, 'c72bedb8cc95ae2232767537f7832087', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at > tickets.updated_at OR (excluded.updated_at = tickets.updated_at AND excluded.hash > tickets.hash);
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FCM9GAQ2G0KCVMZS67SK3324', 'task', '06FCENXXZFBZ0HVD1VCW4ZASCC', 'Wire FileLogSink into test harness + ci/test.sh (CLIDE_LOG=debug, log dir outside build tree, upload as CI artifact in always() step)', NULL, 'in_progress', 'medium', NULL, NULL, NULL, '2026-06-15 07:19:11', '2026-06-15 08:49:49', NULL, '21a64df388e4f8de2ed217fe93b29207', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at > tickets.updated_at OR (excluded.updated_at = tickets.updated_at AND excluded.hash > tickets.hash);
|
||||
|
||||
+2
-2
@@ -22,8 +22,8 @@ heading, and (b) bumping `pubspec.yaml` `version:` in the same commit.
|
||||
persistent per-platform dir (Windows `%LOCALAPPDATA%`, macOS `~/Library/Logs`,
|
||||
Linux `$XDG_STATE_HOME`), fsyncing warn/error + pty/ffi records immediately so
|
||||
a freeze leaves on-disk evidence. `CLIDE_LOG` (dart-define / env) or the
|
||||
`app.log.level` setting sets verbosity (warn in release, info in debug).
|
||||
(T-432)
|
||||
`app.log.level` setting sets verbosity (warn in release, info in debug);
|
||||
`CLIDE_LOG_DIR` redirects where the logs land. (T-432, T-436)
|
||||
- **PTY FFI breadcrumbs.** Each PTY backend drops a breadcrumb before/after
|
||||
every risky syscall (`CreatePseudoConsole`/`CreateProcessW`/`ReadFile`,
|
||||
`posix_spawn`/`read`); the reader/waiter isolates fsync their OWN file handle
|
||||
|
||||
+14
-6
@@ -65,18 +65,26 @@ String socketDirectory() {
|
||||
/// on logout/reboot — this is a DURABLE location. The whole point of the
|
||||
/// FileLogSink is that a freeze's last breadcrumbs survive the power-cycle, so
|
||||
/// the log dir must outlive a reboot.
|
||||
String logDirectory() {
|
||||
///
|
||||
/// `CLIDE_LOG_DIR` overrides everything: CI points it at a workspace dir
|
||||
/// outside the build tree so a wedged run's logs can be uploaded as an
|
||||
/// artifact (T-436), and tests redirect it to a temp dir.
|
||||
/// [env] defaults to [Platform.environment]; injectable for tests.
|
||||
String logDirectory([Map<String, String>? env]) {
|
||||
final e = env ?? Platform.environment;
|
||||
final override = e['CLIDE_LOG_DIR'];
|
||||
if (override != null && override.isNotEmpty) return override;
|
||||
if (Platform.isWindows) {
|
||||
final local = Platform.environment['LOCALAPPDATA'];
|
||||
final base = (local != null && local.isNotEmpty) ? local : '${Platform.environment['USERPROFILE'] ?? r'C:\'}\\AppData\\Local';
|
||||
final local = e['LOCALAPPDATA'];
|
||||
final base = (local != null && local.isNotEmpty) ? local : '${e['USERPROFILE'] ?? r'C:\'}\\AppData\\Local';
|
||||
return '$base\\clide\\logs';
|
||||
}
|
||||
if (Platform.isMacOS) {
|
||||
final home = Platform.environment['HOME'] ?? '/tmp';
|
||||
final home = e['HOME'] ?? '/tmp';
|
||||
return '$home/Library/Logs/clide';
|
||||
}
|
||||
final state = Platform.environment['XDG_STATE_HOME'];
|
||||
final base = (state != null && state.isNotEmpty) ? state : '${Platform.environment['HOME'] ?? '/tmp'}/.local/state';
|
||||
final state = e['XDG_STATE_HOME'];
|
||||
final base = (state != null && state.isNotEmpty) ? state : '${e['HOME'] ?? '/tmp'}/.local/state';
|
||||
return '$base/clide/logs';
|
||||
}
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ import 'kernel/kernel.dart';
|
||||
import 'src/pty/ffi/libc.dart' as libc;
|
||||
import 'src/daemon/pane_commands.dart';
|
||||
import 'src/ipc/envelope.dart';
|
||||
import 'src/ipc/paths.dart' show logDirectory;
|
||||
import 'src/panes/event_sink.dart';
|
||||
import 'src/panes/registry.dart';
|
||||
import 'src/daemon/dispatcher.dart';
|
||||
@@ -59,6 +60,7 @@ class _ClideTestAppState extends State<ClideTestApp> {
|
||||
@override
|
||||
void initState() {
|
||||
super.initState();
|
||||
_attachCrashLogging();
|
||||
WidgetsBinding.instance.addPostFrameCallback((_) => _runTests());
|
||||
Timer(_timeout, () {
|
||||
_say('timeout reached — exiting');
|
||||
@@ -66,6 +68,28 @@ class _ClideTestAppState extends State<ClideTestApp> {
|
||||
});
|
||||
}
|
||||
|
||||
/// Opt-in crash evidence for testmode (T-436): when `CLIDE_LOG_DIR` is set
|
||||
/// (CI / a manual Windows repro), tee this harness's logger to a
|
||||
/// FileLogSink and spawn the watchdog, so a wedged testmode run leaves the
|
||||
/// same log + watchdog files the real app would. `_say` breadcrumbs each
|
||||
/// test through the logger, so they land in the file too. Off by default —
|
||||
/// normal `make run-testmode` keeps the stderr-only path, no isolate.
|
||||
void _attachCrashLogging() {
|
||||
final dir = Platform.environment['CLIDE_LOG_DIR'];
|
||||
if (dir == null || dir.isEmpty) return;
|
||||
final logDir = logDirectory();
|
||||
try {
|
||||
_logger.addSink(FileLogSink(dir: Directory(logDir)).call);
|
||||
} catch (_) {}
|
||||
unawaited(_spawnWatchdog(logDir));
|
||||
}
|
||||
|
||||
Future<void> _spawnWatchdog(String logDir) async {
|
||||
try {
|
||||
await Isolate.spawn(watchdogEntry, ('$logDir/clide-watchdog.log', 500, 2000));
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
Future<void> _runTests() async {
|
||||
const workspace = String.fromEnvironment('CLIDE_PROJECT');
|
||||
const category = String.fromEnvironment('CLIDE_TESTMODE');
|
||||
|
||||
@@ -62,6 +62,12 @@ void main() {
|
||||
final home = Platform.environment['HOME']!;
|
||||
expect(logDirectory(), '$home/Library/Logs/clide');
|
||||
});
|
||||
|
||||
test('CLIDE_LOG_DIR overrides everything (CI artifact / test redirect)', () {
|
||||
expect(logDirectory({'CLIDE_LOG_DIR': '/tmp/ci-logs'}), '/tmp/ci-logs');
|
||||
// Empty override is ignored — falls through to the platform default.
|
||||
expect(logDirectory({'CLIDE_LOG_DIR': '', 'XDG_STATE_HOME': '/x', 'HOME': '/h'}), isNot('/'));
|
||||
});
|
||||
});
|
||||
|
||||
group('fnv1a64Hex (T-126 cross-check)', () {
|
||||
|
||||
@@ -25,6 +25,7 @@ library;
|
||||
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:clide/src/pty/pty_log.dart';
|
||||
import 'package:clide/src/pty/windows_pty.dart';
|
||||
|
||||
Future<void> main(List<String> args) async {
|
||||
@@ -34,6 +35,13 @@ Future<void> main(List<String> args) async {
|
||||
}
|
||||
final count = args.isNotEmpty ? (int.tryParse(args.first) ?? 1) : 1;
|
||||
|
||||
// When CLIDE_LOG_DIR is set (the soak workflow sets it), emit FFI breadcrumbs
|
||||
// so that when soak-conpty-kill.ps1 force-kills this process mid-life, the
|
||||
// reader/waiter isolates' LAST crumb (e.g. "ReadFile enter") is on disk —
|
||||
// CI then uploads it, naming what the reader was doing when killed (T-436).
|
||||
final logDir = Platform.environment['CLIDE_LOG_DIR'];
|
||||
final ptyLog = (logDir == null || logDir.isEmpty) ? PtyLog.none : PtyLog(crumbPath: '$logDir/clide-pty.crumbs.log', verbose: true);
|
||||
|
||||
final sessions = <WindowsPty>[];
|
||||
for (var i = 0; i < count; i++) {
|
||||
final s = WindowsPty.start(
|
||||
@@ -44,6 +52,7 @@ Future<void> main(List<String> args) async {
|
||||
columns: 80,
|
||||
rows: 24,
|
||||
environment: {...Platform.environment, 'TERM': 'xterm-256color'},
|
||||
log: ptyLog,
|
||||
);
|
||||
// Drain output so the reader isolate is actively pumping, closest to a
|
||||
// real live pane. Discard the bytes.
|
||||
|
||||
Reference in New Issue
Block a user