compounding.tech / pty
This repository has no description
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

fix(supervisor): keep permanent sessions alive after slow first start

Two related issues caused permanent supervised sessions to be silently
dropped after a single slow startup (observed on reboot with heavy
claude --resume sessions, see ~/.local/state/pty/supervisor.log).

1. waitForSocket timeout was hardcoded to 3000ms in spawn.ts. Heavy
children like `claude --resume` of a large conversation routinely
take 5-15s to open their socket, so the supervisor's first restart
attempt would time out before the child was ready.

2. doRestart's recovery path on retry was broken. The flow:
- First attempt: readMetadata(name) → fresh data; cleanupAll(name)
removes the metadata file; spawnDaemon throws on timeout.
- scheduleRestart catches, bumps restartCount, schedules retry.
- Second attempt: readMetadata(name) → null (file gone); doRestart
hits "skipping restart for X (no metadata)" and the session is
permanently abandoned.

Fix:
- Bump waitForSocket default to 30000ms via a new
DEFAULT_START_TIMEOUT_MS constant. Add an optional startTimeoutMs
field on SpawnDaemonOptions for callers that want tighter or looser
bounds. Earlier-exit detection still surfaces genuine failures
within milliseconds; this only governs the "alive but slow" case.
- Cache the spawn config on SupervisedSession.spawnConfig at
evaluateSession time and update it again immediately before
cleanupAll + spawnDaemon. When doRestart's readMetadata returns
null but the session is still tracked as permanent with a cached
config and not yet marked failed, fall back to
respawnFromCachedConfig instead of bailing.

Existing 17 supervisor tests stay green. A targeted regression test
for the slow-start-then-retry sequence would need spawnDaemon mocking
infrastructure that the test suite doesn't currently have; deferred.

Nathan Herald (May 7, 2026, 10:53 AM +0200) b9a07bb6 8301727d

+89 -3
+13 -1
src/spawn.ts
··· 58 58 * ``` 59 59 */ 60 60 launcher?: { command: string; args?: string[] }; 61 + /** Time in ms to wait for the daemon's Unix socket to appear before 62 + * giving up. Defaults to 30000 (30s) — generous enough for heavy 63 + * startups like `claude --resume` of a large session, while still 64 + * bounded so a hung child doesn't block forever. The earlyExit 65 + * handler still surfaces immediate failures within milliseconds, so 66 + * this only governs the "alive but slow" case. */ 67 + startTimeoutMs?: number; 61 68 } 69 + 70 + /** Default time we wait for a daemon's Unix socket to appear after 71 + * spawn before declaring the start a failure. See SpawnDaemonOptions 72 + * for rationale. */ 73 + export const DEFAULT_START_TIMEOUT_MS = 30_000; 62 74 63 75 export async function spawnDaemon(options: SpawnDaemonOptions): Promise<void> { 64 76 const stdout = process.stdout as tty.WriteStream; ··· 108 120 child.unref(); 109 121 110 122 try { 111 - await waitForSocket(options.name, 3000, () => { 123 + await waitForSocket(options.name, options.startTimeoutMs ?? DEFAULT_START_TIMEOUT_MS, () => { 112 124 if (earlyExit) { 113 125 const details = stderrOutput.trim(); 114 126 const msg = `Daemon process exited immediately (code ${earlyExitCode ?? "unknown"}).`;
+76 -2
src/supervisor.ts
··· 25 25 const SCAN_INTERVAL_MS = 10_000; 26 26 const TEMPORARY_CLEANUP_DELAY_MS = 1_000; 27 27 28 + /** Spawn parameters cached on each SupervisedSession so a restart can 29 + * retry even after `cleanupAll` has removed the on-disk metadata. 30 + * Without this, the second restart attempt of a slow-starting session 31 + * hits "no metadata" and the supervisor silently gives up. */ 32 + interface CachedSpawnConfig { 33 + command: string; 34 + args: string[]; 35 + displayCommand: string; 36 + cwd: string; 37 + tags?: Record<string, string>; 38 + } 39 + 28 40 interface SupervisedSession { 29 41 name: string; 30 42 strategy: "permanent" | "temporary"; ··· 34 46 nextBackoffMs: number; 35 47 failed: boolean; 36 48 pendingTimer: ReturnType<typeof setTimeout> | null; 49 + /** Last-known spawn parameters. Cached so a restart attempt can 50 + * succeed even if `cleanupAll` has already removed the on-disk 51 + * metadata file. Populated whenever `evaluateSession` reads fresh 52 + * metadata. */ 53 + spawnConfig?: CachedSpawnConfig; 37 54 } 38 55 39 56 interface PersistedState { ··· 200 217 201 218 const tracked = this.sessions.get(name)!; 202 219 tracked.strategy = "permanent"; 220 + tracked.spawnConfig = { 221 + command: metadata.command, 222 + args: metadata.args, 223 + displayCommand: metadata.displayCommand, 224 + cwd: metadata.cwd, 225 + tags: metadata.tags, 226 + }; 203 227 204 228 if (isExited && !tracked.failed && !tracked.pendingTimer) { 205 229 this.scheduleRestart(name); ··· 305 329 } 306 330 307 331 const metadata = readMetadata(name); 332 + const tracked = this.sessions.get(name); 333 + 334 + // Recovery for the "metadata file already removed" case: 335 + // cleanupAll runs before spawnDaemon during a restart, so if the 336 + // first attempt's spawn times out the metadata is gone for the 337 + // second attempt's readMetadata. Fall back to the cached 338 + // spawnConfig on the in-memory SupervisedSession when that 339 + // happens — without this, a single slow start permanently drops 340 + // a permanent session. 308 341 if (!metadata) { 309 - console.log(`[supervisor] skipping restart for ${name} (no metadata)`); 342 + if (tracked?.strategy === "permanent" && tracked.spawnConfig && !tracked.failed) { 343 + console.log(`[supervisor] no on-disk metadata for ${name}; recovering from cached spawnConfig`); 344 + await this.respawnFromCachedConfig(name, tracked); 345 + } else { 346 + console.log(`[supervisor] skipping restart for ${name} (no metadata)`); 347 + } 310 348 return; 311 349 } 312 350 if (metadata.tags?.strategy !== "permanent") { ··· 327 365 } 328 366 } 329 367 330 - const tracked = this.sessions.get(name); 331 368 if (!tracked) { 332 369 console.log(`[supervisor] skipping restart for ${name} (not tracked)`); 333 370 return; ··· 360 397 } 361 398 } 362 399 400 + // Cache the spawn config before cleanup so a slow start that 401 + // wipes the metadata file can still recover on the next attempt. 402 + tracked.spawnConfig = { command, args, displayCommand, cwd, tags }; 403 + 363 404 // Clean up the dead session 364 405 cleanupAll(name); 365 406 ··· 371 412 tracked.nextBackoffMs = Math.min(tracked.nextBackoffMs * BACKOFF_MULTIPLIER, MAX_BACKOFF_MS); 372 413 373 414 console.log(`[supervisor] restarted ${name} (attempt ${tracked.restartCount}/${MAX_RESTARTS})`); 415 + 416 + this.emitEvent(EventType.SESSION_RESTART, { 417 + session: name, 418 + restartCount: tracked.restartCount, 419 + backoffMs: tracked.nextBackoffMs, 420 + }); 421 + 422 + this.persistState(); 423 + } 424 + 425 + /** Restart path used when the on-disk metadata is missing. The 426 + * cached spawnConfig on the SupervisedSession was populated last 427 + * time evaluateSession saw fresh metadata, so it's still valid as 428 + * a recovery source. */ 429 + private async respawnFromCachedConfig(name: string, tracked: SupervisedSession): Promise<void> { 430 + const cfg = tracked.spawnConfig; 431 + if (!cfg) return; // caller checked already, but be defensive 432 + 433 + cleanupAll(name); 434 + await spawnDaemon({ 435 + name, 436 + command: cfg.command, 437 + args: cfg.args, 438 + displayCommand: cfg.displayCommand, 439 + cwd: cfg.cwd, 440 + tags: cfg.tags, 441 + }); 442 + 443 + tracked.restartCount++; 444 + tracked.lastRestartAt = Date.now(); 445 + tracked.nextBackoffMs = Math.min(tracked.nextBackoffMs * BACKOFF_MULTIPLIER, MAX_BACKOFF_MS); 446 + 447 + console.log(`[supervisor] restarted ${name} from cached config (attempt ${tracked.restartCount}/${MAX_RESTARTS})`); 374 448 375 449 this.emitEvent(EventType.SESSION_RESTART, { 376 450 session: name,