From 1600ced7764d3d0d055d836725bee0554f9e2cc2 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 9 Jul 2026 16:35:35 -0400 Subject: [PATCH 1/7] fix(lifecycle): reconcile owned mux sessions --- docs/lifecycle.md | 18 + src/agents/instruction-source.ts | 1 + src/commands/project.ts | 834 +++++++++++++----- src/lib/lifecycle-runtime.ts | 34 +- src/lib/project-lifecycle-sessions.ts | 267 ++++++ src/mux/mux-backend.ts | 8 + src/mux/tmux-backend.ts | 61 ++ src/mux/zellij-backend.ts | 74 +- tests/e2e/fixture.ts | 56 +- tests/e2e/run.ts | 2 + tests/e2e/scenarios/lifecycle-host-process.ts | 206 ++++- .../scenarios/lifecycle-session-recovery.ts | 449 ++++++++++ tests/project-lifecycle-sessions.test.ts | 155 ++++ tests/project-restart-command.test.ts | 29 +- 14 files changed, 1920 insertions(+), 274 deletions(-) create mode 100644 src/lib/project-lifecycle-sessions.ts create mode 100644 tests/e2e/scenarios/lifecycle-session-recovery.ts create mode 100644 tests/project-lifecycle-sessions.test.ts diff --git a/docs/lifecycle.md b/docs/lifecycle.md index ed6cefa1..a3b66800 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -179,6 +179,20 @@ Lifecycle output is now surfaced across CLI/runtime views: 5. Run `docker compose up` (or `up -d` when `--detach`). 6. Run `lifecycle.up.after` hooks. +Before starting persistent work, Hack reconciles the expected lifecycle session with +`.hack/.internal/lifecycle/state.json`: + +- A session whose mux ownership token, saved definition hash, and live process windows all match is + adopted without starting duplicate processes. +- A token-owned stale session is replaced. Pre-ownership tmux sessions are replaced only when saved + state, checkout path, creation time, and window count provide deterministic legacy ownership proof. +- A same-name session without matching ownership proof is left untouched and `hack up` fails with an + actionable collision error. + +If the operation later fails in Compose or `up.after`, Hack removes only the lifecycle session created +by that operation. The same exact-token cleanup runs for supported `SIGINT` and `SIGTERM` paths, including +signals received while the mux session is still being initialized. + ### `hack down` 1. Run `lifecycle.down.before` hooks. @@ -189,6 +203,8 @@ Lifecycle output is now surfaced across CLI/runtime views: ### `hack restart` `hack restart` performs the same lifecycle steps as `hack down` followed by `hack up`. +From the primary checkout, it targets only the base Compose/lifecycle instance. A linked worktree uses +its isolated derived branch instance, and `--branch ` targets only that explicit branch. ### `--json` @@ -214,6 +230,8 @@ Notes: - Teardown is implemented by killing the lifecycle session; anything running inside that session will be stopped. - For tmux-backed lifecycle sessions, Hack also persists the pane PID and the wrapped command's actual process-group metadata to `.hack/.internal/lifecycle/state.json`, with per-hook/process output logged to +- Current lifecycle sessions carry the same random ownership token in mux metadata and persisted state. + Cleanup requires an exact token match; deterministic names alone never authorize session teardown. `.hack/.internal/lifecycle/*.log`. If tmux pane state disappears before teardown, `hack down` still uses that persisted metadata to clean up any live lifecycle process groups instead of leaving orphaned host processes behind. - `hack doctor` reports stale lifecycle state when the persisted lifecycle entry no longer has a live mux session and points operators to `hack down` so cleanup and state removal happen through the supported path. `hack doctor --fix` does not tear down lifecycle sessions itself — use `hack down` for that. diff --git a/src/agents/instruction-source.ts b/src/agents/instruction-source.ts index 67fb2a8d..260741d7 100644 --- a/src/agents/instruction-source.ts +++ b/src/agents/instruction-source.ts @@ -185,6 +185,7 @@ export const INSTRUCTION_SECTIONS: readonly InstructionSection[] = [ 'For fixed-port host helpers such as SSM tunnels or local proxies, set `singleton.ports` and usually `onConflict: "adopt"` so Hack reuses a healthy existing listener instead of starting duplicate tunnel stacks.', "`singleton` is a listener guard, not process ownership transfer; adopted external processes are left running on `hack down`.", "Inspect lifecycle status via `hack projects --details` and stream via `hack logs `.", + "Lifecycle session recovery is ownership-proven: Hack adopts healthy token-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof.", ], }, { diff --git a/src/commands/project.ts b/src/commands/project.ts index 79f84f0c..a52a41bd 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -120,6 +120,7 @@ import { type LifecycleStateEntry, readLifecycleState, removeLifecycleStateEntry, + removeLifecycleStateEntryIfOwned, resolveLifecycleComposeProjectName, resolveLifecycleLogPath, upsertLifecycleStateEntry, @@ -162,6 +163,13 @@ import { resolveLifecycleProcessGroupIdsForTmuxState, resolveLifecycleStopProcessGroupIds, } from "../lib/project-lifecycle-processes.ts"; +import { + createLifecycleOwnershipToken, + inspectLifecycleSession, + killInspectedLifecycleSession, + killLifecycleSessionWithOwnership, + resolveLifecycleDefinitionHash, +} from "../lib/project-lifecycle-sessions.ts"; import { inspectListeningTcpPorts, resolveLifecycleSingletonDecision, @@ -185,7 +193,7 @@ import { parseTimeInput } from "../lib/time.ts"; import { upsertAgentDocs } from "../mcp/agent-docs.ts"; import type { McpTarget } from "../mcp/install.ts"; import { installMcpConfig } from "../mcp/install.ts"; -import type { MuxBackendName } from "../mux/mux-backend.ts"; +import type { MuxBackend, MuxBackendName } from "../mux/mux-backend.ts"; import { getMuxBackends, resolveDefaultBackendName, @@ -1587,6 +1595,15 @@ type StartedLifecycleProcess = { readonly logPath: string; }; +type LifecycleOperationCleanup = () => Promise; + +type LifecycleUpResult = { + readonly code: number; + readonly sessionName: string | null; + readonly cleanup: LifecycleOperationCleanup | null; + readonly signalCleanup: { readonly dispose: () => void } | null; +}; + function hasPersistentLifecycleCommands( commands: readonly ProjectLifecycleCommand[] | undefined ): boolean { @@ -1601,8 +1618,12 @@ async function runLifecycleUpBeforeAndProcesses(opts: { readonly branch: string | null; readonly env: Readonly>; readonly composeProject: string; -}): Promise<{ readonly code: number; readonly sessionName: string | null }> { +}): Promise { const beforeCommands = opts.cfg.lifecycle?.up?.before; + const definitionHash = resolveProjectLifecycleDefinitionHash({ + beforeCommands, + processes: opts.cfg.lifecycle?.processes, + }); if (!hasPersistentLifecycleCommands(beforeCommands)) { const beforeCode = await runLifecycleCommands({ title: opts.title, @@ -1613,15 +1634,21 @@ async function runLifecycleUpBeforeAndProcesses(opts: { composeProject: opts.composeProject, }); if (beforeCode !== 0) { - return { code: beforeCode, sessionName: null }; + return { + code: beforeCode, + sessionName: null, + cleanup: null, + signalCleanup: null, + }; } - await startLifecycleProcesses({ + const lifecycleStart = await startLifecycleProcesses({ project: opts.project, cfg: opts.cfg, projectName: opts.projectName, branch: opts.branch, env: opts.env, composeProject: opts.composeProject, + definitionHash, }); const hasProcesses = (opts.cfg.lifecycle?.processes ?? []).length > 0; return { @@ -1632,6 +1659,8 @@ async function runLifecycleUpBeforeAndProcesses(opts: { branch: opts.branch, }) : null, + cleanup: lifecycleStart.cleanup, + signalCleanup: lifecycleStart.signalCleanup, }; } @@ -1641,6 +1670,7 @@ async function runLifecycleUpBeforeAndProcesses(opts: { branch: opts.branch, env: opts.env, composeProject: opts.composeProject, + definitionHash, }); const beforeCode = await runLifecycleCommands({ title: opts.title, @@ -1658,7 +1688,12 @@ async function runLifecycleUpBeforeAndProcesses(opts: { }); if (beforeCode !== 0) { await starter.abort(); - return { code: beforeCode, sessionName: null }; + return { + code: beforeCode, + sessionName: null, + cleanup: null, + signalCleanup: null, + }; } try { @@ -1677,7 +1712,137 @@ async function runLifecycleUpBeforeAndProcesses(opts: { return { code: 0, sessionName: starter.hasStarted() ? starter.sessionName : null, + cleanup: starter.getOperationCleanup(), + signalCleanup: starter.getSignalCleanup(), + }; +} + +function installLifecycleSignalCleanup(opts: { + readonly cleanup: LifecycleOperationCleanup | null; +}): { readonly dispose: () => void } { + if (!opts.cleanup) { + return { dispose: () => undefined }; + } + + let handlingSignal = false; + const handlers = new Map void>(); + const dispose = (): void => { + for (const [signal, handler] of handlers) { + process.off(signal, handler); + } + handlers.clear(); }; + for (const signal of ["SIGINT", "SIGTERM"] as const) { + const handler = (): void => { + if (handlingSignal) { + return; + } + handlingSignal = true; + void opts.cleanup?.().finally(() => { + dispose(); + process.kill(process.pid, signal); + }); + }; + handlers.set(signal, handler); + process.on(signal, handler); + } + return { dispose }; +} + +function resolveProjectLifecycleDefinitionHash(opts: { + readonly beforeCommands: readonly ProjectLifecycleCommand[] | undefined; + readonly processes: readonly ProjectLifecycleProcess[] | undefined; +}): string { + const persistentBefore = (opts.beforeCommands ?? []) + .map((command, index) => ({ command, index })) + .filter(({ command }) => command.persistent === true) + .map(({ command, index }) => ({ + kind: "up-before", + name: resolveLifecycleCommandServiceName({ command, index }), + command: command.command, + cwd: command.cwd ?? null, + singleton: command.singleton ?? null, + })); + const processes = (opts.processes ?? []).map((process) => ({ + kind: "process", + name: process.name, + command: process.command, + cwd: process.cwd ?? null, + singleton: process.singleton ?? null, + })); + return resolveLifecycleDefinitionHash({ + definitions: [...persistentBefore, ...processes], + }); +} + +async function reconcileChangedLifecycleBackend(opts: { + readonly mux: Awaited>; + readonly entry: LifecycleStateEntry | null; + readonly resolvedBackend: MuxBackendName; + readonly sessionName: string; + readonly projectRoot: string; + readonly projectDir: string; + readonly composeProject: string; + readonly definitionHash: string; +}): Promise { + if (!opts.entry || opts.entry.backend === opts.resolvedBackend) { + return opts.entry; + } + const previousBackend = opts.mux.backends.get(opts.entry.backend); + if (!previousBackend?.available) { + throw new Error( + `Lifecycle backend changed to ${opts.resolvedBackend}, but the owned ${opts.entry.backend} session cannot be inspected safely because ${opts.entry.backend} is unavailable.` + ); + } + const inspection = await inspectLifecycleSession({ + backend: previousBackend, + entry: opts.entry, + expectedSessionName: opts.sessionName, + expectedProjectRoot: opts.projectRoot, + expectedDefinitionHash: opts.definitionHash, + }); + if (inspection.decision.kind === "block") { + throw new Error(inspection.decision.reason); + } + if ( + inspection.classification !== "absent" && + !(await killInspectedLifecycleSession({ + backend: previousBackend, + inspection, + })) + ) { + throw new Error( + `Failed to stop the owned ${opts.entry.backend} lifecycle session before switching to ${opts.resolvedBackend}.` + ); + } + await removeLifecycleStateEntry({ + projectDir: opts.projectDir, + composeProject: opts.composeProject, + }); + return null; +} + +async function replaceInspectedLifecycleSession(opts: { + readonly backend: MuxBackend; + readonly inspection: Awaited>; + readonly sessionName: string; + readonly projectDir: string; + readonly composeProject: string; +}): Promise { + if ( + !(await killInspectedLifecycleSession({ + backend: opts.backend, + inspection: opts.inspection, + })) + ) { + throw new Error( + `Failed to replace stale lifecycle session: ${opts.sessionName}` + ); + } + await removeLifecycleStateEntry({ + projectDir: opts.projectDir, + composeProject: opts.composeProject, + }); } function createLifecycleProcessStarter(opts: { @@ -1686,6 +1851,7 @@ function createLifecycleProcessStarter(opts: { readonly branch: string | null; readonly env: Readonly>; readonly composeProject: string; + readonly definitionHash: string; }): { readonly sessionName: string; startFromCommand: (opts: { @@ -1698,6 +1864,8 @@ function createLifecycleProcessStarter(opts: { finalize: () => Promise; abort: () => Promise; hasStarted: () => boolean; + getOperationCleanup: () => LifecycleOperationCleanup | null; + getSignalCleanup: () => { readonly dispose: () => void } | null; } { const sessionName = resolveLifecycleSessionName({ projectName: opts.projectName, @@ -1706,19 +1874,15 @@ function createLifecycleProcessStarter(opts: { const startedProcesses: StartedLifecycleProcess[] = []; let backendName: MuxBackendName | null = null; let sessionReady = false; - let existingSessionCleared = false; + let sessionDispositionResolved = false; + let createdOwnershipToken: string | null = null; + let adoptedEntry: LifecycleStateEntry | null = null; + let operationSignalCleanup: { readonly dispose: () => void } | null = null; + let sessionCreationSettled: Promise = Promise.resolve(); let nextIndex = 0; - const ensureExistingSessionCleared = async (): Promise => { - if (existingSessionCleared) { - return; - } - await killLifecycleSessionByName({ sessionName }); - existingSessionCleared = true; - }; - - const ensureSession = async (): Promise => { - if (sessionReady) { + const resolveSessionDisposition = async (): Promise => { + if (sessionDispositionResolved) { return; } const mux = await resolveMux({ project: opts.project }); @@ -1738,29 +1902,119 @@ function createLifecycleProcessStarter(opts: { if (!backend?.available) { throw new Error(`${resolvedBackend} is not available`); } - await ensureExistingSessionCleared(); - const created = await backend.createSession({ - name: sessionName, - cwd: opts.project.projectRoot, + const entries = await readLifecycleState({ + projectDir: opts.project.projectDir, }); - if (!created.ok) { - throw new Error(`Failed to create lifecycle session: ${sessionName}`); + const persistedEntry = + entries.find( + (candidate) => candidate.composeProject === opts.composeProject + ) ?? null; + const entry = await reconcileChangedLifecycleBackend({ + mux, + entry: persistedEntry, + resolvedBackend, + sessionName, + projectRoot: opts.project.projectRoot, + projectDir: opts.project.projectDir, + composeProject: opts.composeProject, + definitionHash: opts.definitionHash, + }); + backendName = resolvedBackend; + const inspection = await inspectLifecycleSession({ + backend, + entry, + expectedSessionName: sessionName, + expectedProjectRoot: opts.project.projectRoot, + expectedDefinitionHash: opts.definitionHash, + }); + if (inspection.decision.kind === "block") { + throw new Error(inspection.decision.reason); } - if (resolvedBackend === "tmux") { - for (const [key, value] of Object.entries(opts.env)) { - await exec(["tmux", "set-environment", "-t", sessionName, key, value], { - stdin: "ignore", - }); + if (inspection.decision.kind === "adopt") { + adoptedEntry = inspection.decision.entry; + startedProcesses.push(...inspection.decision.entry.processes); + sessionReady = true; + sessionDispositionResolved = true; + return; + } + if (inspection.decision.kind === "replace") { + await replaceInspectedLifecycleSession({ + backend, + inspection, + sessionName, + projectDir: opts.project.projectDir, + composeProject: opts.composeProject, + }); + } + sessionDispositionResolved = true; + }; + + const ensureSession = async (): Promise => { + await resolveSessionDisposition(); + if (sessionReady) { + return; + } + const backend = backendName + ? (await resolveMux({ project: opts.project })).backends.get(backendName) + : null; + if (!(backendName && backend?.available)) { + throw new Error("Lifecycle mux backend became unavailable"); + } + const ownershipToken = createLifecycleOwnershipToken(); + let settleSessionCreation = (): void => undefined; + sessionCreationSettled = new Promise((resolvePromise) => { + settleSessionCreation = resolvePromise; + }); + operationSignalCleanup = installLifecycleSignalCleanup({ + cleanup: starterAbort, + }); + try { + const created = await backend.createSession({ + name: sessionName, + cwd: opts.project.projectRoot, + lifecycleOwnerToken: ownershipToken, + }); + if (!created.ok) { + throw new Error(`Failed to create lifecycle session: ${sessionName}`); } + createdOwnershipToken = ownershipToken; + await upsertLifecycleStateEntry({ + projectDir: opts.project.projectDir, + entry: { + composeProject: opts.composeProject, + projectName: opts.projectName, + branch: opts.branch, + sessionName, + backend: backendName, + ownershipToken, + definitionHash: opts.definitionHash, + processes: [], + updatedAt: new Date().toISOString(), + }, + }); + if (backendName === "tmux") { + for (const [key, value] of Object.entries(opts.env)) { + await exec( + ["tmux", "set-environment", "-t", sessionName, key, value], + { + stdin: "ignore", + } + ); + } + } + } finally { + settleSessionCreation(); } - backendName = resolvedBackend; sessionReady = true; }; const startProcess = async ( process: ProjectLifecycleProcess ): Promise => { - await ensureExistingSessionCleared(); + await resolveSessionDisposition(); + if (adoptedEntry?.processes.some((entry) => entry.name === process.name)) { + return; + } const singletonDecision = await resolveLifecycleSingletonDecisionForProcess( { process, @@ -1802,6 +2056,7 @@ function createLifecycleProcessStarter(opts: { }); }, startMany: async ({ processes }) => { + await resolveSessionDisposition(); for (const process of processes) { await startProcess(process); } @@ -1814,6 +2069,8 @@ function createLifecycleProcessStarter(opts: { }); return; } + const ownershipToken = + createdOwnershipToken ?? adoptedEntry?.ownershipToken; await upsertLifecycleStateEntry({ projectDir: opts.project.projectDir, entry: { @@ -1822,37 +2079,47 @@ function createLifecycleProcessStarter(opts: { branch: opts.branch, sessionName, backend: backendName, + ...(ownershipToken ? { ownershipToken } : {}), + definitionHash: opts.definitionHash, processes: startedProcesses, updatedAt: new Date().toISOString(), }, }); }, abort: async () => { - if (sessionReady) { - await killLifecycleSessionByName({ sessionName }); - } - await removeLifecycleStateEntry({ - projectDir: opts.project.projectDir, - composeProject: opts.composeProject, - }); + await starterAbort(); + operationSignalCleanup?.dispose(); }, hasStarted: () => startedProcesses.length > 0, + getOperationCleanup: () => + createdOwnershipToken + ? async () => { + await starterAbort(); + } + : null, + getSignalCleanup: () => operationSignalCleanup, }; -} -async function killLifecycleSessionByName(opts: { - readonly sessionName: string; -}): Promise { - const backends = getMuxBackends(); - for (const backend of backends.values()) { - if (!backend.available) { - continue; + async function starterAbort(): Promise { + await sessionCreationSettled; + if (!(createdOwnershipToken && backendName)) { + return; } - const sessions = await backend.listSessions(); - if (!sessions.some((session) => session.name === opts.sessionName)) { - continue; + const backend = (await resolveMux({ project: opts.project })).backends.get( + backendName + ); + if (backend?.available) { + await killLifecycleSessionWithOwnership({ + backend, + sessionName, + ownershipToken: createdOwnershipToken, + }); } - await backend.killSession({ name: opts.sessionName }); + await removeLifecycleStateEntryIfOwned({ + projectDir: opts.project.projectDir, + composeProject: opts.composeProject, + ownershipToken: createdOwnershipToken, + }); } } @@ -1863,14 +2130,23 @@ async function startLifecycleProcesses(opts: { readonly branch: string | null; readonly env: Readonly>; readonly composeProject: string; -}): Promise { + readonly definitionHash: string; +}): Promise<{ + readonly cleanup: LifecycleOperationCleanup | null; + readonly signalCleanup: { readonly dispose: () => void } | null; +}> { const processes = opts.cfg.lifecycle?.processes ?? []; if (processes.length === 0) { - await removeLifecycleStateEntry({ + const existingEntries = await readLifecycleState({ projectDir: opts.project.projectDir, - composeProject: opts.composeProject, }); - return; + if ( + !existingEntries.some( + (entry) => entry.composeProject === opts.composeProject + ) + ) { + return { cleanup: null, signalCleanup: null }; + } } const starter = createLifecycleProcessStarter({ @@ -1879,10 +2155,15 @@ async function startLifecycleProcesses(opts: { branch: opts.branch, env: opts.env, composeProject: opts.composeProject, + definitionHash: opts.definitionHash, }); try { await starter.startMany({ processes }); await starter.finalize(); + return { + cleanup: starter.getOperationCleanup(), + signalCleanup: starter.getSignalCleanup(), + }; } catch (error: unknown) { await starter.abort(); if (error instanceof Error) { @@ -2059,7 +2340,17 @@ async function startLifecycleProcess(opts: { serviceName: opts.process.name, }); const result = await exec( - ["zellij", "run", "--", "sh", "-c", wrappedCommand], + [ + "zellij", + "run", + "--close-on-exit", + "--name", + windowName, + "--", + "sh", + "-c", + wrappedCommand, + ], { stdin: "ignore", cwd, @@ -2095,15 +2386,6 @@ async function stopLifecycleProcesses(opts: { readonly branch: string | null; readonly composeProject: string; }): Promise { - const lifecycle = opts.cfg.lifecycle; - if (!lifecycle) { - await removeLifecycleStateEntry({ - projectDir: opts.project.projectDir, - composeProject: opts.composeProject, - }); - return; - } - const sessionName = resolveLifecycleSessionName({ projectName: opts.projectName, branch: opts.branch, @@ -2116,24 +2398,53 @@ async function stopLifecycleProcesses(opts: { (entry) => entry.composeProject === opts.composeProject ) ?? null; - const backends = getMuxBackends(); - let matchedLiveSession = false; - for (const backend of backends.values()) { - if (!backend.available) { - continue; - } - const sessions = await backend.listSessions(); - if (!sessions.some((s) => s.name === sessionName)) { - continue; - } - matchedLiveSession = true; - if (backend.name === "tmux") { - await interruptLifecycleTmuxProcesses({ - sessionName, - lifecycleEntry, - }); + const backend = lifecycleEntry + ? getMuxBackends().get(lifecycleEntry.backend) + : null; + if (!lifecycleEntry) { + return; + } + if (!backend?.available) { + throw new Error( + `Lifecycle backend ${lifecycleEntry.backend} is unavailable; refusing unverified session cleanup` + ); + } + + const definitionHash = + lifecycleEntry.definitionHash ?? + resolveProjectLifecycleDefinitionHash({ + beforeCommands: opts.cfg.lifecycle?.up?.before, + processes: opts.cfg.lifecycle?.processes, + }); + const inspection = await inspectLifecycleSession({ + backend, + entry: lifecycleEntry, + expectedSessionName: sessionName, + expectedProjectRoot: opts.project.projectRoot, + expectedDefinitionHash: definitionHash, + }); + if (inspection.decision.kind === "block") { + throw new Error(inspection.decision.reason); + } + + const matchedLiveSession = inspection.classification !== "absent"; + if (matchedLiveSession && backend.name === "tmux") { + await interruptLifecycleTmuxProcesses({ + sessionName, + lifecycleEntry, + }); + } + if (matchedLiveSession) { + const killed = lifecycleEntry.ownershipToken + ? await killLifecycleSessionWithOwnership({ + backend, + sessionName, + ownershipToken: lifecycleEntry.ownershipToken, + }) + : (await backend.killSession({ name: sessionName })).exitCode === 0; + if (!killed) { + throw new Error(`Failed to stop owned lifecycle session: ${sessionName}`); } - await backend.killSession({ name: sessionName }); } await terminateLifecycleProcessGroups({ @@ -2149,6 +2460,20 @@ async function stopLifecycleProcesses(opts: { }); } +async function stopLifecycleProcessesBestEffort( + opts: Parameters[0] +): Promise { + try { + await stopLifecycleProcesses(opts); + } catch (error: unknown) { + const message = + error instanceof Error + ? error.message + : "Failed to stop lifecycle processes"; + logger.warn({ message }); + } +} + async function interruptLifecycleTmuxProcesses(opts: { readonly sessionName: string; readonly lifecycleEntry: LifecycleStateEntry | null; @@ -5455,6 +5780,8 @@ async function runUpCommand({ envName: envOverrides.effectiveEnvName, }); + let lifecycleCleanup: LifecycleOperationCleanup | null = null; + let lifecycleSignalCleanup: { readonly dispose: () => void } | null = null; try { const lifecycleUp = await runLifecycleUpBeforeAndProcesses({ title: "Lifecycle (up before)", @@ -5482,6 +5809,8 @@ async function runUpCommand({ message: `Lifecycle processes running in session: ${lifecycleUp.sessionName}`, }); } + lifecycleCleanup = lifecycleUp.cleanup; + lifecycleSignalCleanup = lifecycleUp.signalCleanup; } catch (error: unknown) { const message = error instanceof Error @@ -5497,76 +5826,88 @@ async function runUpCommand({ return 1; } - const upCode = await composeRuntimeBackend.up({ - composeFiles: composeFilesWithEnv, - composeProject: composeProjectName, - profiles, - detach, - cwd: dirname(project.composeFile), - env: envOverrides.env, - routeStdoutToStderr: json, - }); - if (upCode !== 0) { - await removeProjectRuntimeStateEntry({ + const signalCleanup = + lifecycleSignalCleanup ?? + installLifecycleSignalCleanup({ cleanup: lifecycleCleanup }); + try { + const upCode = await composeRuntimeBackend.up({ + composeFiles: composeFilesWithEnv, + composeProject: composeProjectName, + profiles, + detach, + cwd: dirname(project.composeFile), + env: envOverrides.env, + routeStdoutToStderr: json, + }); + if (upCode !== 0) { + await lifecycleCleanup?.(); + await removeProjectRuntimeStateEntry({ + projectDir: project.projectDir, + composeProject: lifecycleComposeProject, + }); + if (json) { + return emitLifecycleResult({ + result: errorResult({ + code: "E_COMPOSE_FAILED", + message: `docker compose up failed (exit ${upCode})`, + detail: { exitCode: upCode }, + }), + exitCode: upCode, + }); + } + return upCode; + } + + const afterCode = await runLifecycleCommands({ + title: "Lifecycle (up after)", + commands: cfg.lifecycle?.up?.after, + projectRoot: project.projectRoot, + env: envOverrides.env, projectDir: project.projectDir, composeProject: lifecycleComposeProject, }); - if (json) { + + if (!json) { + if (afterCode !== 0) { + await lifecycleCleanup?.(); + } + return afterCode; + } + + if (afterCode !== 0) { + await lifecycleCleanup?.(); return emitLifecycleResult({ result: errorResult({ - code: "E_COMPOSE_FAILED", - message: `docker compose up failed (exit ${upCode})`, - detail: { exitCode: upCode }, + code: "E_LIFECYCLE_FAILED", + message: `Lifecycle (up after) failed (exit ${afterCode})`, }), - exitCode: upCode, + exitCode: afterCode, }); } - return upCode; - } - - const afterCode = await runLifecycleCommands({ - title: "Lifecycle (up after)", - commands: cfg.lifecycle?.up?.after, - projectRoot: project.projectRoot, - env: envOverrides.env, - projectDir: project.projectDir, - composeProject: lifecycleComposeProject, - }); - - if (!json) { - return afterCode; - } - if (afterCode !== 0) { + const states = await readComposeServiceStates({ + project, + composeProjectName, + profiles, + }); + const { running, notRunning } = splitServiceStates(states); return emitLifecycleResult({ - result: errorResult({ - code: "E_LIFECYCLE_FAILED", - message: `Lifecycle (up after) failed (exit ${afterCode})`, + result: okResult({ + data: buildLifecycleJsonData({ + action: "up", + project: baseProjectName, + branch, + composeProject: composeProjectName ?? baseProjectName, + started: running, + failed: notRunning, + durationMs: Date.now() - (startedAtMs ?? Date.now()), + }), }), - exitCode: afterCode, + exitCode: 0, }); + } finally { + signalCleanup.dispose(); } - - const states = await readComposeServiceStates({ - project, - composeProjectName, - profiles, - }); - const { running, notRunning } = splitServiceStates(states); - return emitLifecycleResult({ - result: okResult({ - data: buildLifecycleJsonData({ - action: "up", - project: baseProjectName, - branch, - composeProject: composeProjectName ?? baseProjectName, - started: running, - failed: notRunning, - durationMs: Date.now() - (startedAtMs ?? Date.now()), - }), - }), - exitCode: 0, - }); } async function maybePromptToStartGlobal(opts: { @@ -5698,7 +6039,6 @@ async function runDownCommand({ const baseProjectName = await resolveComposeProjectName({ project, cfg }); const composeProjectName = branch ? `${baseProjectName}--${branch}` : null; - const projectName = sanitizeProjectSlug(baseProjectName); const lifecycleComposeProject = resolveLifecycleComposeProjectName({ projectName, @@ -5721,49 +6061,52 @@ async function runDownCommand({ ? await readComposeServiceStates({ project, composeProjectName, profiles }) : []; - const beforeCode = await runLifecycleCommands({ - title: "Lifecycle (down before)", - commands: cfg.lifecycle?.down?.before, - projectRoot: project.projectRoot, - env: lifecycleEnv, - projectDir: project.projectDir, - composeProject: lifecycleComposeProject, - }); - if (beforeCode !== 0) { - if (json) { - return emitLifecycleResult({ - result: errorResult({ - code: "E_LIFECYCLE_FAILED", - message: `Lifecycle (down before) failed (exit ${beforeCode})`, - }), - exitCode: beforeCode, - }); - } - return beforeCode; - } - - const code = await composeRuntimeBackend.down({ - composeFiles: [project.composeFile], - composeProject: composeProjectName, - profiles, - cwd: dirname(project.composeFile), - routeStdoutToStderr: json, - }); - - try { - await stopLifecycleProcesses({ + const lifecycleDownCleanup = async (): Promise => { + await stopLifecycleProcessesBestEffort({ project, cfg, projectName, branch, composeProject: lifecycleComposeProject, }); - } catch (error: unknown) { - const message = - error instanceof Error - ? error.message - : "Failed to stop lifecycle processes"; - logger.warn({ message }); + }; + const signalCleanup = installLifecycleSignalCleanup({ + cleanup: lifecycleDownCleanup, + }); + let code: number; + try { + const beforeCode = await runLifecycleCommands({ + title: "Lifecycle (down before)", + commands: cfg.lifecycle?.down?.before, + projectRoot: project.projectRoot, + env: lifecycleEnv, + projectDir: project.projectDir, + composeProject: lifecycleComposeProject, + }); + if (beforeCode !== 0) { + await lifecycleDownCleanup(); + if (json) { + return emitLifecycleResult({ + result: errorResult({ + code: "E_LIFECYCLE_FAILED", + message: `Lifecycle (down before) failed (exit ${beforeCode})`, + }), + exitCode: beforeCode, + }); + } + return beforeCode; + } + + code = await composeRuntimeBackend.down({ + composeFiles: [project.composeFile], + composeProject: composeProjectName, + profiles, + cwd: dirname(project.composeFile), + routeStdoutToStderr: json, + }); + await lifecycleDownCleanup(); + } finally { + signalCleanup.dispose(); } if (code !== 0) { @@ -5834,39 +6177,43 @@ async function runRestartDownPhase(opts: { readonly routeStdoutToStderr?: boolean; readonly envForCompose: Readonly>; }): Promise { - const downBefore = await runLifecycleCommands({ - title: "Lifecycle (restart down before)", - commands: opts.cfg.lifecycle?.down?.before, - projectRoot: opts.project.projectRoot, - env: opts.envForCompose, - projectDir: opts.project.projectDir, - composeProject: opts.lifecycleComposeProject, - }); - if (downBefore !== 0) { - return downBefore; - } - - const downCode = await composeRuntimeBackend.down({ - composeFiles: [opts.project.composeFile], - composeProject: opts.composeProjectName, - profiles: opts.profiles, - cwd: dirname(opts.project.composeFile), - routeStdoutToStderr: opts.routeStdoutToStderr === true, - }); - try { - await stopLifecycleProcesses({ + const lifecycleDownCleanup = async (): Promise => { + await stopLifecycleProcessesBestEffort({ project: opts.project, cfg: opts.cfg, projectName: opts.projectName, branch: opts.branch, composeProject: opts.lifecycleComposeProject, }); - } catch (error: unknown) { - const message = - error instanceof Error - ? error.message - : "Failed to stop lifecycle processes"; - logger.warn({ message }); + }; + const signalCleanup = installLifecycleSignalCleanup({ + cleanup: lifecycleDownCleanup, + }); + let downCode: number; + try { + const downBefore = await runLifecycleCommands({ + title: "Lifecycle (restart down before)", + commands: opts.cfg.lifecycle?.down?.before, + projectRoot: opts.project.projectRoot, + env: opts.envForCompose, + projectDir: opts.project.projectDir, + composeProject: opts.lifecycleComposeProject, + }); + if (downBefore !== 0) { + await lifecycleDownCleanup(); + return downBefore; + } + + downCode = await composeRuntimeBackend.down({ + composeFiles: [opts.project.composeFile], + composeProject: opts.composeProjectName, + profiles: opts.profiles, + cwd: dirname(opts.project.composeFile), + routeStdoutToStderr: opts.routeStdoutToStderr === true, + }); + await lifecycleDownCleanup(); + } finally { + signalCleanup.dispose(); } if (downCode !== 0) { return downCode; @@ -5950,6 +6297,8 @@ async function runRestartUpPhase(opts: { envName: envOverrides.effectiveEnvName, }); + let lifecycleCleanup: LifecycleOperationCleanup | null = null; + let lifecycleSignalCleanup: { readonly dispose: () => void } | null = null; try { const lifecycleUp = await runLifecycleUpBeforeAndProcesses({ title: "Lifecycle (restart up before)", @@ -5968,6 +6317,8 @@ async function runRestartUpPhase(opts: { message: `Lifecycle processes running in session: ${lifecycleUp.sessionName}`, }); } + lifecycleCleanup = lifecycleUp.cleanup; + lifecycleSignalCleanup = lifecycleUp.signalCleanup; } catch (error: unknown) { const message = error instanceof Error @@ -5977,32 +6328,43 @@ async function runRestartUpPhase(opts: { return 1; } - const upCode = await composeRuntimeBackend.up({ - composeFiles: composeFilesWithEnv, - composeProject: opts.composeProjectName, - profiles: opts.profiles, - detach: opts.detach === true, - cwd: dirname(opts.project.composeFile), - env: envOverrides.env, - routeStdoutToStderr: opts.routeStdoutToStderr === true, - }); - if (upCode !== 0) { - await removeProjectRuntimeStateEntry({ + const signalCleanup = + lifecycleSignalCleanup ?? + installLifecycleSignalCleanup({ cleanup: lifecycleCleanup }); + try { + const upCode = await composeRuntimeBackend.up({ + composeFiles: composeFilesWithEnv, + composeProject: opts.composeProjectName, + profiles: opts.profiles, + detach: opts.detach === true, + cwd: dirname(opts.project.composeFile), + env: envOverrides.env, + routeStdoutToStderr: opts.routeStdoutToStderr === true, + }); + if (upCode !== 0) { + await lifecycleCleanup?.(); + await removeProjectRuntimeStateEntry({ + projectDir: opts.project.projectDir, + composeProject: opts.lifecycleComposeProject, + }); + return upCode; + } + + const upAfter = await runLifecycleCommands({ + title: "Lifecycle (restart up after)", + commands: opts.cfg.lifecycle?.up?.after, + projectRoot: opts.project.projectRoot, + env: envOverrides.env, projectDir: opts.project.projectDir, composeProject: opts.lifecycleComposeProject, }); - return upCode; + if (upAfter !== 0) { + await lifecycleCleanup?.(); + } + return upAfter; + } finally { + signalCleanup.dispose(); } - - const upAfter = await runLifecycleCommands({ - title: "Lifecycle (restart up after)", - commands: opts.cfg.lifecycle?.up?.after, - projectRoot: opts.project.projectRoot, - env: envOverrides.env, - projectDir: opts.project.projectDir, - composeProject: opts.lifecycleComposeProject, - }); - return upAfter; } async function handleRestart({ @@ -6095,13 +6457,11 @@ async function runRestartCommand({ } const baseProjectName = await resolveComposeProjectName({ project, cfg }); - const composeProjectName = branch ? `${baseProjectName}--${branch}` : null; - - const projectName = sanitizeProjectSlug(baseProjectName); - const lifecycleComposeProject = resolveLifecycleComposeProjectName({ - projectName, + const { composeProjectName, lifecycleComposeProject } = resolveRestartTarget({ + baseProjectName, branch, }); + const projectName = sanitizeProjectSlug(baseProjectName); const effectiveEnvName = await resolveStoredRuntimeEnvName({ requestedEnvName: envName, projectDir: project.projectDir, @@ -6210,6 +6570,24 @@ export async function resolveStoredRuntimeEnvName(opts: { return state?.envName ?? undefined; } +export function resolveRestartTarget(opts: { + readonly baseProjectName: string; + readonly branch: string | null; +}): { + readonly composeProjectName: string | null; + readonly lifecycleComposeProject: string; +} { + return { + composeProjectName: opts.branch + ? `${opts.baseProjectName}--${opts.branch}` + : null, + lifecycleComposeProject: resolveLifecycleComposeProjectName({ + projectName: sanitizeProjectSlug(opts.baseProjectName), + branch: opts.branch, + }), + }; +} + async function persistProjectRuntimeEnvSelection(opts: { readonly projectDir: string; readonly composeProject: string; diff --git a/src/lib/lifecycle-runtime.ts b/src/lib/lifecycle-runtime.ts index 29bce4f0..be199949 100644 --- a/src/lib/lifecycle-runtime.ts +++ b/src/lib/lifecycle-runtime.ts @@ -19,6 +19,8 @@ export type LifecycleStateEntry = { readonly branch: string | null; readonly sessionName: string; readonly backend: LifecycleBackend; + readonly ownershipToken?: string; + readonly definitionHash?: string; readonly processes: readonly LifecycleStateProcess[]; readonly updatedAt: string; }; @@ -193,6 +195,29 @@ export async function removeLifecycleStateEntry(opts: { }); } +export async function removeLifecycleStateEntryIfOwned(opts: { + readonly projectDir: string; + readonly composeProject: string; + readonly ownershipToken: string; +}): Promise { + const state = await readLifecycleState({ projectDir: opts.projectDir }); + const current = state.find( + (entry) => entry.composeProject === opts.composeProject + ); + if (current?.ownershipToken !== opts.ownershipToken) { + return false; + } + await writeLifecycleStateFile({ + projectDir: opts.projectDir, + state: { + entries: state.filter( + (entry) => entry.composeProject !== opts.composeProject + ), + }, + }); + return true; +} + function parseLifecycleStateEntry(value: unknown): LifecycleStateEntry | null { if (!isRecord(value)) { return null; @@ -202,6 +227,8 @@ function parseLifecycleStateEntry(value: unknown): LifecycleStateEntry | null { const sessionName = getString(value, "sessionName")?.trim(); const backendRaw = getString(value, "backend")?.trim(); const updatedAt = getString(value, "updatedAt")?.trim(); + const ownershipToken = getString(value, "ownershipToken")?.trim(); + const definitionHash = getString(value, "definitionHash")?.trim(); if ( !(composeProject && projectName && sessionName && backendRaw && updatedAt) ) { @@ -216,7 +243,7 @@ function parseLifecycleStateEntry(value: unknown): LifecycleStateEntry | null { ? branchRaw.trim() : null; const processesRaw = value.processes; - if (!Array.isArray(processesRaw) || processesRaw.length === 0) { + if (!Array.isArray(processesRaw)) { return null; } const processes: LifecycleStateProcess[] = []; @@ -226,15 +253,14 @@ function parseLifecycleStateEntry(value: unknown): LifecycleStateEntry | null { processes.push(parsed); } } - if (processes.length === 0) { - return null; - } return { composeProject, projectName, branch, sessionName, backend: backendRaw, + ...(ownershipToken ? { ownershipToken } : {}), + ...(definitionHash ? { definitionHash } : {}), processes, updatedAt, }; diff --git a/src/lib/project-lifecycle-sessions.ts b/src/lib/project-lifecycle-sessions.ts new file mode 100644 index 00000000..974ed65b --- /dev/null +++ b/src/lib/project-lifecycle-sessions.ts @@ -0,0 +1,267 @@ +import { randomUUID } from "node:crypto"; +import { realpath } from "node:fs/promises"; +import type { + MuxBackend, + MuxBackendName, + MuxSession, +} from "../mux/mux-backend.ts"; +import type { LifecycleStateEntry } from "./lifecycle-runtime.ts"; + +export type LifecycleSessionClassification = + | "absent" + | "owned-healthy" + | "owned-stale" + | "legacy-owned" + | "foreign"; + +export type LifecycleSessionDecision = + | { readonly kind: "create" } + | { readonly kind: "adopt"; readonly entry: LifecycleStateEntry } + | { + readonly kind: "replace"; + readonly entry: LifecycleStateEntry; + readonly classification: "owned-stale" | "legacy-owned"; + } + | { readonly kind: "block"; readonly reason: string }; + +export type LifecycleSessionInspection = { + readonly classification: LifecycleSessionClassification; + readonly decision: LifecycleSessionDecision; + readonly session: MuxSession | null; + readonly observedOwnershipToken: string | null; +}; + +export function createLifecycleOwnershipToken(): string { + return randomUUID(); +} + +export function resolveLifecycleDefinitionHash(opts: { + readonly definitions: readonly unknown[]; +}): string { + return new Bun.CryptoHasher("sha256") + .update(JSON.stringify(opts.definitions)) + .digest("hex"); +} + +export function classifyLifecycleSession(opts: { + readonly session: MuxSession | null; + readonly entry: LifecycleStateEntry | null; + readonly observedOwnershipToken: string | null; + readonly expectedBackend: MuxBackendName; + readonly expectedSessionName: string; + readonly expectedProjectRoot: string; + readonly expectedDefinitionHash: string; + readonly liveWindowNames: ReadonlySet | null; +}): LifecycleSessionInspection { + if (!opts.session) { + return { + classification: "absent", + decision: { kind: "create" }, + session: null, + observedOwnershipToken: null, + }; + } + + const entryMatchesSession = + opts.entry?.backend === opts.expectedBackend && + opts.entry.sessionName === opts.expectedSessionName; + if (!(entryMatchesSession && opts.entry)) { + return foreignInspection({ + session: opts.session, + observedOwnershipToken: opts.observedOwnershipToken, + reason: `Lifecycle session "${opts.expectedSessionName}" already exists without matching Hack lifecycle state. Refusing to replace it without ownership proof.`, + }); + } + + if (opts.entry.ownershipToken) { + if (opts.observedOwnershipToken !== opts.entry.ownershipToken) { + return foreignInspection({ + session: opts.session, + observedOwnershipToken: opts.observedOwnershipToken, + reason: `Lifecycle session "${opts.expectedSessionName}" ownership metadata does not match persisted Hack state. Refusing destructive recovery.`, + }); + } + + const definitionMatches = + opts.entry.definitionHash === opts.expectedDefinitionHash; + const windowsHealthy = + opts.entry.processes.length > 0 && + opts.liveWindowNames !== null && + opts.entry.processes.every((process) => + opts.liveWindowNames?.has(process.windowName) + ); + if (definitionMatches && windowsHealthy) { + return { + classification: "owned-healthy", + decision: { kind: "adopt", entry: opts.entry }, + session: opts.session, + observedOwnershipToken: opts.observedOwnershipToken, + }; + } + + return { + classification: "owned-stale", + decision: { + kind: "replace", + entry: opts.entry, + classification: "owned-stale", + }, + session: opts.session, + observedOwnershipToken: opts.observedOwnershipToken, + }; + } + + if ( + hasLegacyTmuxOwnershipProof({ + entry: opts.entry, + session: opts.session, + expectedProjectRoot: opts.expectedProjectRoot, + }) + ) { + return { + classification: "legacy-owned", + decision: { + kind: "replace", + entry: opts.entry, + classification: "legacy-owned", + }, + session: opts.session, + observedOwnershipToken: null, + }; + } + + return foreignInspection({ + session: opts.session, + observedOwnershipToken: null, + reason: `Lifecycle session "${opts.expectedSessionName}" predates ownership metadata and cannot be proven to belong to this project. Refusing destructive recovery.`, + }); +} + +export async function inspectLifecycleSession(opts: { + readonly backend: MuxBackend; + readonly entry: LifecycleStateEntry | null; + readonly expectedSessionName: string; + readonly expectedProjectRoot: string; + readonly expectedDefinitionHash: string; +}): Promise { + const sessions = await opts.backend.listSessions(); + const session = + sessions.find((candidate) => candidate.name === opts.expectedSessionName) ?? + null; + if (!session) { + return classifyLifecycleSession({ + session: null, + entry: opts.entry, + observedOwnershipToken: null, + expectedBackend: opts.backend.name, + expectedSessionName: opts.expectedSessionName, + expectedProjectRoot: opts.expectedProjectRoot, + expectedDefinitionHash: opts.expectedDefinitionHash, + liveWindowNames: null, + }); + } + + const [observedOwnershipToken, liveWindowNames] = await Promise.all([ + opts.backend.readLifecycleOwnerToken?.({ + name: opts.expectedSessionName, + }) ?? Promise.resolve(null), + opts.backend.listSessionWindowNames?.({ name: opts.expectedSessionName }) ?? + Promise.resolve(null), + ]); + return classifyLifecycleSession({ + session, + entry: opts.entry, + observedOwnershipToken, + expectedBackend: opts.backend.name, + expectedSessionName: opts.expectedSessionName, + expectedProjectRoot: await normalizePath(opts.expectedProjectRoot), + expectedDefinitionHash: opts.expectedDefinitionHash, + liveWindowNames, + }); +} + +export async function killLifecycleSessionWithOwnership(opts: { + readonly backend: MuxBackend; + readonly sessionName: string; + readonly ownershipToken: string; +}): Promise { + const observedToken = + (await opts.backend.readLifecycleOwnerToken?.({ + name: opts.sessionName, + })) ?? null; + if (observedToken !== opts.ownershipToken) { + return false; + } + const result = await opts.backend.killSession({ name: opts.sessionName }); + return result.exitCode === 0; +} + +export async function killInspectedLifecycleSession(opts: { + readonly backend: MuxBackend; + readonly inspection: LifecycleSessionInspection; +}): Promise { + if ( + opts.inspection.decision.kind === "create" || + opts.inspection.decision.kind === "block" + ) { + return false; + } + const entry = opts.inspection.decision.entry; + if (entry.ownershipToken) { + return await killLifecycleSessionWithOwnership({ + backend: opts.backend, + sessionName: entry.sessionName, + ownershipToken: entry.ownershipToken, + }); + } + const result = await opts.backend.killSession({ name: entry.sessionName }); + return result.exitCode === 0; +} + +function hasLegacyTmuxOwnershipProof(opts: { + readonly entry: LifecycleStateEntry; + readonly session: MuxSession; + readonly expectedProjectRoot: string; +}): boolean { + if ( + opts.entry.backend !== "tmux" || + opts.session.backend !== "tmux" || + !opts.session.path || + !opts.session.createdAt + ) { + return false; + } + const createdAt = Date.parse(opts.session.createdAt); + const stateUpdatedAt = Date.parse(opts.entry.updatedAt); + const enoughWindows = + opts.session.windows !== null && + opts.session.windows >= opts.entry.processes.length + 1; + return ( + opts.session.path === opts.expectedProjectRoot && + Number.isFinite(createdAt) && + Number.isFinite(stateUpdatedAt) && + createdAt <= stateUpdatedAt && + enoughWindows + ); +} + +function foreignInspection(opts: { + readonly session: MuxSession; + readonly observedOwnershipToken: string | null; + readonly reason: string; +}): LifecycleSessionInspection { + return { + classification: "foreign", + decision: { kind: "block", reason: opts.reason }, + session: opts.session, + observedOwnershipToken: opts.observedOwnershipToken, + }; +} + +async function normalizePath(path: string): Promise { + try { + return await realpath(path); + } catch { + return path; + } +} diff --git a/src/mux/mux-backend.ts b/src/mux/mux-backend.ts index dc42617f..447b49ef 100644 --- a/src/mux/mux-backend.ts +++ b/src/mux/mux-backend.ts @@ -24,9 +24,17 @@ export interface MuxBackend { readonly name: string; readonly cwd?: string; readonly env?: Readonly>; + readonly lifecycleOwnerToken?: string; }): Promise; killSession(opts: { readonly name: string }): Promise; + readLifecycleOwnerToken?(opts: { + readonly name: string; + }): Promise; + listSessionWindowNames?(opts: { + readonly name: string; + }): Promise | null>; + execInSession(opts: { readonly name: string; readonly command: string; diff --git a/src/mux/tmux-backend.ts b/src/mux/tmux-backend.ts index f2a95460..434fd3c7 100644 --- a/src/mux/tmux-backend.ts +++ b/src/mux/tmux-backend.ts @@ -7,6 +7,8 @@ import type { MuxSessionCreateResult, } from "./mux-backend.ts"; +const LIFECYCLE_OWNER_OPTION = "@hack_lifecycle_owner"; + function parseIntOrNull(value: string | undefined): number | null { const n = Number.parseInt(value ?? "", 10); return Number.isFinite(n) ? n : null; @@ -87,6 +89,7 @@ export function createTmuxBackend(): MuxBackend { readonly name: string; readonly cwd?: string; readonly env?: Readonly>; + readonly lifecycleOwnerToken?: string; }): Promise => { if (!available) { return { ok: false, error: "tmux_unavailable" }; @@ -113,6 +116,30 @@ export function createTmuxBackend(): MuxBackend { }; } + if (opts.lifecycleOwnerToken) { + const ownerResult = await exec( + [ + "tmux", + "set-option", + "-t", + opts.name, + LIFECYCLE_OWNER_OPTION, + opts.lifecycleOwnerToken, + ], + { stdin: "ignore" } + ); + if (ownerResult.exitCode !== 0) { + await exec(["tmux", "kill-session", "-t", opts.name], { + stdin: "ignore", + }); + return { + ok: false, + error: "owner_metadata_failed", + stderr: ownerResult.stderr.trim(), + }; + } + } + const sessions = await listSessions(); const session = sessions.find((s) => s.name === opts.name) ?? null; return { ok: true, session }; @@ -126,6 +153,38 @@ export function createTmuxBackend(): MuxBackend { }); }; + const readLifecycleOwnerToken = async (opts: { + readonly name: string; + }): Promise => { + const result = await exec( + ["tmux", "show-options", "-v", "-t", opts.name, LIFECYCLE_OWNER_OPTION], + { stdin: "ignore" } + ); + if (result.exitCode !== 0) { + return null; + } + const token = result.stdout.trim(); + return token.length > 0 ? token : null; + }; + + const listSessionWindowNames = async (opts: { + readonly name: string; + }): Promise | null> => { + const result = await exec( + ["tmux", "list-windows", "-t", opts.name, "-F", "#{window_name}"], + { stdin: "ignore" } + ); + if (result.exitCode !== 0) { + return null; + } + return new Set( + result.stdout + .split("\n") + .map((name) => name.trim()) + .filter((name) => name.length > 0) + ); + }; + const execInSession = async (opts: { readonly name: string; readonly command: string; @@ -164,6 +223,8 @@ export function createTmuxBackend(): MuxBackend { listSessions, createSession, killSession, + readLifecycleOwnerToken, + listSessionWindowNames, execInSession, sendInput, }; diff --git a/src/mux/zellij-backend.ts b/src/mux/zellij-backend.ts index d270b789..136d1429 100644 --- a/src/mux/zellij-backend.ts +++ b/src/mux/zellij-backend.ts @@ -7,6 +7,11 @@ import type { MuxSessionCreateResult, } from "./mux-backend.ts"; +const LIFECYCLE_OWNER_TAB_PREFIX = "hack-lifecycle-owner-"; +const LIFECYCLE_OWNER_TAB_PATTERN = + /tab name="hack-lifecycle-owner-([a-zA-Z0-9-]+)"/; +const NAMED_PANE_PATTERN = /pane name="([^"]+)"/g; + export function createZellijBackend(): MuxBackend { const available = Boolean(findExecutableInPath("zellij")); @@ -47,6 +52,7 @@ export function createZellijBackend(): MuxBackend { readonly name: string; readonly cwd?: string; readonly env?: Readonly>; + readonly lifecycleOwnerToken?: string; }): Promise => { if (!available) { return { ok: false, error: "zellij_unavailable" }; @@ -58,25 +64,38 @@ export function createZellijBackend(): MuxBackend { { stdin: "ignore", cwd: opts.cwd, - env: opts.name - ? { - ...(opts.env ?? {}), - ZELLIJ_SESSION_NAME: opts.name, - } - : opts.env, + env: opts.env, } ); if (result.exitCode !== 0) { const stderr = result.stderr.trim(); - if (stderr.toLowerCase().includes("session already exists")) { - const sessions = await listSessions(); - const session = sessions.find((s) => s.name === opts.name) ?? null; - return { ok: true, session }; - } return { ok: false, error: "create_failed", stderr }; } + if (opts.lifecycleOwnerToken) { + const ownerResult = await exec( + [ + "zellij", + "action", + "rename-tab", + `${LIFECYCLE_OWNER_TAB_PREFIX}${opts.lifecycleOwnerToken}`, + ], + { + stdin: "ignore", + env: { ZELLIJ_SESSION_NAME: opts.name }, + } + ); + if (ownerResult.exitCode !== 0) { + await exec(["zellij", "kill-session", opts.name], { stdin: "ignore" }); + return { + ok: false, + error: "owner_metadata_failed", + stderr: ownerResult.stderr.trim(), + }; + } + } + const sessions = await listSessions(); const session = sessions.find((s) => s.name === opts.name) ?? null; return { ok: true, session }; @@ -90,6 +109,37 @@ export function createZellijBackend(): MuxBackend { }); }; + const readLifecycleOwnerToken = async (opts: { + readonly name: string; + }): Promise => { + const result = await exec(["zellij", "action", "dump-layout"], { + stdin: "ignore", + env: { ZELLIJ_SESSION_NAME: opts.name }, + }); + if (result.exitCode !== 0) { + return null; + } + const match = result.stdout.match(LIFECYCLE_OWNER_TAB_PATTERN); + return match?.[1] ?? null; + }; + + const listSessionWindowNames = async (opts: { + readonly name: string; + }): Promise | null> => { + const result = await exec(["zellij", "action", "dump-layout"], { + stdin: "ignore", + env: { ZELLIJ_SESSION_NAME: opts.name }, + }); + if (result.exitCode !== 0) { + return null; + } + return new Set( + [...result.stdout.matchAll(NAMED_PANE_PATTERN)] + .map((match) => match[1]?.trim() ?? "") + .filter((name) => name.length > 0) + ); + }; + const execInSession = async (opts: { readonly name: string; readonly command: string; @@ -122,6 +172,8 @@ export function createZellijBackend(): MuxBackend { listSessions, createSession, killSession, + readLifecycleOwnerToken, + listSessionWindowNames, execInSession, sendInput, }; diff --git a/tests/e2e/fixture.ts b/tests/e2e/fixture.ts index e1e44964..ca1b8cd8 100644 --- a/tests/e2e/fixture.ts +++ b/tests/e2e/fixture.ts @@ -36,6 +36,14 @@ export type LifecycleFixtureOptions = { readonly upBeforeMarkerFile?: string; /** Adds a long-running lifecycle host process (bun sleep loop). */ readonly persistentProcess?: boolean; + /** Makes docker compose reject the fixture after lifecycle startup. */ + readonly composeFailure?: boolean; + /** Runs fixture containers without a host bind mount. */ + readonly standaloneContainers?: boolean; + /** Disables global DNS/TLS integration for local lifecycle-only scenarios. */ + readonly disableInternal?: boolean; + /** Adds a failing lifecycle.down.before hook. */ + readonly downBeforeFailure?: boolean; }; /** Random, collision-proof project slug (also used for dev_host). */ @@ -298,8 +306,8 @@ async function writeHackConfig(opts: { clear_on_down: false, }, internal: { - dns: true, - tls: true, + dns: opts.lifecycle?.disableInternal !== true, + tls: opts.lifecycle?.disableInternal !== true, }, oauth: { enabled: false, @@ -314,7 +322,12 @@ async function writeHackConfig(opts: { await Bun.write( join(opts.hackDir, "docker-compose.yml"), - renderComposeYaml({ name: opts.name, devHost: opts.devHost }) + renderComposeYaml({ + name: opts.name, + devHost: opts.devHost, + composeFailure: opts.lifecycle?.composeFailure, + standaloneContainers: opts.lifecycle?.standaloneContainers, + }) ); await Bun.write( @@ -339,6 +352,7 @@ function buildLifecycleConfig(opts: { } const upBefore: Record[] = []; const processes: Record[] = []; + const downBefore: Record[] = []; if (opts.lifecycle.upBeforeMarkerFile) { upBefore.push({ @@ -354,12 +368,19 @@ function buildLifecycleConfig(opts: { cwd: ".", }); } + if (opts.lifecycle.downBeforeFailure === true) { + downBefore.push({ + name: "e2e-down-failure", + command: "exit 23", + cwd: ".", + }); + } if (upBefore.length === 0 && processes.length === 0) { return null; } return { up: { before: upBefore, after: [] }, - down: { before: [], after: [] }, + down: { before: downBefore, after: [] }, ...(processes.length > 0 ? { processes } : {}), }; } @@ -367,16 +388,23 @@ function buildLifecycleConfig(opts: { function renderComposeYaml(opts: { readonly name: string; readonly devHost: string; + readonly composeFailure?: boolean; + readonly standaloneContainers?: boolean; }): string { return [ `name: ${opts.name}`, "services:", " web:", + ...(opts.composeFailure ? [" unsupported_e2e_key: true"] : []), " image: oven/bun:1", - " working_dir: /app", - " volumes:", - " - ..:/app", - " command: bun apps/web/index.ts", + ...(opts.standaloneContainers + ? [" command: bun -e 'setInterval(() => {}, 60000)'"] + : [ + " working_dir: /app", + " volumes:", + " - ..:/app", + " command: bun apps/web/index.ts", + ]), " environment:", ` PORT: "${WEB_PORT}"`, " labels:", @@ -388,10 +416,14 @@ function renderComposeYaml(opts: { " - default", " api:", " image: oven/bun:1", - " working_dir: /app", - " volumes:", - " - ..:/app", - " command: bun apps/api/index.ts", + ...(opts.standaloneContainers + ? [" command: bun -e 'setInterval(() => {}, 60000)'"] + : [ + " working_dir: /app", + " volumes:", + " - ..:/app", + " command: bun apps/api/index.ts", + ]), " environment:", ` PORT: "${API_PORT}"`, " labels:", diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts index a57494b2..c2e336fc 100644 --- a/tests/e2e/run.ts +++ b/tests/e2e/run.ts @@ -5,6 +5,7 @@ import { doctorScenario } from "./scenarios/doctor.ts"; import { envSecretsScenario } from "./scenarios/env-secrets.ts"; import { initScenario } from "./scenarios/init.ts"; import { lifecycleHostProcessScenario } from "./scenarios/lifecycle-host-process.ts"; +import { lifecycleSessionRecoveryScenario } from "./scenarios/lifecycle-session-recovery.ts"; import { upDownScenario } from "./scenarios/up-down.ts"; import { worktreeBranchDefaultScenario } from "./scenarios/worktree-branch-default.ts"; import { worktreeParallelUpScenario } from "./scenarios/worktree-parallel-up.ts"; @@ -34,6 +35,7 @@ const ALL_SCENARIOS: readonly Scenario[] = [ worktreeBranchDefaultScenario, agentDocsSyncScenario, doctorScenario, + lifecycleSessionRecoveryScenario, upDownScenario, lifecycleHostProcessScenario, worktreeParallelUpScenario, diff --git a/tests/e2e/scenarios/lifecycle-host-process.ts b/tests/e2e/scenarios/lifecycle-host-process.ts index 1ed279a0..c009527f 100644 --- a/tests/e2e/scenarios/lifecycle-host-process.ts +++ b/tests/e2e/scenarios/lifecycle-host-process.ts @@ -6,16 +6,6 @@ import { downBestEffort, requireDockerPreconditions } from "./docker-shared.ts"; const UP_TIMEOUT_MS = 420_000; -type LifecycleStateFile = { - readonly entries?: readonly { - readonly processes?: readonly { - readonly name?: string; - readonly panePid?: number; - readonly processGroupId?: number; - }[]; - }[]; -}; - /** * Lifecycle host hooks run as part of `hack up` (there is no standalone * lifecycle start/stop command today), so this scenario lives in the docker @@ -38,6 +28,7 @@ export const lifecycleHostProcessScenario: Scenario = { lifecycle: { persistentProcess: true, upBeforeMarkerFile: markerFile, + standaloneContainers: true, }, }); @@ -65,16 +56,20 @@ export const lifecycleHostProcessScenario: Scenario = { result: up, }); - const stateFile = Bun.file( - join(fixture.hackDir, ".internal", "lifecycle", "state.json") + const statePath = join( + fixture.hackDir, + ".internal", + "lifecycle", + "state.json" ); + const firstState = await readLifecycleState({ path: statePath }); + const firstEntry = firstState.entries[0]; expect({ - that: await stateFile.exists(), - message: "persistent lifecycle process did not write lifecycle state", + that: Boolean(firstEntry?.ownershipToken), + message: "lifecycle state should persist an ownership token", result: up, }); - const state = (await stateFile.json()) as LifecycleStateFile; - const sleeper = state.entries?.[0]?.processes?.find( + const sleeper = firstEntry?.processes?.find( (process) => process.name === "e2e-sleeper" ); const panePid = sleeper?.panePid ?? 0; @@ -107,7 +102,112 @@ export const lifecycleHostProcessScenario: Scenario = { result: groupLeader, }); - await downBestEffort({ ctx, fixture }); + const owner = await runCommand({ + argv: [ + "tmux", + "show-options", + "-v", + "-t", + `${fixture.name}--lifecycle`, + "@hack_lifecycle_owner", + ], + cwd: fixture.root, + }); + expectExit({ + result: owner, + codes: [0], + message: "tmux lifecycle session should expose ownership metadata", + }); + expect({ + that: owner.stdout.trim() === firstEntry?.ownershipToken, + message: "tmux and lifecycle state ownership tokens should match", + result: owner, + }); + + const repeatedUp = await ctx.cli({ + args: ["up", "--detach"], + cwd: fixture.root, + timeoutMs: UP_TIMEOUT_MS, + }); + expectExit({ + result: repeatedUp, + codes: [0], + message: "hack up should adopt its healthy lifecycle session", + }); + const adoptedState = await readLifecycleState({ path: statePath }); + expect({ + that: + adoptedState.entries[0]?.ownershipToken === + firstEntry?.ownershipToken, + message: "healthy lifecycle adoption should preserve ownership token", + result: repeatedUp, + }); + + const legacyState = { + entries: adoptedState.entries.map((entry) => { + const { + definitionHash: _definitionHash, + ownershipToken: _token, + ...rest + } = entry; + return rest; + }), + }; + await Bun.write(statePath, `${JSON.stringify(legacyState, null, 2)}\n`); + const clearOwner = await runCommand({ + argv: [ + "tmux", + "set-option", + "-u", + "-t", + `${fixture.name}--lifecycle`, + "@hack_lifecycle_owner", + ], + cwd: fixture.root, + }); + expectExit({ + result: clearOwner, + codes: [0], + message: "failed to simulate a pre-ownership lifecycle session", + }); + + const recoveredUp = await ctx.cli({ + args: ["up", "--detach"], + cwd: fixture.root, + timeoutMs: UP_TIMEOUT_MS, + }); + expectExit({ + result: recoveredUp, + codes: [0], + message: + "hack up should replace a legacy session with strong tmux proof", + }); + const recoveredState = await readLifecycleState({ path: statePath }); + expect({ + that: Boolean(recoveredState.entries[0]?.ownershipToken), + message: "legacy recovery should write current ownership metadata", + result: recoveredUp, + }); + + const down = await ctx.cli({ + args: ["down"], + cwd: fixture.root, + timeoutMs: UP_TIMEOUT_MS, + }); + expectExit({ + result: down, + codes: [0], + message: "hack down should clean the owned lifecycle session", + }); + const sessionAfterDown = await runCommand({ + argv: ["tmux", "has-session", "-t", `${fixture.name}--lifecycle`], + cwd: fixture.root, + }); + expect({ + that: sessionAfterDown.exitCode !== 0, + message: "lifecycle session survived hack down", + result: sessionAfterDown, + }); let commandGroupStillExists = true; try { process.kill(-processGroupId, 0); @@ -117,10 +217,80 @@ export const lifecycleHostProcessScenario: Scenario = { expect({ that: !commandGroupStillExists, message: "hack down left the lifecycle command process group alive", - result: up, + result: down, + }); + + const failingFixture = await createMonorepoFixture({ + parentDir: ctx.tempRoot, + withHackConfig: true, + lifecycle: { + persistentProcess: true, + composeFailure: true, + standaloneContainers: true, + }, + }); + const failedUp = await ctx.cli({ + args: ["up", "--detach", "--json"], + cwd: failingFixture.root, + timeoutMs: UP_TIMEOUT_MS, + }); + expect({ + that: failedUp.exitCode !== 0, + message: "invalid compose fixture should fail after lifecycle startup", + result: failedUp, + }); + const failedSession = await runCommand({ + argv: [ + "tmux", + "has-session", + "-t", + `${failingFixture.name}--lifecycle`, + ], + cwd: failingFixture.root, + }); + expect({ + that: failedSession.exitCode !== 0, + message: "compose failure left its lifecycle session behind", + result: failedSession, + }); + const failedState = await readLifecycleState({ + path: join( + failingFixture.hackDir, + ".internal", + "lifecycle", + "state.json" + ), + }); + expect({ + that: failedState.entries.length === 0, + message: "compose failure left lifecycle ownership state behind", + result: failedUp, }); } finally { await downBestEffort({ ctx, fixture }); } }, }; + +type LifecycleStateFixture = { + readonly entries: ReadonlyArray<{ + readonly ownershipToken?: string; + readonly definitionHash?: string; + readonly processes?: ReadonlyArray<{ + readonly name?: string; + readonly panePid?: number; + readonly processGroupId?: number; + }>; + readonly [key: string]: unknown; + }>; +}; + +async function readLifecycleState(opts: { + readonly path: string; +}): Promise { + const file = Bun.file(opts.path); + if (!(await file.exists())) { + return { entries: [] }; + } + return (await file.json()) as LifecycleStateFixture; +} diff --git a/tests/e2e/scenarios/lifecycle-session-recovery.ts b/tests/e2e/scenarios/lifecycle-session-recovery.ts new file mode 100644 index 00000000..2d3e46d7 --- /dev/null +++ b/tests/e2e/scenarios/lifecycle-session-recovery.ts @@ -0,0 +1,449 @@ +import { chmod, mkdir } from "node:fs/promises"; +import { join } from "node:path"; + +import { createMonorepoFixture } from "../fixture.ts"; +import { + buildCliEnv, + CLI_ENTRYPOINT, + expect, + expectExit, + runCommand, + type Scenario, +} from "../harness.ts"; + +const SESSION_WAIT_TIMEOUT_MS = 10_000; + +export const lifecycleSessionRecoveryScenario: Scenario = { + name: "lifecycle-session-recovery", + tier: "local", + summary: "owned lifecycle sessions reconcile and clean up without Docker", + run: async (ctx) => { + const fixture = await createMonorepoFixture({ + parentDir: ctx.tempRoot, + withHackConfig: true, + lifecycle: { + persistentProcess: true, + disableInternal: true, + }, + }); + const fakeBin = join(ctx.tempRoot, "fake-bin"); + await writeFakeDocker({ fakeBin }); + const path = `${fakeBin}:${process.env.PATH ?? ""}`; + const dockerLog = join(ctx.tempRoot, "fake-docker.log"); + const cliEnv = { PATH: path, E2E_DOCKER_LOG: dockerLog }; + const sessionName = `${fixture.name}--lifecycle`; + const statePath = join( + fixture.hackDir, + ".internal", + "lifecycle", + "state.json" + ); + + try { + const firstUp = await ctx.cli({ + args: ["up", "--detach", "--json"], + cwd: fixture.root, + env: cliEnv, + }); + expectExit({ + result: firstUp, + codes: [0], + message: "local lifecycle fixture should start", + }); + const firstToken = (await readState({ statePath })).entries[0] + ?.ownershipToken; + expect({ + that: Boolean(firstToken), + message: "lifecycle startup did not persist ownership metadata", + result: firstUp, + }); + + const repeatedUp = await ctx.cli({ + args: ["up", "--detach", "--json"], + cwd: fixture.root, + env: cliEnv, + }); + expectExit({ + result: repeatedUp, + codes: [0], + message: "a repeated up should adopt the healthy owned session", + }); + expect({ + that: + (await readState({ statePath })).entries[0]?.ownershipToken === + firstToken, + message: "healthy adoption replaced the owner token", + result: repeatedUp, + }); + + const branchName = "feat-msp-human-handoff"; + const branchUp = await ctx.cli({ + args: ["up", "--detach", "--json", "--branch", branchName], + cwd: fixture.root, + env: cliEnv, + }); + expectExit({ + result: branchUp, + codes: [0], + message: "explicit branch fixture should start", + }); + const branchComposeProject = `${fixture.name}--${branchName}`; + const branchToken = (await readState({ statePath })).entries.find( + (entry) => entry.composeProject === branchComposeProject + )?.ownershipToken; + expect({ + that: Boolean(branchToken), + message: "branch lifecycle entry should have an ownership token", + result: branchUp, + }); + + await Bun.write(dockerLog, ""); + const baseRestart = await ctx.cli({ + args: ["restart", "--json"], + cwd: fixture.root, + env: cliEnv, + }); + expectExit({ + result: baseRestart, + codes: [0], + message: "primary checkout restart should succeed", + }); + const restartCalls = (await Bun.file(dockerLog).text()) + .split("\n") + .filter((line) => line.length > 0); + expect({ + that: + restartCalls.length >= 3 && + restartCalls.every( + (line) => + !( + line.includes(`-p ${branchComposeProject}`) || + line.includes(branchName) + ) + ), + message: `primary restart routed to a branch compose project:\n${restartCalls.join("\n")}`, + result: baseRestart, + }); + const branchSession = await runCommand({ + argv: [ + "tmux", + "has-session", + "-t", + `${fixture.name}--lifecycle-${branchName}`, + ], + cwd: fixture.root, + }); + expectExit({ + result: branchSession, + codes: [0], + message: "primary restart stopped the running branch lifecycle session", + }); + expect({ + that: + (await readState({ statePath })).entries.find( + (entry) => entry.composeProject === branchComposeProject + )?.ownershipToken === branchToken, + message: "primary restart replaced the branch lifecycle owner", + result: baseRestart, + }); + + const down = await ctx.cli({ + args: ["down", "--json"], + cwd: fixture.root, + env: cliEnv, + }); + expectExit({ + result: down, + codes: [0], + message: "down should stop the owned lifecycle session", + }); + await expectSessionAbsent({ + sessionName, + cwd: fixture.root, + phase: "base down", + }); + const branchDown = await ctx.cli({ + args: ["down", "--json", "--branch", branchName], + cwd: fixture.root, + env: cliEnv, + }); + expectExit({ + result: branchDown, + codes: [0], + message: "explicit branch down should clean its lifecycle session", + }); + + const foreignSession = await runCommand({ + argv: [ + "tmux", + "new-session", + "-d", + "-s", + sessionName, + "-c", + fixture.root, + ], + cwd: fixture.root, + }); + expectExit({ + result: foreignSession, + codes: [0], + message: "failed to create isolated foreign collision session", + }); + const collidedUp = await ctx.cli({ + args: ["up", "--detach", "--json"], + cwd: fixture.root, + env: cliEnv, + }); + expect({ + that: + collidedUp.exitCode !== 0 && + collidedUp.combined.includes("ownership proof"), + message: "same-name foreign session should block destructive recovery", + result: collidedUp, + }); + const foreignStillPresent = await runCommand({ + argv: ["tmux", "has-session", "-t", sessionName], + cwd: fixture.root, + }); + expectExit({ + result: foreignStillPresent, + codes: [0], + message: "foreign collision session was destroyed", + }); + await runCommand({ + argv: ["tmux", "kill-session", "-t", sessionName], + cwd: fixture.root, + }); + + const failedUp = await ctx.cli({ + args: ["up", "--detach", "--json"], + cwd: fixture.root, + env: { ...cliEnv, E2E_DOCKER_FAIL_UP: "1" }, + }); + expect({ + that: failedUp.exitCode !== 0, + message: "fake compose failure should propagate", + result: failedUp, + }); + await expectSessionAbsent({ + sessionName, + cwd: fixture.root, + phase: "compose failure", + }); + expect({ + that: (await readState({ statePath })).entries.length === 0, + message: "compose failure left lifecycle state behind", + result: failedUp, + }); + + const signalResult = await runSignalProbe({ + cwd: fixture.root, + hackHome: ctx.hackHome, + path, + sessionName, + }); + expect({ + that: signalResult.exitCode === 143, + message: `signaled up should exit 143, got ${signalResult.exitCode}\n${signalResult.stderr}`, + }); + await expectSessionAbsent({ + sessionName, + cwd: fixture.root, + phase: "SIGTERM", + }); + expect({ + that: (await readState({ statePath })).entries.length === 0, + message: "SIGTERM left lifecycle state behind", + }); + + const downFailureFixture = await createMonorepoFixture({ + parentDir: ctx.tempRoot, + withHackConfig: true, + lifecycle: { + persistentProcess: true, + disableInternal: true, + downBeforeFailure: true, + }, + }); + const downFailureSession = `${downFailureFixture.name}--lifecycle`; + try { + const downFailureUp = await ctx.cli({ + args: ["up", "--detach", "--json"], + cwd: downFailureFixture.root, + env: cliEnv, + }); + expectExit({ + result: downFailureUp, + codes: [0], + message: "down-failure fixture should start", + }); + const failedDown = await ctx.cli({ + args: ["down", "--json"], + cwd: downFailureFixture.root, + env: cliEnv, + }); + expect({ + that: + failedDown.exitCode !== 0 && + failedDown.combined.includes("E_LIFECYCLE_FAILED"), + message: "down.before failure should propagate", + result: failedDown, + }); + await expectSessionAbsent({ + sessionName: downFailureSession, + cwd: downFailureFixture.root, + phase: "down.before failure", + }); + } finally { + await runCommand({ + argv: ["tmux", "kill-session", "-t", downFailureSession], + cwd: downFailureFixture.root, + }); + } + } finally { + await runCommand({ + argv: ["tmux", "kill-session", "-t", sessionName], + cwd: fixture.root, + }); + } + }, +}; + +async function writeFakeDocker(opts: { + readonly fakeBin: string; +}): Promise { + await mkdir(opts.fakeBin, { recursive: true }); + const path = join(opts.fakeBin, "docker"); + await Bun.write( + path, + [ + "#!/bin/sh", + 'if [ -n "${E2E_DOCKER_LOG:-}" ]; then printf "%s\\n" "$*" >> "$E2E_DOCKER_LOG"; fi', + 'if [ "$1" = "compose" ]; then', + ' case " $* " in', + ' *" up "*)', + ' if [ "${E2E_DOCKER_BLOCK_UP:-}" = "1" ]; then', + ' parent_pid="$PPID"', + ' while kill -0 "$parent_pid" 2>/dev/null; do sleep 0.1; done', + " exit 143", + " fi", + ' if [ "${E2E_DOCKER_FAIL_UP:-}" = "1" ]; then exit 42; fi', + " exit 0", + " ;;", + " *) exit 0 ;;", + " esac", + "fi", + 'if [ "$1" = "info" ]; then exit 0; fi', + "exit 1", + "", + ].join("\n") + ); + await chmod(path, 0o755); +} + +async function runSignalProbe(opts: { + readonly cwd: string; + readonly hackHome: string; + readonly path: string; + readonly sessionName: string; +}): Promise<{ readonly exitCode: number; readonly stderr: string }> { + const proc = Bun.spawn(["bun", CLI_ENTRYPOINT, "up", "--detach", "--json"], { + cwd: opts.cwd, + env: buildCliEnv({ + hackHome: opts.hackHome, + extra: { + PATH: opts.path, + E2E_DOCKER_BLOCK_UP: "1", + }, + }), + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + + const sessionReady = await waitForSession({ + sessionName: opts.sessionName, + cwd: opts.cwd, + }); + if (!sessionReady) { + proc.kill("SIGKILL"); + await proc.exited; + throw new Error("timed out waiting for lifecycle session before SIGTERM"); + } + proc.kill("SIGTERM"); + const [exitCode, stderr] = await Promise.all([ + proc.exited, + new Response(proc.stderr).text(), + ]); + return { exitCode, stderr }; +} + +async function waitForSession(opts: { + readonly sessionName: string; + readonly cwd: string; +}): Promise { + const deadline = Date.now() + SESSION_WAIT_TIMEOUT_MS; + while (Date.now() < deadline) { + const presence = await inspectSessionPresence(opts); + if (presence.present) { + return true; + } + await Bun.sleep(100); + } + return false; +} + +async function expectSessionAbsent(opts: { + readonly sessionName: string; + readonly cwd: string; + readonly phase: string; +}): Promise { + const deadline = Date.now() + SESSION_WAIT_TIMEOUT_MS; + let presence = await inspectSessionPresence(opts); + while (presence.present && Date.now() < deadline) { + await Bun.sleep(100); + presence = await inspectSessionPresence(opts); + } + expect({ + that: !presence.present, + message: `${opts.phase}: lifecycle session ${opts.sessionName} is still present`, + result: presence.result, + }); +} + +async function inspectSessionPresence(opts: { + readonly sessionName: string; + readonly cwd: string; +}): Promise<{ + readonly present: boolean; + readonly result: Awaited>; +}> { + const result = await runCommand({ + argv: ["tmux", "list-sessions", "-F", "#{session_name}"], + cwd: opts.cwd, + }); + return { + present: result.stdout + .split("\n") + .some((name) => name.trim() === opts.sessionName), + result, + }; +} + +type LifecycleState = { + readonly entries: ReadonlyArray<{ + readonly composeProject: string; + readonly ownershipToken?: string; + }>; +}; + +async function readState(opts: { + readonly statePath: string; +}): Promise { + const file = Bun.file(opts.statePath); + if (!(await file.exists())) { + return { entries: [] }; + } + return (await file.json()) as LifecycleState; +} diff --git a/tests/project-lifecycle-sessions.test.ts b/tests/project-lifecycle-sessions.test.ts new file mode 100644 index 00000000..a31b8ff1 --- /dev/null +++ b/tests/project-lifecycle-sessions.test.ts @@ -0,0 +1,155 @@ +import { expect, test } from "bun:test"; + +import type { LifecycleStateEntry } from "../src/lib/lifecycle-runtime.ts"; +import { + classifyLifecycleSession, + killLifecycleSessionWithOwnership, + resolveLifecycleDefinitionHash, +} from "../src/lib/project-lifecycle-sessions.ts"; +import type { MuxBackend, MuxSession } from "../src/mux/mux-backend.ts"; + +const ownershipToken = ["test", "lifecycle", "owner"].join("-"); +const definitionHash = resolveLifecycleDefinitionHash({ + definitions: [{ name: "proxy", command: "bun proxy.ts" }], +}); +const session: MuxSession = { + backend: "tmux", + name: "event-agent--lifecycle", + attached: false, + path: "/tmp/event-agent", + windows: 2, + createdAt: "2026-07-09T12:00:00.000Z", +}; +const entry: LifecycleStateEntry = { + composeProject: "event-agent", + projectName: "event-agent", + branch: null, + sessionName: session.name, + backend: "tmux", + ownershipToken, + definitionHash, + updatedAt: "2026-07-09T12:00:01.000Z", + processes: [ + { + name: "proxy", + windowName: "proxy", + logPath: "/tmp/event-agent.log", + }, + ], +}; + +test("classifyLifecycleSession creates when the expected session is absent", () => { + const inspection = classifyLifecycleSession({ + session: null, + entry, + observedOwnershipToken: null, + expectedBackend: "tmux", + expectedSessionName: session.name, + expectedProjectRoot: "/tmp/event-agent", + expectedDefinitionHash: definitionHash, + liveWindowNames: null, + }); + + expect(inspection.classification).toBe("absent"); + expect(inspection.decision).toEqual({ kind: "create" }); +}); + +test("classifyLifecycleSession adopts a token-owned healthy session", () => { + const inspection = classifyLifecycleSession({ + session, + entry, + observedOwnershipToken: ownershipToken, + expectedBackend: "tmux", + expectedSessionName: session.name, + expectedProjectRoot: "/tmp/event-agent", + expectedDefinitionHash: definitionHash, + liveWindowNames: new Set(["shell", "proxy"]), + }); + + expect(inspection.classification).toBe("owned-healthy"); + expect(inspection.decision.kind).toBe("adopt"); +}); + +test("classifyLifecycleSession replaces a token-owned stale session", () => { + const inspection = classifyLifecycleSession({ + session, + entry, + observedOwnershipToken: ownershipToken, + expectedBackend: "tmux", + expectedSessionName: session.name, + expectedProjectRoot: "/tmp/event-agent", + expectedDefinitionHash: definitionHash, + liveWindowNames: new Set(["shell"]), + }); + + expect(inspection.classification).toBe("owned-stale"); + expect(inspection.decision).toMatchObject({ kind: "replace" }); +}); + +test("classifyLifecycleSession replaces a legacy tmux session only with path and timestamp proof", () => { + const legacyEntry = { ...entry, ownershipToken: undefined }; + const inspection = classifyLifecycleSession({ + session, + entry: legacyEntry, + observedOwnershipToken: null, + expectedBackend: "tmux", + expectedSessionName: session.name, + expectedProjectRoot: "/tmp/event-agent", + expectedDefinitionHash: definitionHash, + liveWindowNames: new Set(["shell", "proxy"]), + }); + + expect(inspection.classification).toBe("legacy-owned"); + expect(inspection.decision).toMatchObject({ kind: "replace" }); +}); + +test("classifyLifecycleSession blocks same-name sessions without ownership proof", () => { + const inspection = classifyLifecycleSession({ + session: { ...session, path: "/tmp/unrelated" }, + entry: { ...entry, ownershipToken: undefined }, + observedOwnershipToken: null, + expectedBackend: "tmux", + expectedSessionName: session.name, + expectedProjectRoot: "/tmp/event-agent", + expectedDefinitionHash: definitionHash, + liveWindowNames: new Set(["shell", "proxy"]), + }); + + expect(inspection.classification).toBe("foreign"); + expect(inspection.decision).toMatchObject({ kind: "block" }); +}); + +test("killLifecycleSessionWithOwnership cleans up only an exact token match", async () => { + const killed: string[] = []; + let observedToken: string | null = ownershipToken; + const backend: MuxBackend = { + name: "tmux", + available: true, + listSessions: async () => [session], + createSession: async () => ({ ok: true, session }), + killSession: async ({ name }) => { + killed.push(name); + return { exitCode: 0, stdout: "", stderr: "" }; + }, + readLifecycleOwnerToken: async () => observedToken, + execInSession: async () => ({ exitCode: 0, stdout: "", stderr: "" }), + sendInput: async () => ({ exitCode: 0, stdout: "", stderr: "" }), + }; + + expect( + await killLifecycleSessionWithOwnership({ + backend, + sessionName: session.name, + ownershipToken, + }) + ).toBe(true); + observedToken = "different-token"; + expect( + await killLifecycleSessionWithOwnership({ + backend, + sessionName: session.name, + ownershipToken, + }) + ).toBe(false); + expect(killed).toEqual([session.name]); +}); diff --git a/tests/project-restart-command.test.ts b/tests/project-restart-command.test.ts index 50d2566a..b66e0b17 100644 --- a/tests/project-restart-command.test.ts +++ b/tests/project-restart-command.test.ts @@ -3,7 +3,10 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import { resolveStoredRuntimeEnvName } from "../src/commands/project.ts"; +import { + resolveRestartTarget, + resolveStoredRuntimeEnvName, +} from "../src/commands/project.ts"; const tempDirs = new Set(); @@ -59,6 +62,30 @@ test("restart preserves an explicit base env selection over stored runtime state expect(resolved).toBeNull(); }); +test("restart from the primary checkout targets only the base compose instance", () => { + expect( + resolveRestartTarget({ + baseProjectName: "event-agent", + branch: null, + }) + ).toEqual({ + composeProjectName: null, + lifecycleComposeProject: "event-agent", + }); +}); + +test("restart with an explicit branch targets only that branch instance", () => { + expect( + resolveRestartTarget({ + baseProjectName: "event-agent", + branch: "feat-msp-human-handoff", + }) + ).toEqual({ + composeProjectName: "event-agent--feat-msp-human-handoff", + lifecycleComposeProject: "event-agent--feat-msp-human-handoff", + }); +}); + async function createProjectDirWithRuntimeState(opts: { readonly composeProject: string; readonly envName: string | null; From d008a98be3d8546adbb51266631aa002b8d03e2e Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 9 Jul 2026 17:08:11 -0400 Subject: [PATCH 2/7] fix(doctor): reap owned orphan lifecycle sessions --- .codex/skills/hack-cli/SKILL.md | 2 + .cursor/rules/hack.mdc | 2 + AGENTS.md | 2 + CLAUDE.md | 2 + docs/architecture.md | 7 +- docs/lifecycle.md | 10 +- src/agents/instruction-source.ts | 1 + src/commands/doctor.ts | 66 ++++- src/commands/project.ts | 35 +-- src/lib/lifecycle-runtime.ts | 28 ++ src/lib/project-lifecycle-hygiene.ts | 279 +++++++++++++++++- src/lib/project-lifecycle-processes.ts | 35 +++ tests/doctor-command.test.ts | 23 +- tests/e2e/harness.ts | 16 +- tests/e2e/run.ts | 1 + .../scenarios/lifecycle-session-recovery.ts | 98 +++++- tests/e2e/scenarios/worktree-parallel-up.ts | 72 +++++ tests/project-lifecycle-hygiene.test.ts | 123 +++++++- 18 files changed, 741 insertions(+), 61 deletions(-) diff --git a/.codex/skills/hack-cli/SKILL.md b/.codex/skills/hack-cli/SKILL.md index acae552d..827b6eb4 100644 --- a/.codex/skills/hack-cli/SKILL.md +++ b/.codex/skills/hack-cli/SKILL.md @@ -113,6 +113,8 @@ Use `hack` as the primary interface for local-first development. - For fixed-port host helpers such as SSM tunnels or local proxies, set `singleton.ports` and usually `onConflict: "adopt"` so Hack reuses a healthy existing listener instead of starting duplicate tunnel stacks. - `singleton` is a listener guard, not process ownership transfer; adopted external processes are left running on `hack down`. - Inspect lifecycle status via `hack projects --details` and stream via `hack logs `. +- Lifecycle session recovery is ownership-proven: Hack adopts healthy token-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof. +- `hack doctor --fix` reaps an orphan lifecycle session only when mux ownership is proven and its Compose instance is absent; unverified same-name sessions are never modified. ## Workspaces (mux-managed, tmux-first by default) diff --git a/.cursor/rules/hack.mdc b/.cursor/rules/hack.mdc index 23a2377b..b0ee93e9 100644 --- a/.cursor/rules/hack.mdc +++ b/.cursor/rules/hack.mdc @@ -61,6 +61,8 @@ Prefer `hack` when shell access is available. Use MCP only when shell access is - For fixed-port host helpers such as SSM tunnels or local proxies, set `singleton.ports` and usually `onConflict: "adopt"` so Hack reuses a healthy existing listener instead of starting duplicate tunnel stacks. - `singleton` is a listener guard, not process ownership transfer; adopted external processes are left running on `hack down`. - Inspect lifecycle status via `hack projects --details` and stream via `hack logs `. +- Lifecycle session recovery is ownership-proven: Hack adopts healthy token-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof. +- `hack doctor --fix` reaps an orphan lifecycle session only when mux ownership is proven and its Compose instance is absent; unverified same-name sessions are never modified. ## Host-side env helpers diff --git a/AGENTS.md b/AGENTS.md index 1cbb89c4..cf6ee07e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -299,6 +299,8 @@ Lifecycle + startup: - For fixed-port host helpers such as SSM tunnels or local proxies, set `singleton.ports` and usually `onConflict: "adopt"` so Hack reuses a healthy existing listener instead of starting duplicate tunnel stacks. - `singleton` is a listener guard, not process ownership transfer; adopted external processes are left running on `hack down`. - Inspect lifecycle status via `hack projects --details` and stream via `hack logs `. +- Lifecycle session recovery is ownership-proven: Hack adopts healthy token-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof. +- `hack doctor --fix` reaps an orphan lifecycle session only when mux ownership is proven and its Compose instance is absent; unverified same-name sessions are never modified. Workspaces (mux-managed, tmux-first by default): - Picker: `hack session` for persistent project workspaces. diff --git a/CLAUDE.md b/CLAUDE.md index 9e77597a..7659bd17 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -169,6 +169,8 @@ Lifecycle + startup: - For fixed-port host helpers such as SSM tunnels or local proxies, set `singleton.ports` and usually `onConflict: "adopt"` so Hack reuses a healthy existing listener instead of starting duplicate tunnel stacks. - `singleton` is a listener guard, not process ownership transfer; adopted external processes are left running on `hack down`. - Inspect lifecycle status via `hack projects --details` and stream via `hack logs `. +- Lifecycle session recovery is ownership-proven: Hack adopts healthy token-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof. +- `hack doctor --fix` reaps an orphan lifecycle session only when mux ownership is proven and its Compose instance is absent; unverified same-name sessions are never modified. Workspaces (mux-managed, tmux-first by default): - Picker: `hack session` for persistent project workspaces. diff --git a/docs/architecture.md b/docs/architecture.md index df76fad3..435b16d1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -125,13 +125,18 @@ stable home and can be torn down on `hack down`. Lifecycle state is checkout-local at `.hack/.internal/lifecycle/state.json`. For tmux processes, the saved process group belongs to the wrapped user command rather than its mux pane shell. Each compose instance has its own entry, so starting a branch instance does not replace the base instance's entry -in the same checkout. Cleanup reconciles the saved process group with the live process table; if the group +in the same checkout. A random token mirrored in mux metadata and lifecycle state proves current +session ownership; deterministic session names never authorize teardown. Cleanup reconciles the saved process group with the live process table; if the group leader exited while members remain, `hack doctor` reports the orphan. `hack down` terminates the persisted leaderless group and descendant process groups only while a matching lifecycle mux session still proves ownership; without that session, cleanup stays non-destructive because the numeric PGID may have been reused. A live group leader without its saved pane PID is likewise not trusted as lifecycle ownership. +Doctor also compares ownership-proven lifecycle sessions with live Compose instances. `doctor --fix` +can remove a proven session only when its corresponding instance is absent and a second liveness and +ownership check still agrees. Foreign or ambiguous same-name sessions remain non-destructive findings. + For fixed-port helpers such as SSM/database/search tunnels, lifecycle config can also declare a `singleton` listener set. This lets Hack reuse an already-running equivalent helper or fail fast on partial conflicts instead of launching a competing duplicate supervisor. The intent is to reduce local diff --git a/docs/lifecycle.md b/docs/lifecycle.md index a3b66800..249a368b 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -228,12 +228,16 @@ Lifecycle session name: Notes: - If no mux backend is available, lifecycle process startup fails with an actionable error. - Teardown is implemented by killing the lifecycle session; anything running inside that session will be stopped. -- For tmux-backed lifecycle sessions, Hack also persists the pane PID and the wrapped command's actual - process-group metadata to `.hack/.internal/lifecycle/state.json`, with per-hook/process output logged to - Current lifecycle sessions carry the same random ownership token in mux metadata and persisted state. Cleanup requires an exact token match; deterministic names alone never authorize session teardown. +- For tmux-backed lifecycle sessions, Hack also persists the pane PID and the wrapped command's actual + process-group metadata to `.hack/.internal/lifecycle/state.json`, with per-hook/process output logged to `.hack/.internal/lifecycle/*.log`. If tmux pane state disappears before teardown, `hack down` still uses that persisted metadata to clean up any live lifecycle process groups instead of leaving orphaned host processes behind. -- `hack doctor` reports stale lifecycle state when the persisted lifecycle entry no longer has a live mux session and points operators to `hack down` so cleanup and state removal happen through the supported path. `hack doctor --fix` does not tear down lifecycle sessions itself — use `hack down` for that. +- `hack doctor` reports stale lifecycle state, leaderless process groups, ownership collisions, and + ownership-proven sessions with no running Compose instance. Recently updated sessions are treated + as possible in-flight startups for five minutes and stay untouched. `hack doctor --fix` reaps only + established orphans after rechecking runtime liveness and mux ownership; unverified same-name + sessions stay untouched. ## Tips diff --git a/src/agents/instruction-source.ts b/src/agents/instruction-source.ts index 260741d7..3a830c73 100644 --- a/src/agents/instruction-source.ts +++ b/src/agents/instruction-source.ts @@ -186,6 +186,7 @@ export const INSTRUCTION_SECTIONS: readonly InstructionSection[] = [ "`singleton` is a listener guard, not process ownership transfer; adopted external processes are left running on `hack down`.", "Inspect lifecycle status via `hack projects --details` and stream via `hack logs `.", "Lifecycle session recovery is ownership-proven: Hack adopts healthy token-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof.", + "`hack doctor --fix` reaps an orphan lifecycle session only when mux ownership is proven and its Compose instance is absent; unverified same-name sessions are never modified.", ], }, { diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 8ef9c566..f3500fd3 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -90,7 +90,10 @@ import { repairLegacyComposeEnvFileReferences, resolveProjectEnvConfig, } from "../lib/project-env-config.ts"; -import { inspectProjectLifecycleHygiene } from "../lib/project-lifecycle-hygiene.ts"; +import { + inspectProjectLifecycleHygiene, + repairProjectLifecycleSessions, +} from "../lib/project-lifecycle-hygiene.ts"; import { findMissingRegistryEntries, findOrphanRuntimeProjects, @@ -1844,10 +1847,13 @@ async function checkProjectLifecycleHygiene({ const inspection = await inspectProjectLifecycleHygiene({ projectDir: ctx.projectDir, + projectRoot: ctx.projectRoot, }); if ( inspection.staleEntries.length === 0 && - inspection.orphanedProcessGroups.length === 0 + inspection.orphanedProcessGroups.length === 0 && + inspection.orphanedSessions.length === 0 && + inspection.unverifiedSessions.length === 0 ) { return { name: "lifecycle hygiene", @@ -1875,12 +1881,29 @@ async function checkProjectLifecycleHygiene({ `${orphanedGroups.size} orphaned lifecycle process group${orphanedGroups.size === 1 ? "" : "s"}`, ] : []), + ...(inspection.orphanedSessions.length > 0 + ? [ + `${inspection.orphanedSessions.length} owned lifecycle session${inspection.orphanedSessions.length === 1 ? "" : "s"} without a running instance`, + ] + : []), + ...(inspection.unverifiedSessions.length > 0 + ? [ + `${inspection.unverifiedSessions.length} same-name lifecycle session collision${inspection.unverifiedSessions.length === 1 ? "" : "s"} with unverified ownership`, + ] + : []), ]; + let nextStep = "run: hack down"; + if (inspection.orphanedSessions.length > 0) { + nextStep = "run: hack doctor --fix"; + } else if (inspection.unverifiedSessions.length > 0) { + nextStep = "manual review required; no session was modified"; + } + return { name: "lifecycle hygiene", status: "warn", - message: `${details.join("; ")} (run: hack down)`, + message: `${details.join("; ")} (${nextStep})`, }; } @@ -2403,6 +2426,7 @@ async function runDoctorFix(opts: { return; } + await maybeRepairProjectLifecycleSessions({ startDir: opts.startDir }); await maybeRepairHackd(); const paths = getGlobalPaths(); @@ -2680,6 +2704,11 @@ export async function buildDoctorRemediationPlanLines(opts: { ]; const project = await findProjectContext(opts.startDir); + if (project) { + steps.push( + "Reconcile lifecycle sessions and remove only ownership-proven sessions whose Compose instance is absent." + ); + } const trackedGenerated = project ? await inspectTrackedGeneratedFiles({ projectRoot: project.projectRoot, @@ -2897,6 +2926,37 @@ function describeLegacyRepairStatus(input: { return input.didPruneLegacy ? "pruned" : "prune failed"; } +async function maybeRepairProjectLifecycleSessions(opts: { + readonly startDir: string; +}): Promise { + const project = await findProjectContext(opts.startDir); + if (!project) { + return; + } + const repaired = await repairProjectLifecycleSessions({ + projectDir: project.projectDir, + projectRoot: project.projectRoot, + }); + if (repaired.repairedSessions.length > 0) { + note( + [ + `Removed ${repaired.repairedSessions.length} ownership-proven orphan lifecycle session${repaired.repairedSessions.length === 1 ? "" : "s"}:`, + ...repaired.repairedSessions.map((sessionName) => `- ${sessionName}`), + ].join("\n"), + "lifecycle repair" + ); + } + if (repaired.failures.length > 0) { + note( + [ + "Lifecycle sessions left unchanged because safe repair could not be re-proven:", + ...repaired.failures.map((failure) => `- ${failure}`), + ].join("\n"), + "lifecycle repair" + ); + } +} + async function maybeInstallMutagenForDoctorFix(): Promise { if (getMutagenPath()) { return; diff --git a/src/commands/project.ts b/src/commands/project.ts index a52a41bd..858c12f2 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -162,6 +162,7 @@ import { readProcessSnapshot, resolveLifecycleProcessGroupIdsForTmuxState, resolveLifecycleStopProcessGroupIds, + terminateLifecycleProcessGroups, } from "../lib/project-lifecycle-processes.ts"; import { createLifecycleOwnershipToken, @@ -2596,40 +2597,6 @@ async function resolveLifecycleStopProcessGroupIdsForEntry(opts: { }); } -async function terminateLifecycleProcessGroups(opts: { - readonly processGroupIds: readonly number[]; -}): Promise { - const groups = [...new Set(opts.processGroupIds)].filter( - (processGroupId) => processGroupId > 1 - ); - if (groups.length === 0) { - return; - } - - for (const processGroupId of groups) { - try { - process.kill(-processGroupId, "SIGTERM"); - } catch { - // Ignore groups that already exited between snapshot and shutdown. - } - } - - await Bun.sleep(500); - - for (const processGroupId of groups) { - try { - process.kill(-processGroupId, 0); - } catch { - continue; - } - try { - process.kill(-processGroupId, "SIGKILL"); - } catch { - // Ignore groups that exited after the SIGTERM grace period. - } - } -} - function resolveLifecycleCommandServiceName(opts: { readonly command: ProjectLifecycleCommand; readonly index: number; diff --git a/src/lib/lifecycle-runtime.ts b/src/lib/lifecycle-runtime.ts index be199949..bb1e1230 100644 --- a/src/lib/lifecycle-runtime.ts +++ b/src/lib/lifecycle-runtime.ts @@ -218,6 +218,34 @@ export async function removeLifecycleStateEntryIfOwned(opts: { return true; } +export async function removeLifecycleStateEntryIfMatching(opts: { + readonly projectDir: string; + readonly expectedEntry: LifecycleStateEntry; +}): Promise { + const state = await readLifecycleState({ projectDir: opts.projectDir }); + const current = state.find( + (entry) => entry.composeProject === opts.expectedEntry.composeProject + ); + if ( + !current || + current.sessionName !== opts.expectedEntry.sessionName || + current.backend !== opts.expectedEntry.backend || + current.updatedAt !== opts.expectedEntry.updatedAt || + current.ownershipToken !== opts.expectedEntry.ownershipToken + ) { + return false; + } + await writeLifecycleStateFile({ + projectDir: opts.projectDir, + state: { + entries: state.filter( + (entry) => entry.composeProject !== opts.expectedEntry.composeProject + ), + }, + }); + return true; +} + function parseLifecycleStateEntry(value: unknown): LifecycleStateEntry | null { if (!isRecord(value)) { return null; diff --git a/src/lib/project-lifecycle-hygiene.ts b/src/lib/project-lifecycle-hygiene.ts index 1b2a65a6..d2fc50c5 100644 --- a/src/lib/project-lifecycle-hygiene.ts +++ b/src/lib/project-lifecycle-hygiene.ts @@ -1,45 +1,108 @@ +import { realpath } from "node:fs/promises"; + import { getMuxBackends } from "../mux/mux-resolver.ts"; import type { LifecycleBackend, LifecycleStateEntry, } from "./lifecycle-runtime.ts"; -import { readLifecycleState } from "./lifecycle-runtime.ts"; +import { + readLifecycleState, + removeLifecycleStateEntryIfMatching, +} from "./lifecycle-runtime.ts"; import { type ProcessSnapshotRow, readProcessSnapshot, resolveLeaderlessPersistedLifecycleProcessGroupIds, + resolveLifecycleStopProcessGroupIds, resolvePersistedLifecycleProcessGroupIds, + terminateLifecycleProcessGroups, } from "./project-lifecycle-processes.ts"; +import { + inspectLifecycleSession, + killInspectedLifecycleSession, + type LifecycleSessionClassification, + type LifecycleSessionInspection, +} from "./project-lifecycle-sessions.ts"; +import { + type RuntimeProject, + readRuntimeProjects, +} from "./runtime-projects.ts"; + +const LIFECYCLE_SESSION_ORPHAN_GRACE_MS = 5 * 60 * 1000; export type StaleLifecycleStateEntry = { readonly entry: LifecycleStateEntry; readonly liveProcessGroups: readonly number[]; }; +export type OrphanedLifecycleSession = { + readonly entry: LifecycleStateEntry; + readonly classification: Exclude< + LifecycleSessionClassification, + "absent" | "foreign" + >; +}; + +export type UnverifiedLifecycleSession = { + readonly entry: LifecycleStateEntry; + readonly reason: string; +}; + export type LifecycleHygieneInspection = { readonly staleEntries: readonly StaleLifecycleStateEntry[]; readonly orphanedProcessGroups: readonly number[]; + readonly orphanedSessions: readonly OrphanedLifecycleSession[]; + readonly unverifiedSessions: readonly UnverifiedLifecycleSession[]; + readonly runtimeAvailable: boolean; }; -/** Inspect persisted lifecycle state for entries whose mux session is gone. */ +export type LifecycleHygieneRepairResult = { + readonly repairedSessions: readonly string[]; + readonly failures: readonly string[]; +}; + +/** Inspect persisted lifecycle state, mux ownership, processes, and Compose liveness. */ export async function inspectProjectLifecycleHygiene(opts: { readonly projectDir: string; + readonly projectRoot: string; }): Promise { const entries = await readLifecycleState({ projectDir: opts.projectDir }); if (entries.length === 0) { - return { staleEntries: [], orphanedProcessGroups: [] }; + return { + staleEntries: [], + orphanedProcessGroups: [], + orphanedSessions: [], + unverifiedSessions: [], + runtimeAvailable: true, + }; } - const [sessionsByBackend, snapshot] = await Promise.all([ - listSessionsByBackend(), - readProcessSnapshot(), - ]); - - return inspectLifecycleStateEntries({ + const [sessionsByBackend, snapshot, activeComposeProjects] = + await Promise.all([ + listSessionsByBackend(), + readProcessSnapshot(), + readActiveComposeProjects({ projectDir: opts.projectDir }), + ]); + const stateInspection = inspectLifecycleStateEntries({ entries, sessionsByBackend, snapshot, }); + const sessionInspections = await inspectOwnedLifecycleSessions({ + entries, + projectRoot: opts.projectRoot, + }); + const sessionHygiene = classifyLifecycleSessionHygiene({ + entries, + sessionInspections, + activeComposeProjects, + }); + + return { + ...stateInspection, + ...sessionHygiene, + runtimeAvailable: activeComposeProjects !== null, + }; } /** Classify lifecycle entries that no longer have a matching mux session. */ @@ -50,7 +113,7 @@ export function inspectLifecycleStateEntries(opts: { ReadonlySet >; readonly snapshot: readonly ProcessSnapshotRow[]; -}): LifecycleHygieneInspection { +}): Pick { const staleEntries: StaleLifecycleStateEntry[] = []; const orphanedProcessGroups = new Set(); @@ -86,6 +149,194 @@ export function inspectLifecycleStateEntries(opts: { }; } +export function classifyLifecycleSessionHygiene(opts: { + readonly entries: readonly LifecycleStateEntry[]; + readonly sessionInspections: ReadonlyMap; + readonly activeComposeProjects: ReadonlySet | null; + readonly now?: Date; +}): Pick< + LifecycleHygieneInspection, + "orphanedSessions" | "unverifiedSessions" +> { + const orphanedSessions: OrphanedLifecycleSession[] = []; + const unverifiedSessions: UnverifiedLifecycleSession[] = []; + const nowMs = (opts.now ?? new Date()).getTime(); + + for (const entry of opts.entries) { + const inspection = opts.sessionInspections.get(entry.composeProject); + if (!inspection || inspection.classification === "absent") { + continue; + } + if ( + inspection.classification === "foreign" || + inspection.decision.kind === "block" + ) { + unverifiedSessions.push({ + entry, + reason: + inspection.decision.kind === "block" + ? inspection.decision.reason + : `Lifecycle session "${entry.sessionName}" ownership could not be verified.`, + }); + continue; + } + if ( + opts.activeComposeProjects !== null && + !opts.activeComposeProjects.has(entry.composeProject) + ) { + const stateAgeMs = nowMs - Date.parse(entry.updatedAt); + if ( + !Number.isFinite(stateAgeMs) || + stateAgeMs < LIFECYCLE_SESSION_ORPHAN_GRACE_MS + ) { + unverifiedSessions.push({ + entry, + reason: `Lifecycle session "${entry.sessionName}" is recent enough to belong to an in-flight startup.`, + }); + continue; + } + orphanedSessions.push({ + entry, + classification: inspection.classification, + }); + } + } + + return { orphanedSessions, unverifiedSessions }; +} + +/** Safely reap sessions that remain ownership-proven and runtime-orphaned. */ +export async function repairProjectLifecycleSessions(opts: { + readonly projectDir: string; + readonly projectRoot: string; +}): Promise { + const initial = await inspectProjectLifecycleHygiene(opts); + const repairedSessions: string[] = []; + const failures: string[] = []; + + for (const candidate of initial.orphanedSessions) { + const current = await inspectProjectLifecycleHygiene(opts); + const orphan = current.orphanedSessions.find( + (item) => item.entry.composeProject === candidate.entry.composeProject + ); + if (!orphan) { + failures.push( + `${candidate.entry.sessionName}: liveness or ownership changed before repair` + ); + continue; + } + const backend = getMuxBackends().get(orphan.entry.backend); + if (!backend?.available) { + failures.push(`${orphan.entry.sessionName}: mux backend unavailable`); + continue; + } + const inspection = await inspectLifecycleSession({ + backend, + entry: orphan.entry, + expectedSessionName: orphan.entry.sessionName, + expectedProjectRoot: opts.projectRoot, + expectedDefinitionHash: orphan.entry.definitionHash ?? "", + }); + if ( + inspection.decision.kind === "block" || + inspection.decision.kind === "create" + ) { + failures.push( + `${orphan.entry.sessionName}: ownership proof changed before repair` + ); + continue; + } + const processGroupIds = resolveLifecycleStopProcessGroupIds({ + matchedLiveSession: true, + lifecycleEntry: orphan.entry, + snapshot: await readProcessSnapshot(), + }); + const killed = await killInspectedLifecycleSession({ + backend, + inspection, + }); + if (!killed) { + failures.push(`${orphan.entry.sessionName}: mux teardown failed`); + continue; + } + await terminateLifecycleProcessGroups({ processGroupIds }); + const stateRemoved = await removeLifecycleStateEntryIfMatching({ + projectDir: opts.projectDir, + expectedEntry: orphan.entry, + }); + if (!stateRemoved) { + failures.push( + `${orphan.entry.sessionName}: session stopped, but lifecycle state changed concurrently` + ); + continue; + } + repairedSessions.push(orphan.entry.sessionName); + } + + return { repairedSessions, failures }; +} + +async function inspectOwnedLifecycleSessions(opts: { + readonly entries: readonly LifecycleStateEntry[]; + readonly projectRoot: string; +}): Promise> { + const backends = getMuxBackends(); + const inspections = new Map(); + for (const entry of opts.entries) { + const backend = backends.get(entry.backend); + if (!backend?.available) { + continue; + } + inspections.set( + entry.composeProject, + await inspectLifecycleSession({ + backend, + entry, + expectedSessionName: entry.sessionName, + expectedProjectRoot: opts.projectRoot, + expectedDefinitionHash: entry.definitionHash ?? "", + }) + ); + } + return inspections; +} + +async function readActiveComposeProjects(opts: { + readonly projectDir: string; +}): Promise | null> { + const runtime = await readRuntimeProjects({ includeGlobal: false }); + if (!runtime.ok) { + return null; + } + const normalizedProjectDir = await normalizePath(opts.projectDir); + const activeProjects = new Set(); + for (const project of runtime.runtime) { + if ( + !(project.workingDir && hasRunningComposeContainer(project)) || + (await normalizePath(project.workingDir)) !== normalizedProjectDir + ) { + continue; + } + activeProjects.add(project.project); + } + return activeProjects; +} + +function hasRunningComposeContainer(project: RuntimeProject): boolean { + for (const service of project.services.values()) { + if ( + service.containers.some( + (container) => + container.state === "running" && + container.labels?.["hack.lifecycle.process"] !== "true" + ) + ) { + return true; + } + } + return false; +} + async function listSessionsByBackend(): Promise< ReadonlyMap> > { @@ -106,3 +357,11 @@ async function listSessionsByBackend(): Promise< return sessionsByBackend; } + +async function normalizePath(path: string): Promise { + try { + return await realpath(path); + } catch { + return path; + } +} diff --git a/src/lib/project-lifecycle-processes.ts b/src/lib/project-lifecycle-processes.ts index 38045976..66a9459e 100644 --- a/src/lib/project-lifecycle-processes.ts +++ b/src/lib/project-lifecycle-processes.ts @@ -191,3 +191,38 @@ export async function readProcessSnapshot(): Promise { } return parseProcessSnapshotOutput(result.stdout); } + +/** Terminate ownership-proven lifecycle groups with a bounded TERM/KILL sequence. */ +export async function terminateLifecycleProcessGroups(opts: { + readonly processGroupIds: readonly number[]; +}): Promise { + const groups = [...new Set(opts.processGroupIds)].filter( + (processGroupId) => processGroupId > 1 + ); + if (groups.length === 0) { + return; + } + + for (const processGroupId of groups) { + try { + process.kill(-processGroupId, "SIGTERM"); + } catch { + // Ignore groups that already exited between snapshot and shutdown. + } + } + + await Bun.sleep(500); + + for (const processGroupId of groups) { + try { + process.kill(-processGroupId, 0); + } catch { + continue; + } + try { + process.kill(-processGroupId, "SIGKILL"); + } catch { + // Ignore groups that exited after the SIGTERM grace period. + } + } +} diff --git a/tests/doctor-command.test.ts b/tests/doctor-command.test.ts index 28e03776..3d7dee17 100644 --- a/tests/doctor-command.test.ts +++ b/tests/doctor-command.test.ts @@ -80,7 +80,8 @@ test("doctor remediation plan mentions env migration when requested for a legacy expect(lines).toEqual([ "1. Review and repair local network, CoreDNS, CA, host TLS env, and daemon drift where needed.", "2. Repair tickets refs if the project repo needs it.", - "3. Prompt to migrate legacy env config (.hack/hack.env.json) to hack.env.*.yaml.", + "3. Reconcile lifecycle sessions and remove only ownership-proven sessions whose Compose instance is absent.", + "4. Prompt to migrate legacy env config (.hack/hack.env.json) to hack.env.*.yaml.", ]); } finally { await rm(root, { recursive: true, force: true }); @@ -98,7 +99,8 @@ test("doctor remediation plan skips env migration when modern env files already expect(lines).toEqual([ "1. Review and repair local network, CoreDNS, CA, host TLS env, and daemon drift where needed.", "2. Repair tickets refs if the project repo needs it.", - "3. Skip env migration because this project already uses hack.env.*.yaml.", + "3. Reconcile lifecycle sessions and remove only ownership-proven sessions whose Compose instance is absent.", + "4. Skip env migration because this project already uses hack.env.*.yaml.", ]); } finally { await rm(root, { recursive: true, force: true }); @@ -190,6 +192,23 @@ test("doctor guidance routes stale lifecycle state to hack down", () => { expect(guidance.followUp).toEqual([]); }); +test("doctor guidance routes owned orphan lifecycle sessions to doctor fix", () => { + const guidance = buildDoctorRecoveryGuidance({ + results: [ + { + name: "lifecycle hygiene", + status: "warn", + message: + "1 owned lifecycle session without a running instance (run: hack doctor --fix)", + }, + ], + }); + + expect(guidance.temporaryBreakage).toEqual([]); + expect(guidance.configurationRepair).toEqual(["hack doctor --fix"]); + expect(guidance.followUp).toEqual([]); +}); + test("doctor guidance routes env materialization drift to hack env materialize", () => { const guidance = buildDoctorRecoveryGuidance({ results: [ diff --git a/tests/e2e/harness.ts b/tests/e2e/harness.ts index e36936ab..5e9c4ddf 100644 --- a/tests/e2e/harness.ts +++ b/tests/e2e/harness.ts @@ -6,9 +6,9 @@ import { join, resolve } from "node:path"; /** * Core end-to-end harness for the hack CLI. * - * Every scenario drives the ACTUAL working-tree CLI by spawning - * `bun /index.ts ` (non-TTY by design — this is the - * agent-reality surface we are testing) against disposable fixture repos. + * Every scenario drives the actual working-tree CLI by spawning + * `bun /index.ts `, or the compiled binary selected by + * `HACK_E2E_CLI_BIN`, against disposable fixture repos. * * Isolation model: every CLI invocation runs with `HACK_HOME=` so * global state (projects registry, global config) never touches the real @@ -22,6 +22,12 @@ export const REPO_ROOT = resolve(import.meta.dir, "..", ".."); /** CLI entrypoint spawned for every scenario command. */ export const CLI_ENTRYPOINT = resolve(REPO_ROOT, "index.ts"); +/** Resolve the source or compiled CLI command used by the E2E harness. */ +export function resolveCliSpawnArgs(args: readonly string[]): string[] { + const binary = process.env.HACK_E2E_CLI_BIN?.trim(); + return binary ? [resolve(binary), ...args] : ["bun", CLI_ENTRYPOINT, ...args]; +} + const DEFAULT_COMMAND_TIMEOUT_MS = 120_000; const REGISTRY_SCAN_MAX_DEPTH = 6; const OUTPUT_EXCERPT_LIMIT = 2000; @@ -237,7 +243,7 @@ export async function seedIsolatedHackHome(opts: { let outputCaptureCounter = 0; /** - * Spawn the working-tree CLI (`bun index.ts `) with stdin closed so any + * Spawn the selected working-tree CLI with stdin closed so any * unexpected prompt fails fast instead of hanging. stdout/stderr are captured * via temp files (not pipes) so a spawned grandchild (e.g. a daemon) can * never hold the capture open past the CLI's own exit. @@ -260,7 +266,7 @@ export async function runCli(opts: { const stdoutPath = join(captureDir, `${outputCaptureCounter}.out`); const stderrPath = join(captureDir, `${outputCaptureCounter}.err`); - const proc = Bun.spawn(["bun", CLI_ENTRYPOINT, ...opts.invocation.args], { + const proc = Bun.spawn(resolveCliSpawnArgs(opts.invocation.args), { cwd: opts.invocation.cwd, env, stdin: "ignore", diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts index c2e336fc..2b277a40 100644 --- a/tests/e2e/run.ts +++ b/tests/e2e/run.ts @@ -21,6 +21,7 @@ import { worktreeSecretsScenario } from "./scenarios/worktree-secrets.ts"; * bun tests/e2e/run.ts --only=init,doctor # subset by name * bun tests/e2e/run.ts --list # list scenarios and exit * HACK_E2E_KEEP=1 ... # keep temp fixtures for debugging + * HACK_E2E_CLI_BIN=./dist/hack ... # exercise the compiled binary * * Exit codes: 0 all pass/skip, 1 any scenario failed, 2 isolation canary * failed (nothing ran). diff --git a/tests/e2e/scenarios/lifecycle-session-recovery.ts b/tests/e2e/scenarios/lifecycle-session-recovery.ts index 2d3e46d7..1dd062d5 100644 --- a/tests/e2e/scenarios/lifecycle-session-recovery.ts +++ b/tests/e2e/scenarios/lifecycle-session-recovery.ts @@ -4,9 +4,9 @@ import { join } from "node:path"; import { createMonorepoFixture } from "../fixture.ts"; import { buildCliEnv, - CLI_ENTRYPOINT, expect, expectExit, + resolveCliSpawnArgs, runCommand, type Scenario, } from "../harness.ts"; @@ -301,6 +301,80 @@ export const lifecycleSessionRecoveryScenario: Scenario = { cwd: downFailureFixture.root, }); } + + const doctorFixture = await createMonorepoFixture({ + parentDir: ctx.tempRoot, + withHackConfig: true, + lifecycle: { + persistentProcess: true, + disableInternal: true, + }, + }); + const doctorSession = `${doctorFixture.name}--lifecycle`; + const doctorStatePath = join( + doctorFixture.hackDir, + ".internal", + "lifecycle", + "state.json" + ); + try { + const doctorUp = await ctx.cli({ + args: ["up", "--detach", "--json"], + cwd: doctorFixture.root, + env: cliEnv, + }); + expectExit({ + result: doctorUp, + codes: [0], + message: "doctor orphan fixture should start its lifecycle session", + }); + await ageLifecycleState({ statePath: doctorStatePath }); + const doctor = await ctx.cli({ + args: ["doctor"], + cwd: doctorFixture.root, + env: cliEnv, + timeoutMs: 240_000, + }); + expect({ + that: + !doctor.timedOut && + doctor.combined.includes("owned lifecycle session") && + doctor.combined.includes("hack doctor --fix"), + message: "doctor did not report the ownership-proven orphan session", + result: doctor, + }); + const doctorFix = await ctx.cli({ + args: ["doctor", "--fix"], + cwd: doctorFixture.root, + env: cliEnv, + timeoutMs: 240_000, + }); + expect({ + that: + !doctorFix.timedOut && + (doctorFix.exitCode === 0 || doctorFix.exitCode === 1) && + doctorFix.combined.includes("lifecycle repair"), + message: "doctor --fix did not run lifecycle repair", + result: doctorFix, + }); + await expectSessionAbsent({ + sessionName: doctorSession, + cwd: doctorFixture.root, + phase: "doctor --fix", + }); + expect({ + that: + (await readState({ statePath: doctorStatePath })).entries.length === + 0, + message: "doctor --fix left lifecycle ownership state behind", + result: doctorFix, + }); + } finally { + await runCommand({ + argv: ["tmux", "kill-session", "-t", doctorSession], + cwd: doctorFixture.root, + }); + } } finally { await runCommand({ argv: ["tmux", "kill-session", "-t", sessionName], @@ -335,6 +409,7 @@ async function writeFakeDocker(opts: { " esac", "fi", 'if [ "$1" = "info" ]; then exit 0; fi', + 'if [ "$1" = "ps" ]; then exit 0; fi', "exit 1", "", ].join("\n") @@ -342,13 +417,32 @@ async function writeFakeDocker(opts: { await chmod(path, 0o755); } +async function ageLifecycleState(opts: { + readonly statePath: string; +}): Promise { + const state = await readState({ statePath: opts.statePath }); + await Bun.write( + opts.statePath, + `${JSON.stringify( + { + entries: state.entries.map((entry) => ({ + ...entry, + updatedAt: "2020-01-01T00:00:00.000Z", + })), + }, + null, + 2 + )}\n` + ); +} + async function runSignalProbe(opts: { readonly cwd: string; readonly hackHome: string; readonly path: string; readonly sessionName: string; }): Promise<{ readonly exitCode: number; readonly stderr: string }> { - const proc = Bun.spawn(["bun", CLI_ENTRYPOINT, "up", "--detach", "--json"], { + const proc = Bun.spawn(resolveCliSpawnArgs(["up", "--detach", "--json"]), { cwd: opts.cwd, env: buildCliEnv({ hackHome: opts.hackHome, diff --git a/tests/e2e/scenarios/worktree-parallel-up.ts b/tests/e2e/scenarios/worktree-parallel-up.ts index 07bf1b69..157b8912 100644 --- a/tests/e2e/scenarios/worktree-parallel-up.ts +++ b/tests/e2e/scenarios/worktree-parallel-up.ts @@ -88,6 +88,53 @@ export const worktreeParallelUpScenario: Scenario = { message: `docker compose ls should show a distinct branch-instance project for "${BRANCH}" while the primary is also running`, }); + const primaryProject = fixture.name; + const branchProject = `${fixture.name}--${BRANCH}`; + const primaryIdsBefore = await readContainerIds({ + cwd: fixture.root, + composeProject: primaryProject, + }); + const branchIdsBefore = await readContainerIds({ + cwd: fixture.root, + composeProject: branchProject, + }); + expect({ + that: primaryIdsBefore.length > 0 && branchIdsBefore.length > 0, + message: "both compose instances should expose running container ids", + }); + + const restartPrimary = await ctx.cli({ + args: ["restart", "--json"], + cwd: fixture.root, + timeoutMs: UP_TIMEOUT_MS, + }); + expectExit({ + result: restartPrimary, + codes: [0], + message: "primary checkout restart should succeed", + }); + const primaryIdsAfter = await readContainerIds({ + cwd: fixture.root, + composeProject: primaryProject, + }); + const branchIdsAfter = await readContainerIds({ + cwd: fixture.root, + composeProject: branchProject, + }); + expect({ + that: + primaryIdsAfter.length > 0 && + primaryIdsAfter.join(",") !== primaryIdsBefore.join(","), + message: "primary restart did not replace the primary containers", + result: restartPrimary, + }); + expect({ + that: branchIdsAfter.join(",") === branchIdsBefore.join(","), + message: + "primary restart changed the linked-worktree branch containers", + result: restartPrimary, + }); + const downWorktree = await ctx.cli({ args: ["down"], cwd: worktreePath, @@ -114,6 +161,31 @@ export const worktreeParallelUpScenario: Scenario = { }, }; +async function readContainerIds(opts: { + readonly cwd: string; + readonly composeProject: string; +}): Promise { + const result = await runCommand({ + argv: [ + "docker", + "ps", + "--filter", + `label=com.docker.compose.project=${opts.composeProject}`, + "--format", + "{{.ID}}", + ], + cwd: opts.cwd, + }); + if (result.exitCode !== 0) { + return []; + } + return result.stdout + .split("\n") + .map((id) => id.trim()) + .filter((id) => id.length > 0) + .sort((left, right) => left.localeCompare(right)); +} + async function resolveOpenUrl(opts: { readonly ctx: Parameters[0]; readonly cwd: string; diff --git a/tests/project-lifecycle-hygiene.test.ts b/tests/project-lifecycle-hygiene.test.ts index 8793515c..dd352961 100644 --- a/tests/project-lifecycle-hygiene.test.ts +++ b/tests/project-lifecycle-hygiene.test.ts @@ -1,7 +1,11 @@ import { expect, test } from "bun:test"; import type { LifecycleStateEntry } from "../src/lib/lifecycle-runtime.ts"; -import { inspectLifecycleStateEntries } from "../src/lib/project-lifecycle-hygiene.ts"; +import { + classifyLifecycleSessionHygiene, + inspectLifecycleStateEntries, +} from "../src/lib/project-lifecycle-hygiene.ts"; +import { classifyLifecycleSession } from "../src/lib/project-lifecycle-sessions.ts"; const staleEntry: LifecycleStateEntry = { composeProject: "event-agent", @@ -65,3 +69,120 @@ test("inspectLifecycleStateEntries reports leaderless groups behind a live mux s expect(inspection.staleEntries).toEqual([]); expect(inspection.orphanedProcessGroups).toEqual([500, 503]); }); + +test("classifyLifecycleSessionHygiene reports an owned session without a running instance", () => { + const ownedEntry: LifecycleStateEntry = { + ...staleEntry, + ownershipToken: "owner-token", + definitionHash: "definition-hash", + }; + const sessionInspection = classifyLifecycleSession({ + session: { + backend: "tmux", + name: ownedEntry.sessionName, + attached: false, + path: "/tmp/event-agent", + windows: 2, + createdAt: "2026-04-01T13:59:59.000Z", + }, + entry: ownedEntry, + observedOwnershipToken: "owner-token", + expectedBackend: "tmux", + expectedSessionName: ownedEntry.sessionName, + expectedProjectRoot: "/tmp/event-agent", + expectedDefinitionHash: "definition-hash", + liveWindowNames: new Set(["shell", "proxy"]), + }); + + const inspection = classifyLifecycleSessionHygiene({ + entries: [ownedEntry], + sessionInspections: new Map([ + [ownedEntry.composeProject, sessionInspection], + ]), + activeComposeProjects: new Set(), + now: new Date("2026-04-01T14:10:00.000Z"), + }); + + expect(inspection.orphanedSessions).toEqual([ + { entry: ownedEntry, classification: "owned-healthy" }, + ]); + expect(inspection.unverifiedSessions).toEqual([]); +}); + +test("classifyLifecycleSessionHygiene preserves a recent owned session during startup", () => { + const ownedEntry: LifecycleStateEntry = { + ...staleEntry, + ownershipToken: "owner-token", + definitionHash: "definition-hash", + }; + const sessionInspection = classifyLifecycleSession({ + session: { + backend: "tmux", + name: ownedEntry.sessionName, + attached: false, + path: "/tmp/event-agent", + windows: 2, + createdAt: "2026-04-01T13:59:59.000Z", + }, + entry: ownedEntry, + observedOwnershipToken: "owner-token", + expectedBackend: "tmux", + expectedSessionName: ownedEntry.sessionName, + expectedProjectRoot: "/tmp/event-agent", + expectedDefinitionHash: "definition-hash", + liveWindowNames: new Set(["shell", "proxy"]), + }); + + const inspection = classifyLifecycleSessionHygiene({ + entries: [ownedEntry], + sessionInspections: new Map([ + [ownedEntry.composeProject, sessionInspection], + ]), + activeComposeProjects: new Set(), + now: new Date("2026-04-01T14:01:00.000Z"), + }); + + expect(inspection.orphanedSessions).toEqual([]); + expect(inspection.unverifiedSessions).toEqual([ + { + entry: ownedEntry, + reason: + 'Lifecycle session "event-agent--lifecycle-feature-cleanup" is recent enough to belong to an in-flight startup.', + }, + ]); +}); + +test("classifyLifecycleSessionHygiene preserves foreign collisions for manual review", () => { + const foreignInspection = classifyLifecycleSession({ + session: { + backend: "tmux", + name: staleEntry.sessionName, + attached: false, + path: "/tmp/unrelated", + windows: 1, + createdAt: "2026-04-01T13:59:59.000Z", + }, + entry: staleEntry, + observedOwnershipToken: null, + expectedBackend: "tmux", + expectedSessionName: staleEntry.sessionName, + expectedProjectRoot: "/tmp/event-agent", + expectedDefinitionHash: "", + liveWindowNames: new Set(["shell"]), + }); + + const inspection = classifyLifecycleSessionHygiene({ + entries: [staleEntry], + sessionInspections: new Map([ + [staleEntry.composeProject, foreignInspection], + ]), + activeComposeProjects: new Set(), + now: new Date("2026-04-01T14:10:00.000Z"), + }); + + expect(inspection.orphanedSessions).toEqual([]); + expect(inspection.unverifiedSessions).toHaveLength(1); + expect(inspection.unverifiedSessions[0]?.reason).toContain( + "cannot be proven" + ); +}); From f849649393b6b73a010f788f8f2039dbea76619f Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 9 Jul 2026 17:12:04 -0400 Subject: [PATCH 3/7] ci: run Docker end-to-end scenarios --- .github/workflows/ci.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14e73136..dd2f6dc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,24 @@ jobs: - name: Smoke slim runtime mounted-project env flow run: bash scripts/portable-container-smoke.sh hack-runtime-ci:slim linux/amd64 + docker-e2e: + runs-on: blacksmith-4vcpu-ubuntu-2404 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: "1.3.9" + - name: Install tmux + run: sudo apt-get update && sudo apt-get install --yes tmux + - name: Install dependencies + run: bun install + - name: Run local and Docker E2E + run: bun run test:e2e:local:docker + test: runs-on: blacksmith-6vcpu-macos-15 steps: From fd66e44fe7585f842f946e7885c2ed4bd4fe8139 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 9 Jul 2026 17:15:46 -0400 Subject: [PATCH 4/7] ci: require Docker scenarios to execute --- .github/workflows/ci.yml | 4 +++- tests/e2e/run.ts | 13 ++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd2f6dc3..e7685b15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,8 +71,10 @@ jobs: run: sudo apt-get update && sudo apt-get install --yes tmux - name: Install dependencies run: bun install + - name: Create isolated Hack network + run: docker network create --subnet 172.30.0.0/16 hack-dev - name: Run local and Docker E2E - run: bun run test:e2e:local:docker + run: HACK_E2E_REQUIRE_DOCKER=1 bun run test:e2e:local:docker test: runs-on: blacksmith-6vcpu-macos-15 diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts index 2b277a40..e6e883b6 100644 --- a/tests/e2e/run.ts +++ b/tests/e2e/run.ts @@ -22,6 +22,7 @@ import { worktreeSecretsScenario } from "./scenarios/worktree-secrets.ts"; * bun tests/e2e/run.ts --list # list scenarios and exit * HACK_E2E_KEEP=1 ... # keep temp fixtures for debugging * HACK_E2E_CLI_BIN=./dist/hack ... # exercise the compiled binary + * HACK_E2E_REQUIRE_DOCKER=1 ... # fail if a Docker scenario skips * * Exit codes: 0 all pass/skip, 1 any scenario failed, 2 isolation canary * failed (nothing ran). @@ -108,6 +109,7 @@ async function main(): Promise { ); const dockerEnabled = (process.env.HACK_E2E_DOCKER ?? "") === "1"; + const requireDocker = (process.env.HACK_E2E_REQUIRE_DOCKER ?? "") === "1"; const keepTempDirs = (process.env.HACK_E2E_KEEP ?? "") === "1"; const outcomes = await runScenarios({ scenarios: ALL_SCENARIOS, @@ -116,9 +118,14 @@ async function main(): Promise { keepTempDirs, }); - process.exitCode = outcomes.some((outcome) => outcome.status === "fail") - ? 1 - : 0; + const dockerSkipped = outcomes.some( + (outcome) => outcome.tier === "docker" && outcome.status === "skip" + ); + process.exitCode = + outcomes.some((outcome) => outcome.status === "fail") || + (requireDocker && dockerSkipped) + ? 1 + : 0; } await main(); From 2148cf5c41a5574a8bfe58d756c96e08eb13dc3d Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 9 Jul 2026 17:36:58 -0400 Subject: [PATCH 5/7] fix(lifecycle): refresh sessions for env changes --- .codex/skills/hack-cli/SKILL.md | 2 +- .cursor/rules/hack.mdc | 2 +- .github/workflows/ci.yml | 2 +- AGENTS.md | 2 +- CLAUDE.md | 2 +- docs/architecture.md | 4 +- docs/lifecycle.md | 3 + src/agents/instruction-source.ts | 2 +- src/commands/project.ts | 18 ++- src/lib/project-lifecycle-sessions.ts | 23 ++++ tests/e2e-capabilities.test.ts | 38 ++++++ tests/e2e/README.md | 6 +- tests/e2e/capabilities.ts | 18 +++ tests/e2e/run.ts | 1 + .../scenarios/lifecycle-session-recovery.ts | 111 ++++++++++++++++++ tests/project-lifecycle-sessions.test.ts | 36 ++++++ 16 files changed, 261 insertions(+), 9 deletions(-) create mode 100644 tests/e2e-capabilities.test.ts create mode 100644 tests/e2e/capabilities.ts diff --git a/.codex/skills/hack-cli/SKILL.md b/.codex/skills/hack-cli/SKILL.md index 827b6eb4..28d98aa5 100644 --- a/.codex/skills/hack-cli/SKILL.md +++ b/.codex/skills/hack-cli/SKILL.md @@ -113,7 +113,7 @@ Use `hack` as the primary interface for local-first development. - For fixed-port host helpers such as SSM tunnels or local proxies, set `singleton.ports` and usually `onConflict: "adopt"` so Hack reuses a healthy existing listener instead of starting duplicate tunnel stacks. - `singleton` is a listener guard, not process ownership transfer; adopted external processes are left running on `hack down`. - Inspect lifecycle status via `hack projects --details` and stream via `hack logs `. -- Lifecycle session recovery is ownership-proven: Hack adopts healthy token-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof. +- Lifecycle session recovery is ownership-proven: Hack adopts healthy token-, definition-, and environment-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof. - `hack doctor --fix` reaps an orphan lifecycle session only when mux ownership is proven and its Compose instance is absent; unverified same-name sessions are never modified. ## Workspaces (mux-managed, tmux-first by default) diff --git a/.cursor/rules/hack.mdc b/.cursor/rules/hack.mdc index b0ee93e9..abf8425e 100644 --- a/.cursor/rules/hack.mdc +++ b/.cursor/rules/hack.mdc @@ -61,7 +61,7 @@ Prefer `hack` when shell access is available. Use MCP only when shell access is - For fixed-port host helpers such as SSM tunnels or local proxies, set `singleton.ports` and usually `onConflict: "adopt"` so Hack reuses a healthy existing listener instead of starting duplicate tunnel stacks. - `singleton` is a listener guard, not process ownership transfer; adopted external processes are left running on `hack down`. - Inspect lifecycle status via `hack projects --details` and stream via `hack logs `. -- Lifecycle session recovery is ownership-proven: Hack adopts healthy token-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof. +- Lifecycle session recovery is ownership-proven: Hack adopts healthy token-, definition-, and environment-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof. - `hack doctor --fix` reaps an orphan lifecycle session only when mux ownership is proven and its Compose instance is absent; unverified same-name sessions are never modified. ## Host-side env helpers diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7685b15..016f8f59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: - name: Create isolated Hack network run: docker network create --subnet 172.30.0.0/16 hack-dev - name: Run local and Docker E2E - run: HACK_E2E_REQUIRE_DOCKER=1 bun run test:e2e:local:docker + run: HACK_E2E_REQUIRE_DOCKER=1 HACK_E2E_REQUIRE_TMUX=1 bun run test:e2e:local:docker test: runs-on: blacksmith-6vcpu-macos-15 diff --git a/AGENTS.md b/AGENTS.md index cf6ee07e..d63ef80f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -299,7 +299,7 @@ Lifecycle + startup: - For fixed-port host helpers such as SSM tunnels or local proxies, set `singleton.ports` and usually `onConflict: "adopt"` so Hack reuses a healthy existing listener instead of starting duplicate tunnel stacks. - `singleton` is a listener guard, not process ownership transfer; adopted external processes are left running on `hack down`. - Inspect lifecycle status via `hack projects --details` and stream via `hack logs `. -- Lifecycle session recovery is ownership-proven: Hack adopts healthy token-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof. +- Lifecycle session recovery is ownership-proven: Hack adopts healthy token-, definition-, and environment-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof. - `hack doctor --fix` reaps an orphan lifecycle session only when mux ownership is proven and its Compose instance is absent; unverified same-name sessions are never modified. Workspaces (mux-managed, tmux-first by default): diff --git a/CLAUDE.md b/CLAUDE.md index 7659bd17..12d4c70f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -169,7 +169,7 @@ Lifecycle + startup: - For fixed-port host helpers such as SSM tunnels or local proxies, set `singleton.ports` and usually `onConflict: "adopt"` so Hack reuses a healthy existing listener instead of starting duplicate tunnel stacks. - `singleton` is a listener guard, not process ownership transfer; adopted external processes are left running on `hack down`. - Inspect lifecycle status via `hack projects --details` and stream via `hack logs `. -- Lifecycle session recovery is ownership-proven: Hack adopts healthy token-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof. +- Lifecycle session recovery is ownership-proven: Hack adopts healthy token-, definition-, and environment-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof. - `hack doctor --fix` reaps an orphan lifecycle session only when mux ownership is proven and its Compose instance is absent; unverified same-name sessions are never modified. Workspaces (mux-managed, tmux-first by default): diff --git a/docs/architecture.md b/docs/architecture.md index 435b16d1..c96d14ce 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -126,7 +126,9 @@ Lifecycle state is checkout-local at `.hack/.internal/lifecycle/state.json`. For saved process group belongs to the wrapped user command rather than its mux pane shell. Each compose instance has its own entry, so starting a branch instance does not replace the base instance's entry in the same checkout. A random token mirrored in mux metadata and lifecycle state proves current -session ownership; deterministic session names never authorize teardown. Cleanup reconciles the saved process group with the live process table; if the group +session ownership; deterministic session names never authorize teardown. A versioned SHA-256 digest +of the effective overlay and sorted lifecycle environment participates in session freshness without +persisting environment names, keys, or values. Cleanup reconciles the saved process group with the live process table; if the group leader exited while members remain, `hack doctor` reports the orphan. `hack down` terminates the persisted leaderless group and descendant process groups only while a matching lifecycle mux session still proves ownership; without that session, cleanup stays non-destructive because the numeric PGID diff --git a/docs/lifecycle.md b/docs/lifecycle.md index 249a368b..e44676ee 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -230,6 +230,9 @@ Notes: - Teardown is implemented by killing the lifecycle session; anything running inside that session will be stopped. - Current lifecycle sessions carry the same random ownership token in mux metadata and persisted state. Cleanup requires an exact token match; deterministic names alone never authorize session teardown. +- Healthy-session adoption also matches a stable SHA-256 fingerprint of the effective overlay and + sorted lifecycle environment. Environment names, keys, and values are never persisted; changing an + overlay or value replaces the owned session so host helpers cannot retain stale configuration. - For tmux-backed lifecycle sessions, Hack also persists the pane PID and the wrapped command's actual process-group metadata to `.hack/.internal/lifecycle/state.json`, with per-hook/process output logged to `.hack/.internal/lifecycle/*.log`. If tmux pane state disappears before teardown, `hack down` still uses that persisted metadata to clean up any live lifecycle process groups instead of leaving orphaned host processes behind. diff --git a/src/agents/instruction-source.ts b/src/agents/instruction-source.ts index 3a830c73..7b927ff3 100644 --- a/src/agents/instruction-source.ts +++ b/src/agents/instruction-source.ts @@ -185,7 +185,7 @@ export const INSTRUCTION_SECTIONS: readonly InstructionSection[] = [ 'For fixed-port host helpers such as SSM tunnels or local proxies, set `singleton.ports` and usually `onConflict: "adopt"` so Hack reuses a healthy existing listener instead of starting duplicate tunnel stacks.', "`singleton` is a listener guard, not process ownership transfer; adopted external processes are left running on `hack down`.", "Inspect lifecycle status via `hack projects --details` and stream via `hack logs `.", - "Lifecycle session recovery is ownership-proven: Hack adopts healthy token-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof.", + "Lifecycle session recovery is ownership-proven: Hack adopts healthy token-, definition-, and environment-matched sessions, replaces owned stale sessions, and refuses to kill same-name sessions without deterministic ownership proof.", "`hack doctor --fix` reaps an orphan lifecycle session only when mux ownership is proven and its Compose instance is absent; unverified same-name sessions are never modified.", ], }, diff --git a/src/commands/project.ts b/src/commands/project.ts index 858c12f2..71b53853 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -170,6 +170,7 @@ import { killInspectedLifecycleSession, killLifecycleSessionWithOwnership, resolveLifecycleDefinitionHash, + resolveLifecycleEnvironmentFingerprint, } from "../lib/project-lifecycle-sessions.ts"; import { inspectListeningTcpPorts, @@ -1618,12 +1619,15 @@ async function runLifecycleUpBeforeAndProcesses(opts: { readonly projectName: string; readonly branch: string | null; readonly env: Readonly>; + readonly effectiveEnvName: string | null; readonly composeProject: string; }): Promise { const beforeCommands = opts.cfg.lifecycle?.up?.before; const definitionHash = resolveProjectLifecycleDefinitionHash({ beforeCommands, processes: opts.cfg.lifecycle?.processes, + env: opts.env, + effectiveEnvName: opts.effectiveEnvName, }); if (!hasPersistentLifecycleCommands(beforeCommands)) { const beforeCode = await runLifecycleCommands({ @@ -1753,6 +1757,8 @@ function installLifecycleSignalCleanup(opts: { function resolveProjectLifecycleDefinitionHash(opts: { readonly beforeCommands: readonly ProjectLifecycleCommand[] | undefined; readonly processes: readonly ProjectLifecycleProcess[] | undefined; + readonly env?: Readonly>; + readonly effectiveEnvName?: string | null; }): string { const persistentBefore = (opts.beforeCommands ?? []) .map((command, index) => ({ command, index })) @@ -1771,8 +1777,16 @@ function resolveProjectLifecycleDefinitionHash(opts: { cwd: process.cwd ?? null, singleton: process.singleton ?? null, })); + const environmentFingerprint = resolveLifecycleEnvironmentFingerprint({ + effectiveEnvName: opts.effectiveEnvName ?? null, + env: opts.env ?? {}, + }); return resolveLifecycleDefinitionHash({ - definitions: [...persistentBefore, ...processes], + definitions: [ + ...persistentBefore, + ...processes, + { kind: "environment", fingerprint: environmentFingerprint }, + ], }); } @@ -5757,6 +5771,7 @@ async function runUpCommand({ projectName, branch, env: envOverrides.env, + effectiveEnvName: envOverrides.effectiveEnvName, composeProject: lifecycleComposeProject, }); if (lifecycleUp.code !== 0) { @@ -6274,6 +6289,7 @@ async function runRestartUpPhase(opts: { projectName: opts.projectName, branch: opts.branch, env: envOverrides.env, + effectiveEnvName: envOverrides.effectiveEnvName, composeProject: opts.lifecycleComposeProject, }); if (lifecycleUp.code !== 0) { diff --git a/src/lib/project-lifecycle-sessions.ts b/src/lib/project-lifecycle-sessions.ts index 974ed65b..95ddedc9 100644 --- a/src/lib/project-lifecycle-sessions.ts +++ b/src/lib/project-lifecycle-sessions.ts @@ -43,6 +43,29 @@ export function resolveLifecycleDefinitionHash(opts: { .digest("hex"); } +/** Return a stable digest without persisting environment names, keys, or values. */ +export function resolveLifecycleEnvironmentFingerprint(opts: { + readonly effectiveEnvName: string | null; + readonly env: Readonly>; +}): string { + const entries = Object.entries(opts.env).sort(([left], [right]) => { + if (left === right) { + return 0; + } + return left < right ? -1 : 1; + }); + const digest = new Bun.CryptoHasher("sha256") + .update( + JSON.stringify({ + version: 1, + effectiveEnvName: opts.effectiveEnvName, + entries, + }) + ) + .digest("hex"); + return `sha256:${digest}`; +} + export function classifyLifecycleSession(opts: { readonly session: MuxSession | null; readonly entry: LifecycleStateEntry | null; diff --git a/tests/e2e-capabilities.test.ts b/tests/e2e-capabilities.test.ts new file mode 100644 index 00000000..f6bac7b1 --- /dev/null +++ b/tests/e2e-capabilities.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, test } from "bun:test"; + +import { resolveExecutableCapability } from "./e2e/capabilities.ts"; + +describe("resolveExecutableCapability", () => { + test("runs when the executable is available", () => { + expect( + resolveExecutableCapability({ + executable: "tmux", + executablePath: "/usr/bin/tmux", + required: true, + installHint: "install tmux", + }) + ).toEqual({ kind: "available" }); + }); + + test("skips an unavailable optional local capability", () => { + expect( + resolveExecutableCapability({ + executable: "tmux", + executablePath: null, + required: false, + installHint: "install tmux", + }) + ).toEqual({ kind: "skip", reason: "tmux unavailable (install tmux)" }); + }); + + test("fails an unavailable capability required by CI", () => { + expect( + resolveExecutableCapability({ + executable: "tmux", + executablePath: null, + required: true, + installHint: "install tmux", + }) + ).toEqual({ kind: "fail", reason: "tmux unavailable (install tmux)" }); + }); +}); diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 1baae03b..af7fd6fc 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -14,6 +14,7 @@ bun run test:e2e:local:docker # tier 1 + tier 2 (docker + real global infr bun tests/e2e/run.ts --list # list scenarios bun tests/e2e/run.ts --only=init # run a subset by name HACK_E2E_KEEP=1 bun tests/e2e/run.ts --only=doctor # keep temp dirs for debugging +HACK_E2E_REQUIRE_TMUX=1 bun tests/e2e/run.ts --only=lifecycle-session-recovery ``` Exit codes: `0` all pass/skip, `1` any scenario failed, `2` isolation canary @@ -49,7 +50,10 @@ any e2e result. - **Tier 1 (`local`)** — runs everywhere; no docker needed: `automation-check`, `init`, `env-secrets`, `worktree-secrets`, `worktree-registry`, `worktree-branch-default`, `agent-docs-sync`, - `doctor` (doctor tolerates missing docker — it must report, not crash). + `doctor` (doctor tolerates missing docker — it must report, not crash), and + `lifecycle-session-recovery`. The lifecycle recovery scenario skips when + tmux is unavailable unless `HACK_E2E_REQUIRE_TMUX=1` makes that capability + mandatory (as it is in the dedicated Docker/tmux CI job). - **Tier 2 (`docker`)** — opt-in via `HACK_E2E_DOCKER=1`; requires a running docker daemon, the machine's global `hack-dev` network (`hack global install`), and pulls `oven/bun:1`: `up-down`, `lifecycle-host-process`, diff --git a/tests/e2e/capabilities.ts b/tests/e2e/capabilities.ts new file mode 100644 index 00000000..f0fd7e54 --- /dev/null +++ b/tests/e2e/capabilities.ts @@ -0,0 +1,18 @@ +export type ExecutableCapability = + | { readonly kind: "available" } + | { readonly kind: "skip" | "fail"; readonly reason: string }; + +/** Resolve whether an executable-backed scenario may run, skip locally, or must fail. */ +export function resolveExecutableCapability(opts: { + readonly executable: string; + readonly executablePath: string | null; + readonly required: boolean; + readonly installHint: string; +}): ExecutableCapability { + if (opts.executablePath) { + return { kind: "available" }; + } + + const reason = `${opts.executable} unavailable (${opts.installHint})`; + return opts.required ? { kind: "fail", reason } : { kind: "skip", reason }; +} diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts index e6e883b6..68f55427 100644 --- a/tests/e2e/run.ts +++ b/tests/e2e/run.ts @@ -23,6 +23,7 @@ import { worktreeSecretsScenario } from "./scenarios/worktree-secrets.ts"; * HACK_E2E_KEEP=1 ... # keep temp fixtures for debugging * HACK_E2E_CLI_BIN=./dist/hack ... # exercise the compiled binary * HACK_E2E_REQUIRE_DOCKER=1 ... # fail if a Docker scenario skips + * HACK_E2E_REQUIRE_TMUX=1 ... # fail if tmux recovery cannot run * * Exit codes: 0 all pass/skip, 1 any scenario failed, 2 isolation canary * failed (nothing ran). diff --git a/tests/e2e/scenarios/lifecycle-session-recovery.ts b/tests/e2e/scenarios/lifecycle-session-recovery.ts index 1dd062d5..2f87513f 100644 --- a/tests/e2e/scenarios/lifecycle-session-recovery.ts +++ b/tests/e2e/scenarios/lifecycle-session-recovery.ts @@ -1,6 +1,7 @@ import { chmod, mkdir } from "node:fs/promises"; import { join } from "node:path"; +import { resolveExecutableCapability } from "../capabilities.ts"; import { createMonorepoFixture } from "../fixture.ts"; import { buildCliEnv, @@ -18,6 +19,18 @@ export const lifecycleSessionRecoveryScenario: Scenario = { tier: "local", summary: "owned lifecycle sessions reconcile and clean up without Docker", run: async (ctx) => { + const tmux = resolveExecutableCapability({ + executable: "tmux", + executablePath: Bun.which("tmux"), + required: process.env.HACK_E2E_REQUIRE_TMUX === "1", + installHint: "install tmux to run lifecycle recovery E2E", + }); + if (tmux.kind === "skip") { + ctx.skip(tmux.reason); + } + if (tmux.kind === "fail") { + throw new Error(tmux.reason); + } const fixture = await createMonorepoFixture({ parentDir: ctx.tempRoot, withHackConfig: true, @@ -76,6 +89,66 @@ export const lifecycleSessionRecoveryScenario: Scenario = { result: repeatedUp, }); + await writeEnvOverlay({ + hackDir: fixture.hackDir, + envName: "qa", + value: "qa-first", + }); + const qaUp = await ctx.cli({ + args: ["up", "--detach", "--json", "--env", "qa"], + cwd: fixture.root, + env: cliEnv, + }); + expectExit({ + result: qaUp, + codes: [0], + message: "switching lifecycle overlays should succeed", + }); + const qaToken = (await readState({ statePath })).entries[0] + ?.ownershipToken; + expect({ + that: Boolean(qaToken) && qaToken !== firstToken, + message: + "switching overlays adopted lifecycle helpers from the old environment", + result: qaUp, + }); + await expectSessionEnvironment({ + sessionName, + cwd: fixture.root, + key: "E2E_PLAIN", + value: "qa-first", + }); + + await writeEnvOverlay({ + hackDir: fixture.hackDir, + envName: "qa", + value: "qa-second", + }); + const changedValueUp = await ctx.cli({ + args: ["up", "--detach", "--json", "--env", "qa"], + cwd: fixture.root, + env: cliEnv, + }); + expectExit({ + result: changedValueUp, + codes: [0], + message: "changing lifecycle env values should succeed", + }); + const changedValueToken = (await readState({ statePath })).entries[0] + ?.ownershipToken; + expect({ + that: Boolean(changedValueToken) && changedValueToken !== qaToken, + message: + "changed env values adopted lifecycle helpers from the old environment", + result: changedValueUp, + }); + await expectSessionEnvironment({ + sessionName, + cwd: fixture.root, + key: "E2E_PLAIN", + value: "qa-second", + }); + const branchName = "feat-msp-human-handoff"; const branchUp = await ctx.cli({ args: ["up", "--detach", "--json", "--branch", branchName], @@ -417,6 +490,44 @@ async function writeFakeDocker(opts: { await chmod(path, 0o755); } +async function writeEnvOverlay(opts: { + readonly hackDir: string; + readonly envName: string; + readonly value: string; +}): Promise { + await Bun.write( + join(opts.hackDir, `hack.env.${opts.envName}.yaml`), + [ + "version: 1", + `environment: ${opts.envName}`, + "secretsprovider: project_key", + "values:", + " global:", + ` E2E_PLAIN: ${opts.value}`, + "", + ].join("\n") + ); +} + +async function expectSessionEnvironment(opts: { + readonly sessionName: string; + readonly cwd: string; + readonly key: string; + readonly value: string; +}): Promise { + const result = await runCommand({ + argv: ["tmux", "show-environment", "-t", opts.sessionName, opts.key], + cwd: opts.cwd, + }); + expect({ + that: + result.exitCode === 0 && + result.stdout.trim() === `${opts.key}=${opts.value}`, + message: `lifecycle session did not receive ${opts.key}=${opts.value}`, + result, + }); +} + async function ageLifecycleState(opts: { readonly statePath: string; }): Promise { diff --git a/tests/project-lifecycle-sessions.test.ts b/tests/project-lifecycle-sessions.test.ts index a31b8ff1..ba41b472 100644 --- a/tests/project-lifecycle-sessions.test.ts +++ b/tests/project-lifecycle-sessions.test.ts @@ -5,6 +5,7 @@ import { classifyLifecycleSession, killLifecycleSessionWithOwnership, resolveLifecycleDefinitionHash, + resolveLifecycleEnvironmentFingerprint, } from "../src/lib/project-lifecycle-sessions.ts"; import type { MuxBackend, MuxSession } from "../src/mux/mux-backend.ts"; @@ -38,6 +39,41 @@ const entry: LifecycleStateEntry = { ], }; +test("resolveLifecycleEnvironmentFingerprint is stable without exposing env data", () => { + const first = resolveLifecycleEnvironmentFingerprint({ + effectiveEnvName: "qa", + env: { SECOND: "two", FIRST: "one" }, + }); + const reordered = resolveLifecycleEnvironmentFingerprint({ + effectiveEnvName: "qa", + env: { FIRST: "one", SECOND: "two" }, + }); + + expect(first).toBe(reordered); + expect(first).toMatch(/^sha256:[a-f0-9]{64}$/); + expect(first).not.toContain("qa"); + expect(first).not.toContain("FIRST"); + expect(first).not.toContain("one"); +}); + +test("resolveLifecycleEnvironmentFingerprint changes with overlay or values", () => { + const base = resolveLifecycleEnvironmentFingerprint({ + effectiveEnvName: "qa", + env: { SERVICE_URL: "https://qa.invalid" }, + }); + const changedValue = resolveLifecycleEnvironmentFingerprint({ + effectiveEnvName: "qa", + env: { SERVICE_URL: "https://prod.invalid" }, + }); + const changedOverlay = resolveLifecycleEnvironmentFingerprint({ + effectiveEnvName: "prod", + env: { SERVICE_URL: "https://qa.invalid" }, + }); + + expect(changedValue).not.toBe(base); + expect(changedOverlay).not.toBe(base); +}); + test("classifyLifecycleSession creates when the expected session is absent", () => { const inspection = classifyLifecycleSession({ session: null, From 806693458747ed51de73bdec07e562c7ff10d640 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 9 Jul 2026 17:50:47 -0400 Subject: [PATCH 6/7] fix(lifecycle): preserve sessions when shutdown guards fail --- docs/lifecycle.md | 3 + src/commands/project.ts | 6 +- .../scenarios/lifecycle-session-recovery.ts | 74 ++++++++++++++++++- 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/docs/lifecycle.md b/docs/lifecycle.md index e44676ee..dcc347cf 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -200,6 +200,9 @@ signals received while the mux session is still being initialized. 3. Stop lifecycle processes by killing the lifecycle session. 4. Run `lifecycle.down.after` hooks. +If `lifecycle.down.before` fails, shutdown is aborted before Compose or lifecycle processes are +stopped. `hack restart` preserves the same guard semantics during its down phase. + ### `hack restart` `hack restart` performs the same lifecycle steps as `hack down` followed by `hack up`. diff --git a/src/commands/project.ts b/src/commands/project.ts index 71b53853..fbaa2596 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -2357,6 +2357,8 @@ async function startLifecycleProcess(opts: { const result = await exec( [ "zellij", + "--session", + opts.sessionName, "run", "--close-on-exit", "--name", @@ -2369,7 +2371,7 @@ async function startLifecycleProcess(opts: { { stdin: "ignore", cwd, - env: { ...opts.env, ZELLIJ_SESSION_NAME: opts.sessionName }, + env: { ...opts.env }, } ); if (result.exitCode !== 0) { @@ -6066,7 +6068,6 @@ async function runDownCommand({ composeProject: lifecycleComposeProject, }); if (beforeCode !== 0) { - await lifecycleDownCleanup(); if (json) { return emitLifecycleResult({ result: errorResult({ @@ -6182,7 +6183,6 @@ async function runRestartDownPhase(opts: { composeProject: opts.lifecycleComposeProject, }); if (downBefore !== 0) { - await lifecycleDownCleanup(); return downBefore; } diff --git a/tests/e2e/scenarios/lifecycle-session-recovery.ts b/tests/e2e/scenarios/lifecycle-session-recovery.ts index 2f87513f..3eed9b63 100644 --- a/tests/e2e/scenarios/lifecycle-session-recovery.ts +++ b/tests/e2e/scenarios/lifecycle-session-recovery.ts @@ -340,6 +340,12 @@ export const lifecycleSessionRecoveryScenario: Scenario = { }, }); const downFailureSession = `${downFailureFixture.name}--lifecycle`; + const downFailureStatePath = join( + downFailureFixture.hackDir, + ".internal", + "lifecycle", + "state.json" + ); try { const downFailureUp = await ctx.cli({ args: ["up", "--detach", "--json"], @@ -351,6 +357,15 @@ export const lifecycleSessionRecoveryScenario: Scenario = { codes: [0], message: "down-failure fixture should start", }); + const downFailureToken = ( + await readState({ statePath: downFailureStatePath }) + ).entries[0]?.ownershipToken; + expect({ + that: Boolean(downFailureToken), + message: "down-failure fixture did not persist lifecycle ownership", + result: downFailureUp, + }); + await Bun.write(dockerLog, ""); const failedDown = await ctx.cli({ args: ["down", "--json"], cwd: downFailureFixture.root, @@ -363,11 +378,55 @@ export const lifecycleSessionRecoveryScenario: Scenario = { message: "down.before failure should propagate", result: failedDown, }); - await expectSessionAbsent({ + await expectSessionPresent({ sessionName: downFailureSession, cwd: downFailureFixture.root, phase: "down.before failure", }); + expect({ + that: + (await readState({ statePath: downFailureStatePath })).entries[0] + ?.ownershipToken === downFailureToken, + message: "down.before failure removed lifecycle ownership state", + result: failedDown, + }); + expect({ + that: !(await Bun.file(dockerLog).text()).includes(" down"), + message: "down.before failure still invoked compose down", + result: failedDown, + }); + + await Bun.write(dockerLog, ""); + const failedRestart = await ctx.cli({ + args: ["restart", "--json"], + cwd: downFailureFixture.root, + env: cliEnv, + }); + expect({ + that: + failedRestart.exitCode !== 0 && + failedRestart.combined.includes("Restart down phase failed"), + message: "restart down.before failure should propagate", + result: failedRestart, + }); + await expectSessionPresent({ + sessionName: downFailureSession, + cwd: downFailureFixture.root, + phase: "restart down.before failure", + }); + expect({ + that: + (await readState({ statePath: downFailureStatePath })).entries[0] + ?.ownershipToken === downFailureToken, + message: + "restart down.before failure removed lifecycle ownership state", + result: failedRestart, + }); + expect({ + that: !(await Bun.file(dockerLog).text()).includes(" down"), + message: "restart down.before failure still invoked compose down", + result: failedRestart, + }); } finally { await runCommand({ argv: ["tmux", "kill-session", "-t", downFailureSession], @@ -617,6 +676,19 @@ async function expectSessionAbsent(opts: { }); } +async function expectSessionPresent(opts: { + readonly sessionName: string; + readonly cwd: string; + readonly phase: string; +}): Promise { + const presence = await inspectSessionPresence(opts); + expect({ + that: presence.present, + message: `${opts.phase}: lifecycle session ${opts.sessionName} is absent`, + result: presence.result, + }); +} + async function inspectSessionPresence(opts: { readonly sessionName: string; readonly cwd: string; From 91a0c2e0c80246ab956f5546a6a4be62c003f79a Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 9 Jul 2026 17:50:54 -0400 Subject: [PATCH 7/7] fix(zellij): target detached sessions explicitly --- src/mux/zellij-backend.ts | 51 ++++++++++++-------- tests/zellij-backend.test.ts | 92 ++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 21 deletions(-) create mode 100644 tests/zellij-backend.test.ts diff --git a/src/mux/zellij-backend.ts b/src/mux/zellij-backend.ts index 136d1429..5136bf58 100644 --- a/src/mux/zellij-backend.ts +++ b/src/mux/zellij-backend.ts @@ -77,13 +77,14 @@ export function createZellijBackend(): MuxBackend { const ownerResult = await exec( [ "zellij", + "--session", + opts.name, "action", "rename-tab", `${LIFECYCLE_OWNER_TAB_PREFIX}${opts.lifecycleOwnerToken}`, ], { stdin: "ignore", - env: { ZELLIJ_SESSION_NAME: opts.name }, } ); if (ownerResult.exitCode !== 0) { @@ -112,10 +113,10 @@ export function createZellijBackend(): MuxBackend { const readLifecycleOwnerToken = async (opts: { readonly name: string; }): Promise => { - const result = await exec(["zellij", "action", "dump-layout"], { - stdin: "ignore", - env: { ZELLIJ_SESSION_NAME: opts.name }, - }); + const result = await exec( + ["zellij", "--session", opts.name, "action", "dump-layout"], + { stdin: "ignore" } + ); if (result.exitCode !== 0) { return null; } @@ -126,10 +127,10 @@ export function createZellijBackend(): MuxBackend { const listSessionWindowNames = async (opts: { readonly name: string; }): Promise | null> => { - const result = await exec(["zellij", "action", "dump-layout"], { - stdin: "ignore", - env: { ZELLIJ_SESSION_NAME: opts.name }, - }); + const result = await exec( + ["zellij", "--session", opts.name, "action", "dump-layout"], + { stdin: "ignore" } + ); if (result.exitCode !== 0) { return null; } @@ -145,14 +146,22 @@ export function createZellijBackend(): MuxBackend { readonly command: string; readonly env?: Readonly>; }): Promise => { - // `zellij run` requires an active session; set env to target the desired session. - return await exec(["zellij", "run", "--", "sh", "-lc", opts.command], { - stdin: "ignore", - env: { - ...(opts.env ?? {}), - ZELLIJ_SESSION_NAME: opts.name, - }, - }); + return await exec( + [ + "zellij", + "--session", + opts.name, + "run", + "--", + "sh", + "-lc", + opts.command, + ], + { + stdin: "ignore", + env: { ...(opts.env ?? {}) }, + } + ); }; const sendInput = async (opts: { @@ -160,10 +169,10 @@ export function createZellijBackend(): MuxBackend { readonly keys: string; }): Promise => { // Best-effort. This sends raw characters and does not attempt to encode special key chords. - return await exec(["zellij", "action", "write-chars", opts.keys], { - stdin: "ignore", - env: { ZELLIJ_SESSION_NAME: opts.name }, - }); + return await exec( + ["zellij", "--session", opts.name, "action", "write-chars", opts.keys], + { stdin: "ignore" } + ); }; return { diff --git a/tests/zellij-backend.test.ts b/tests/zellij-backend.test.ts new file mode 100644 index 00000000..ffa5afd8 --- /dev/null +++ b/tests/zellij-backend.test.ts @@ -0,0 +1,92 @@ +import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"; + +import type { ExecOptions } from "../src/lib/shell.ts"; +import { registerScopedModuleMock } from "./helpers/scoped-module-mock.ts"; + +const execCalls: Array<{ + readonly command: readonly string[]; + readonly options: ExecOptions; +}> = []; + +const shellMock = await registerScopedModuleMock({ + importerPath: import.meta.path, + specifier: "../src/lib/shell.ts", + overrides: { + exec: async (command: readonly string[], options: ExecOptions = {}) => { + execCalls.push({ command: [...command], options }); + if (command.includes("list-sessions")) { + return { exitCode: 0, stdout: "demo\n", stderr: "" }; + } + if (command.includes("dump-layout")) { + return { + exitCode: 0, + stdout: + 'layout { tab name="hack-lifecycle-owner-owner-123" { pane name="api" } }', + stderr: "", + }; + } + return { exitCode: 0, stdout: "", stderr: "" }; + }, + findExecutableInPath: () => "/usr/bin/zellij", + }, +}); + +beforeAll(() => { + shellMock.activate(); +}); + +beforeEach(() => { + execCalls.length = 0; +}); + +afterAll(() => { + shellMock.deactivate(); +}); + +async function loadZellijBackend() { + const { createZellijBackend } = await import( + `../src/mux/zellij-backend.ts?test=${Date.now()}-${Math.random()}` + ); + return createZellijBackend(); +} + +test("zellij backend targets detached sessions with the global session flag", async () => { + const backend = await loadZellijBackend(); + const created = await backend.createSession({ + name: "demo", + cwd: "/repo", + env: { PROJECT_ENV: "qa" }, + lifecycleOwnerToken: "owner-123", + }); + const ownerToken = await backend.readLifecycleOwnerToken?.({ name: "demo" }); + const windowNames = await backend.listSessionWindowNames?.({ name: "demo" }); + await backend.execInSession({ + name: "demo", + command: "bun worker.ts", + env: { PROJECT_ENV: "qa" }, + }); + await backend.sendInput({ name: "demo", keys: "hello" }); + + expect(created.ok).toBe(true); + expect(ownerToken).toBe("owner-123"); + expect(windowNames).toEqual(new Set(["api"])); + expect(execCalls.map(({ command }) => command)).toEqual([ + ["zellij", "attach", "--create-background", "demo"], + [ + "zellij", + "--session", + "demo", + "action", + "rename-tab", + "hack-lifecycle-owner-owner-123", + ], + ["zellij", "list-sessions", "--no-formatting", "--short"], + ["zellij", "--session", "demo", "action", "dump-layout"], + ["zellij", "--session", "demo", "action", "dump-layout"], + ["zellij", "--session", "demo", "run", "--", "sh", "-lc", "bun worker.ts"], + ["zellij", "--session", "demo", "action", "write-chars", "hello"], + ]); + expect(execCalls[1]?.options.env?.ZELLIJ_SESSION_NAME).toBeUndefined(); + expect(execCalls[3]?.options.env?.ZELLIJ_SESSION_NAME).toBeUndefined(); + expect(execCalls[5]?.options.env).toEqual({ PROJECT_ENV: "qa" }); +});