fix(runtime): persist wrapped command process group

Author: roodboi

docs/architecture.md

@@ -122,9 +122,10 @@ Projects can run host-side hooks around `hack up/down` and start managed host pr
 local proxies/tunnels). Processes are started inside a mux session (tmux or zellij) so they have a
 stable home and can be torn down on `hack down`.
 
-Lifecycle state is checkout-local at `.hack/.internal/lifecycle/state.json`. Each compose instance
-has its own entry, so starting a branch instance does not replace the base instance's entry in the
-same checkout. Cleanup reconciles the saved process group with the live process table; if the group
+Lifecycle state is checkout-local at `.hack/.internal/lifecycle/state.json`. For tmux processes, the
+saved process group belongs to the wrapped user command rather than its mux pane shell. Each compose
+instance has its own entry, so starting a branch instance does not replace the base instance's entry
+in the same checkout. Cleanup reconciles the saved process group with the live process table; if the group
 leader exited while members remain, `hack doctor` reports the orphan. `hack down` terminates the
 persisted leaderless group and descendant process groups only while a matching lifecycle mux session
 still proves ownership; without that session, cleanup stays non-destructive because the numeric PGID

docs/lifecycle.md

@@ -212,8 +212,8 @@ Lifecycle session name:
 Notes:
 - If no mux backend is available, lifecycle process startup fails with an actionable error.
 - Teardown is implemented by killing the lifecycle session; anything running inside that session will be stopped.
-- For tmux-backed lifecycle sessions, Hack also persists pane PID and process-group metadata to
-  `.hack/.internal/lifecycle/state.json`, with per-hook/process output logged to
+- For tmux-backed lifecycle sessions, Hack also persists the pane PID and the wrapped command's actual
+  process-group metadata to `.hack/.internal/lifecycle/state.json`, with per-hook/process output logged to
   `.hack/.internal/lifecycle/*.log`. If tmux pane state disappears before teardown, `hack down` still uses that persisted metadata to clean up any live lifecycle process groups instead of leaving orphaned host processes behind.
 - `hack doctor` reports stale lifecycle state when the persisted lifecycle entry no longer has a live mux session and points operators to `hack down` so cleanup and state removal happen through the supported path. `hack doctor --fix` does not tear down lifecycle sessions itself — use `hack down` for that.
 

src/commands/project.ts

@@ -1,3 +1,4 @@
+import { rm } from "node:fs/promises";
 import { dirname, resolve } from "node:path";
 import {
   autocompleteMultiselect,
@@ -210,6 +211,9 @@ const LABELS_LINE_PATTERN = /^(\s*)labels:\s*$/;
 /** Regex to extract leading whitespace (indentation). */
 const INDENT_PATTERN = /^(\s*)/;
 
+const LIFECYCLE_COMMAND_PID_WAIT_ATTEMPTS = 50;
+const LIFECYCLE_COMMAND_PID_WAIT_INTERVAL_MS = 20;
+
 /** Regex to match caddy label line in YAML. */
 const CADDY_LABEL_PATTERN = /^(\s*)caddy:\s*(.*)$/;
 
@@ -1974,13 +1978,16 @@ async function startLifecycleProcess(opts: {
     projectDir: opts.projectDir,
     composeProject: opts.composeProject,
   });
-  const wrappedCommand = wrapLifecyclePersistentCommand({
-    command: opts.process.command,
-    logPath,
-    serviceName: opts.process.name,
-  });
-
   if (opts.backend === "tmux") {
+    const commandPidPath = `${logPath}.${windowName}.pid`;
+    await ensureDir(dirname(commandPidPath));
+    await rm(commandPidPath, { force: true });
+    const wrappedCommand = wrapLifecyclePersistentCommand({
+      command: opts.process.command,
+      commandPidPath,
+      logPath,
+      serviceName: opts.process.name,
+    });
     const result = await exec(
       [
         "tmux",
@@ -1998,6 +2005,7 @@ async function startLifecycleProcess(opts: {
       { stdin: "ignore" }
     );
     if (result.exitCode !== 0) {
+      await rm(commandPidPath, { force: true });
       throw new Error(
         `Failed to start lifecycle process "${opts.process.name}": ${result.stderr.trim()}`
       );
@@ -2007,10 +2015,17 @@ async function startLifecycleProcess(opts: {
       windowName,
     });
     const panePid = panePids[0];
-    const processGroupId =
-      panePid !== undefined
-        ? await readProcessGroupIdForPid({ pid: panePid })
-        : null;
+    let processGroupId: number | null = null;
+    try {
+      processGroupId = await waitForLifecycleCommandProcessGroupId({
+        commandPidPath,
+      });
+    } finally {
+      await rm(commandPidPath, { force: true });
+    }
+    if (!processGroupId && panePid !== undefined) {
+      processGroupId = await readProcessGroupIdForPid({ pid: panePid });
+    }
     await appendLifecycleLogRecord({
       projectDir: opts.projectDir,
       composeProject: opts.composeProject,
@@ -2030,6 +2045,12 @@ async function startLifecycleProcess(opts: {
     };
   }
 
+  const wrappedCommand = wrapLifecyclePersistentCommand({
+    command: opts.process.command,
+    commandPidPath: null,
+    logPath,
+    serviceName: opts.process.name,
+  });
   const result = await exec(
     ["zellij", "run", "--", "sh", "-c", wrappedCommand],
     {
@@ -2189,6 +2210,29 @@ async function readProcessGroupIdForPid(opts: {
   return Number.isInteger(parsed) && parsed > 0 ? parsed : null;
 }
 
+async function waitForLifecycleCommandProcessGroupId(opts: {
+  readonly commandPidPath: string;
+}): Promise<number | null> {
+  for (
+    let attempt = 0;
+    attempt < LIFECYCLE_COMMAND_PID_WAIT_ATTEMPTS;
+    attempt += 1
+  ) {
+    const commandPidRaw = await readTextFile(opts.commandPidPath);
+    const commandPid = Number.parseInt(commandPidRaw?.trim() ?? "", 10);
+    if (Number.isInteger(commandPid) && commandPid > 1) {
+      const processGroupId = await readProcessGroupIdForPid({
+        pid: commandPid,
+      });
+      if (processGroupId) {
+        return processGroupId;
+      }
+    }
+    await Bun.sleep(LIFECYCLE_COMMAND_PID_WAIT_INTERVAL_MS);
+  }
+  return null;
+}
+
 async function resolveLifecycleProcessGroupIds(opts: {
   readonly sessionName: string;
   readonly lifecycleEntry: LifecycleStateEntry | null;
@@ -2267,16 +2311,22 @@ function resolveLifecycleCommandServiceName(opts: {
 
 export function wrapLifecyclePersistentCommand(opts: {
   readonly command: string;
+  readonly commandPidPath: string | null;
   readonly logPath: string;
   readonly serviceName: string;
 }): string {
   const logPath = shellSingleQuote(opts.logPath);
   const service = shellSingleQuote(opts.serviceName);
   const command = shellSingleQuote(opts.command);
+  const commandPidPath =
+    opts.commandPidPath === null ? null : shellSingleQuote(opts.commandPidPath);
   return [
     `HACK_LIFECYCLE_LOG=${logPath}`,
     `HACK_LIFECYCLE_SERVICE=${service}`,
     `HACK_LIFECYCLE_COMMAND=${command}`,
+    ...(commandPidPath
+      ? [`HACK_LIFECYCLE_COMMAND_PID_FILE=${commandPidPath}`]
+      : []),
     `fifo="$(mktemp -u "\${TMPDIR:-/tmp}/hack-lifecycle.XXXXXX")"`,
     'mkfifo "$fifo"',
     "cleanup_lifecycle() {",
@@ -2294,6 +2344,7 @@ export function wrapLifecyclePersistentCommand(opts: {
     `  if [ -n "\${reader_pid:-}" ]; then`,
     '    wait "$reader_pid" 2>/dev/null || true',
     "  fi",
+    ...(commandPidPath ? ['  rm -f "$HACK_LIFECYCLE_COMMAND_PID_FILE"'] : []),
     '  rm -f "$fifo"',
     "}",
     'trap "cleanup_lifecycle; exit 130" INT TERM HUP',
@@ -2304,11 +2355,21 @@ export function wrapLifecyclePersistentCommand(opts: {
     '  done < "$fifo" ) &',
     "reader_pid=$!",
     "if command -v python3 >/dev/null 2>&1; then",
-    '  python3 -c \'import os, sys; os.setsid(); os.execvp("sh", ["sh", "-c", sys.argv[1]])\' "$HACK_LIFECYCLE_COMMAND" >"$fifo" 2>&1 &',
+    ...(commandPidPath
+      ? [
+          '  python3 -c \'import os, sys; os.setsid(); pid_file = open(sys.argv[2], "w"); pid_file.write(str(os.getpid())); pid_file.close(); os.execvp("sh", ["sh", "-c", sys.argv[1]])\' "$HACK_LIFECYCLE_COMMAND" "$HACK_LIFECYCLE_COMMAND_PID_FILE" >"$fifo" 2>&1 &',
+        ]
+      : [
+          '  python3 -c \'import os, sys; os.setsid(); os.execvp("sh", ["sh", "-c", sys.argv[1]])\' "$HACK_LIFECYCLE_COMMAND" >"$fifo" 2>&1 &',
+        ]),
+    "  cmd_pid=$!",
     "else",
     '  sh -c "$HACK_LIFECYCLE_COMMAND" >"$fifo" 2>&1 &',
+    "  cmd_pid=$!",
+    ...(commandPidPath
+      ? ['  printf "%s\\n" "$cmd_pid" > "$HACK_LIFECYCLE_COMMAND_PID_FILE"']
+      : []),
     "fi",
-    "cmd_pid=$!",
     'wait "$cmd_pid"',
     "cmd_status=$?",
     'cmd_pid=""',

tests/e2e/scenarios/lifecycle-host-process.ts

@@ -1,30 +1,44 @@
 import { join } from "node:path";
 
 import { createMonorepoFixture } from "../fixture.ts";
-import { expect, expectExit, type Scenario } from "../harness.ts";
+import { expect, expectExit, runCommand, type Scenario } from "../harness.ts";
 import { downBestEffort, requireDockerPreconditions } from "./docker-shared.ts";
 
 const UP_TIMEOUT_MS = 420_000;
 
+type LifecycleStateFile = {
+  readonly entries?: readonly {
+    readonly processes?: readonly {
+      readonly name?: string;
+      readonly panePid?: number;
+      readonly processGroupId?: number;
+    }[];
+  }[];
+};
+
 /**
  * Lifecycle host hooks run as part of `hack up` (there is no standalone
  * lifecycle start/stop command today), so this scenario lives in the docker
  * tier: `hack up --detach` must execute the configured lifecycle.up.before
  * hook on the host (observed via a marker file) before starting services,
- * and record lifecycle runtime state under .hack/.internal/lifecycle/.
+ * record lifecycle runtime state under .hack/.internal/lifecycle/, and retain
+ * the wrapped command's actual process group for teardown.
  */
 export const lifecycleHostProcessScenario: Scenario = {
   name: "lifecycle-host-process",
   tier: "docker",
-  summary: "lifecycle.up.before host hook runs during hack up",
+  summary: "lifecycle hooks and command-group teardown run during up/down",
   run: async (ctx) => {
     await requireDockerPreconditions({ ctx });
 
     const markerFile = join(ctx.tempRoot, "lifecycle-marker.txt");
     const fixture = await createMonorepoFixture({
       parentDir: ctx.tempRoot,
       withHackConfig: true,
-      lifecycle: { upBeforeMarkerFile: markerFile },
+      lifecycle: {
+        persistentProcess: true,
+        upBeforeMarkerFile: markerFile,
+      },
     });
 
     try {
@@ -54,12 +68,57 @@ export const lifecycleHostProcessScenario: Scenario = {
       const stateFile = Bun.file(
         join(fixture.hackDir, ".internal", "lifecycle", "state.json")
       );
-      const hasState = await stateFile.exists();
-      ctx.log(
-        hasState
-          ? "lifecycle state recorded at .hack/.internal/lifecycle/state.json"
-          : "note: no lifecycle state.json written (no persistent processes configured)"
+      expect({
+        that: await stateFile.exists(),
+        message: "persistent lifecycle process did not write lifecycle state",
+        result: up,
+      });
+      const state = (await stateFile.json()) as LifecycleStateFile;
+      const sleeper = state.entries?.[0]?.processes?.find(
+        (process) => process.name === "e2e-sleeper"
       );
+      const panePid = sleeper?.panePid ?? 0;
+      const processGroupId = sleeper?.processGroupId ?? 0;
+      expect({
+        that:
+          Number.isInteger(panePid) &&
+          panePid > 1 &&
+          Number.isInteger(processGroupId) &&
+          processGroupId > 1 &&
+          processGroupId !== panePid,
+        message:
+          "lifecycle state should persist the wrapped command group, not the tmux pane group",
+        result: up,
+      });
+
+      const groupLeader = await runCommand({
+        argv: ["ps", "-o", "pgid=", "-p", String(processGroupId)],
+        cwd: fixture.root,
+      });
+      expectExit({
+        result: groupLeader,
+        codes: [0],
+        message: "persisted lifecycle command group leader should be live",
+      });
+      expect({
+        that: Number.parseInt(groupLeader.stdout.trim(), 10) === processGroupId,
+        message:
+          "persisted lifecycle PGID does not match the live command group",
+        result: groupLeader,
+      });
+
+      await downBestEffort({ ctx, fixture });
+      let commandGroupStillExists = true;
+      try {
+        process.kill(-processGroupId, 0);
+      } catch {
+        commandGroupStillExists = false;
+      }
+      expect({
+        that: !commandGroupStillExists,
+        message: "hack down left the lifecycle command process group alive",
+        result: up,
+      });
     } finally {
       await downBestEffort({ ctx, fixture });
     }

tests/project-lifecycle-processes.test.ts

@@ -339,6 +339,7 @@ test("resolveLifecycleStopProcessGroupIds recovers a leaderless persisted group
 test("wrapLifecyclePersistentCommand uses external kill for process-group cleanup", () => {
   const script = wrapLifecyclePersistentCommand({
     command: "bun run proxy",
+    commandPidPath: "/tmp/event-agent.pid",
     logPath: "/tmp/event-agent.log",
     serviceName: "proxy",
   });
@@ -353,12 +354,18 @@ test("wrapLifecyclePersistentCommand uses external kill for process-group cleanu
 test("wrapLifecyclePersistentCommand avoids login-shell execution", () => {
   const script = wrapLifecyclePersistentCommand({
     command: "bun run proxy",
+    commandPidPath: "/tmp/event-agent.pid",
     logPath: "/tmp/event-agent.log",
     serviceName: "proxy",
   });
 
   expect(script).toContain('os.execvp("sh", ["sh", "-c", sys.argv[1]])');
   expect(script).toContain('sh -c "$HACK_LIFECYCLE_COMMAND" >"$fifo" 2>&1 &');
+  expect(script).toContain("os.setsid()");
+  expect(script).toContain("pid_file.write(str(os.getpid()))");
+  expect(script).toContain(
+    'printf "%s\\n" "$cmd_pid" > "$HACK_LIFECYCLE_COMMAND_PID_FILE"'
+  );
   expect(script).not.toContain('os.execvp("sh", ["sh", "-lc", sys.argv[1]])');
   expect(script).not.toContain(
     'sh -lc "$HACK_LIFECYCLE_COMMAND" >"$fifo" 2>&1 &'