Second instance on one account: settings-file selector and per-instance job names

CHANGELOG.md

@@ -6,6 +6,16 @@ Versions follow [SemVer](https://semver.org).
 
 ## [Unreleased]
 
+- Support separate instances on one cluster account: process-only absolute
+  `OUTERLOOP_ENV_FILE`, with the existing ownership/write-permission checks,
+  and stable settings-path suffixes for resident and per-cadence scheduler jobs.
+  Init, launch, deploy, harness status, and successor recovery use the selected
+  settings and instance identity.
+- Upgrading: no action needed for the default fleet; its settings path and job
+  names remain unchanged. Existing run records, leases, and heartbeats need no
+  migration. Stop additional instances before rolling back to a version without
+  instance isolation.
+
 - Launch, submit, and stale-submit checkpoint scope violations now refuse every
   request and resume the author with the offending paths and bounded allowed
   scope in the kernel inbox. Later refusals say “Refused again:”. Refusals

docs/install.md

@@ -503,6 +503,36 @@ or `scancel --name autoresearch-tick` for the per-cadence chain) and then runs
 `outerloop start`. `start` and the chain refuse a second loop on one root only under the
 current name, `outerloop-resident`.
 
+### Running two instances on one account
+
+Give each instance a separate state root and operator settings file. Keep the
+production settings at `~/.config/outerloop/.env`; select the sandbox file with
+an absolute path in the **process environment** (it cannot select itself):
+
+```bash
+export OUTERLOOP_ENV_FILE="$HOME/.config/outerloop-sandbox/.env"
+outerloop init --root /shared/sandbox-state
+outerloop start
+```
+
+Use the same exported selector for later init, start, limits, and harness status
+commands. The file must be owned by you and not group/world-writable (`chmod 600`
+is recommended). The resident re-reads that file each tick, and successors inherit
+its path. Foreground loops read it once at startup. Use a separate checkout via
+`OUTERLOOP_HOME` if the instances need independent code updates or harness installs.
+
+With `OUTERLOOP_ENV_FILE` unset, or resolving to the default settings path,
+jobs keep the names `outerloop-resident` and `outerloop-tick`, regardless of the
+state root. A different settings file gets `outerloop-resident-<12 hex>` and
+`outerloop-tick-<12 hex>`, using a stable hash of its resolved path. Settings
+file aliases resolve to the same identity. Two instances sharing one settings
+file are not supported. Keep the selected settings path unchanged while its
+chain runs.
+Start's printed `squeue`/`scancel` hints use the instance's name; when inspecting
+or stopping manually, use that exact name. Each root has its own `TICK` lease,
+heartbeat, logs, and `PAUSE` sentinel. Per-user scheduler caps remain shared across
+both instances; separate settings do not increase the account's scheduler limits.
+
 The resident checks its successor every tick and before handover, requeues vanished or terminal jobs with the same dependency, and keeps ticking through the walltime margin if recovery fails.
 
 Experiments run wherever your `compute` backend says. Slurm is the first

scripts/tick_chain.sbatch

@@ -42,6 +42,15 @@
 CADENCE_MIN="${OUTERLOOP_CADENCE_MIN:-30}"
 JOB_NAME="outerloop-tick"
 RESIDENT_JOB_NAME="outerloop-resident"
+# Derive both names from the settings file before any scheduler query/cancellation.
+# The default settings path retains both old names, regardless of state root.
+# Use stdlib Python before deploy has synced dependencies.
+if [ -n "${OUTERLOOP_HOME:-}" ]; then
+    instance_python="$OUTERLOOP_HOME/.venv/bin/python"
+    [ -x "$instance_python" ] || instance_python=python3
+    RESIDENT_JOB_NAME=$("$instance_python" "$OUTERLOOP_HOME/src/outerloop/instance.py") || exit 1
+    JOB_NAME="outerloop-tick${RESIDENT_JOB_NAME#outerloop-resident}"
+fi
 missing=""
 for var in OUTERLOOP_HOME OUTERLOOP_ROOT; do
     eval "val=\${$var:-}"
@@ -105,7 +114,7 @@ while [ "$i" -le "$need" ]; do
     begin=$(date -d "@$begin_epoch" +%Y-%m-%dT%H:%M:%S 2>/dev/null \
             || date -r "$begin_epoch" +%Y-%m-%dT%H:%M:%S)
     for attempt in 1 2 3; do
-        if sbatch --dependency=singleton --begin="$begin" \
+        if sbatch --dependency=singleton --begin="$begin" --job-name="$JOB_NAME" --export=ALL \
                   ${acct_arg:+"$acct_arg"} ${part_arg:+"$part_arg"} ${qos_arg:+"$qos_arg"} \
                   "${OUTERLOOP_HOME:-}/scripts/tick_chain.sbatch"; then
             break

scripts/tick_deploy.sh

@@ -14,7 +14,16 @@
 # Instead single allowlisted keys are read from it, and only when the file is
 # ours and not group/world-writable (a writable one could still inject a
 # malicious VALUE, e.g. a bad codex binary path).
-ENV_FILE="$HOME/.config/outerloop/.env"
+# the selector, trimmed (Python trims it too); empty or blank means the default file
+_sel="${OUTERLOOP_ENV_FILE:-}"
+_sel="${_sel#"${_sel%%[![:space:]]*}"}"; _sel="${_sel%"${_sel##*[![:space:]]}"}"
+ENV_FILE="${_sel:-$HOME/.config/outerloop/.env}"
+case "$ENV_FILE" in
+    /*) ;;
+    *) echo "deploy: OUTERLOOP_ENV_FILE must be an absolute path" >&2
+       export OUTERLOOP_DEPLOY_BROKEN=1
+       return 1 ;;
+esac
 ENV_TRUSTED=""
 if [ -r "$ENV_FILE" ]; then
     # GNU stat first, BSD stat second (a developer's Mac runs this too)

src/outerloop/cli.py

@@ -28,6 +28,7 @@
 from outerloop import paths
 from outerloop.endpoints import author_model_setting, endpoint_config_key
 from outerloop.harness import HARNESS_INSTALL, default_binary
+from outerloop.instance import job_name
 
 if TYPE_CHECKING:
     from outerloop.init import AppPermissionGaps
@@ -96,18 +97,26 @@
 )
 
 
-class StartError(Exception):
+class StartError(paths.ConfigError):
     """A start that cannot proceed; the message is the whole diagnosis."""
 
 
+def operator_env_file(default: Path | None = None) -> Path:
+    try:
+        return paths.env_file(ENV_FILE if default is None else default)
+    except paths.ConfigError as exc:
+        raise StartError(str(exc)) from exc
+
+
 def env_file_values(
-    path: Path = ENV_FILE, keys: tuple[str, ...] | None = START_KEYS
+    path: Path | None = None, keys: tuple[str, ...] | None = START_KEYS
 ) -> dict[str, str]:
     """`keys` from the operator's .env under the deploy step's trust rule: the
     file must be ours and not group/world-writable, or it is refused. Last
     assignment wins; surrounding quotes and a CR are stripped; a key set to
     an empty value is present (an off-switch), an absent key is absent.
     `keys=None` reads all assignments. No file: nothing."""
+    path = operator_env_file() if path is None else path
     try:
         st = path.stat()
     except OSError:
@@ -127,6 +136,8 @@ def env_file_values(
             continue
         key, value = line.split("=", 1)
         key = key.strip()
+        if key == "OUTERLOOP_ENV_FILE":
+            continue
         if (
             keys is not None
             and key not in keys
@@ -184,7 +195,7 @@ def command(self) -> list[str]:
             "--parsable",
             "--dependency=singleton",  # two starts can both submit; only one ever runs
             f"--time={self.resident_minutes}",
-            f"--job-name={RESIDENT_JOB_NAME}",
+            f"--job-name={job_name()}",
         ]
         if self.account:  # unset bills the caller's default Slurm association
             argv.append(f"--account={self.account}")
@@ -280,7 +291,7 @@ def plan_start(
         raise StartError(
             "Slurm mode needs the state root on the shared filesystem: "
             "--root, OUTERLOOP_ROOT in the environment, or OUTERLOOP_ROOT= in "
-            "~/.config/outerloop/.env"
+            f"{operator_env_file()}"
         )
     acc = _setting("OUTERLOOP_ACCOUNT", account, environ, from_file)
     part = _setting("OUTERLOOP_PARTITION", partition, environ, from_file)
@@ -315,7 +326,7 @@ def plan_start(
     return StartPlan(
         mode=mode,
         qos=qos,
-        root=Path(root_s).expanduser(),
+        root=Path(root_s).expanduser().resolve(),
         home=home,
         account=acc,
         partition=part,
@@ -341,7 +352,7 @@ def _resident_jobs() -> list[str] | None:
                 "squeue",
                 "-u",
                 os.environ.get("USER", ""),
-                f"--name={RESIDENT_JOB_NAME}",
+                f"--name={job_name()}",
                 "-h",
                 "-o",
                 "%i",
@@ -522,7 +533,7 @@ def missing_claude_model(values: Mapping[str, str], environ: Mapping[str, str])
         return ""
     return (
         "OUTERLOOP_CLAUDE_MODEL is not set, but this deployment runs Claude roles: "
-        f"{'; '.join(roles)}. Add the line OUTERLOOP_CLAUDE_MODEL=<model> to {ENV_FILE} "
+        f"{'; '.join(roles)}. Add the line OUTERLOOP_CLAUDE_MODEL=<model> to {operator_env_file()} "
         "(or export it in the shell) and start again"
     )
 
@@ -550,7 +561,7 @@ def permissions(args: argparse.Namespace) -> int:
     from outerloop.appmanifest import DEFAULT_PERMISSIONS
 
     try:
-        values = {**env_file_values(ENV_FILE, APP_PERMISSION_KEYS), **os.environ}
+        values = {**env_file_values(operator_env_file(ENV_FILE), APP_PERMISSION_KEYS), **os.environ}
         gaps = _app_gaps_from_env(values)
         if gaps is None:
             if values.get("OUTERLOOP_PAT_FILE", "").strip():
@@ -593,7 +604,9 @@ def permissions(args: argparse.Namespace) -> int:
 
 def start(args: argparse.Namespace) -> int:
     try:
-        values = env_file_values(ENV_FILE, START_KEYS + TICK_ENV_KEYS)  # one read for everything
+        values = env_file_values(
+            operator_env_file(ENV_FILE), START_KEYS + TICK_ENV_KEYS
+        )  # one read for everything
         from outerloop.author_overrides import validate_overrides
 
         try:
@@ -628,10 +641,14 @@ def start(args: argparse.Namespace) -> int:
             sbatch_on_path=shutil.which("sbatch") is not None,
             cwd=Path.cwd(),
         )
-    except StartError as e:
+    except (StartError, ValueError, OSError) as e:
         print(f"outerloop start: {e}", file=sys.stderr)
         return 2
-    cmd = plan.command()
+    try:
+        cmd = plan.command()
+    except (ValueError, OSError) as exc:
+        print(f"outerloop start: {exc}", file=sys.stderr)
+        return 2
     if args.dry_run:
         print(shlex.join(cmd))
         return 0
@@ -710,7 +727,7 @@ def start(args: argparse.Namespace) -> int:
         print(
             "outerloop start: could not ask the scheduler whether a resident tick "
             "exists (squeue failed); nothing submitted. Retry, or check "
-            f"`squeue --name {RESIDENT_JOB_NAME}`.",
+            f"`squeue --name {job_name()}`.",
             file=sys.stderr,
         )
         return 1
@@ -767,7 +784,7 @@ def start(args: argparse.Namespace) -> int:
         f"resident tick submitted: job {job} on {plan.partition}, "
         f"{plan.resident_minutes} min walltime, hands over to itself. "
         f"Logs: {plan.root}/logs. Pause: touch {plan.root}/PAUSE. "
-        f"Stop: scancel --name {RESIDENT_JOB_NAME}."
+        f"Stop: scancel --name {job_name()}."
     )
     return 0
 

src/outerloop/climbboard.py

@@ -890,7 +890,7 @@ def render_html(
 FIXED_JOB_PATTERNS = tuple(
     re.compile(p)
     for p in (
-        r"^outerloop-(resident|tick)$",
+        r"^outerloop-(resident|tick)(-[0-9a-f]{12})?$",
         r"^climb-[\w.-]+-agent-\d+$",
         r"^(steward|climb)-issue-\d+$",
     )

src/outerloop/harness_cli.py

@@ -302,7 +302,7 @@ def main(argv: list[str] | None = None) -> int:
     if args.command == "upgrade" and any(name not in NAMES for name in args.names):
         parser.error("harness names must be claude, codex or hermes")
     try:
-        env_file = paths.ENV_FILE
+        env_file = paths.env_file(paths.ENV_FILE)
         env = {**env_file_values(env_file, keys=CONFIG_KEYS), **os.environ}
         if args.command == "status":
             return status(env)