diff --git a/CHANGELOG.md b/CHANGELOG.md index a2f9322e..4fe1b040 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] +- Support separate instances on one cluster account: process-only absolute + `OUTERLOOP_ENV_FILE`, with the existing ownership/write-permission checks, + and stable settings-path suffixes for resident and per-cadence scheduler jobs. + Init, launch, deploy, harness status, and successor recovery use the selected + settings and instance identity. +- Upgrading: no action needed for the default fleet; its settings path and job + names remain unchanged. Existing run records, leases, and heartbeats need no + migration. Stop additional instances before rolling back to a version without + instance isolation. + - Launch, submit, and stale-submit checkpoint scope violations now refuse every request and resume the author with the offending paths and bounded allowed scope in the kernel inbox. Later refusals say “Refused again:”. Refusals diff --git a/docs/install.md b/docs/install.md index e55daab4..465aad89 100644 --- a/docs/install.md +++ b/docs/install.md @@ -503,6 +503,36 @@ or `scancel --name autoresearch-tick` for the per-cadence chain) and then runs `outerloop start`. `start` and the chain refuse a second loop on one root only under the current name, `outerloop-resident`. +### Running two instances on one account + +Give each instance a separate state root and operator settings file. Keep the +production settings at `~/.config/outerloop/.env`; select the sandbox file with +an absolute path in the **process environment** (it cannot select itself): + +```bash +export OUTERLOOP_ENV_FILE="$HOME/.config/outerloop-sandbox/.env" +outerloop init --root /shared/sandbox-state +outerloop start +``` + +Use the same exported selector for later init, start, limits, and harness status +commands. The file must be owned by you and not group/world-writable (`chmod 600` +is recommended). The resident re-reads that file each tick, and successors inherit +its path. Foreground loops read it once at startup. Use a separate checkout via +`OUTERLOOP_HOME` if the instances need independent code updates or harness installs. + +With `OUTERLOOP_ENV_FILE` unset, or resolving to the default settings path, +jobs keep the names `outerloop-resident` and `outerloop-tick`, regardless of the +state root. A different settings file gets `outerloop-resident-<12 hex>` and +`outerloop-tick-<12 hex>`, using a stable hash of its resolved path. Settings +file aliases resolve to the same identity. Two instances sharing one settings +file are not supported. Keep the selected settings path unchanged while its +chain runs. +Start's printed `squeue`/`scancel` hints use the instance's name; when inspecting +or stopping manually, use that exact name. Each root has its own `TICK` lease, +heartbeat, logs, and `PAUSE` sentinel. Per-user scheduler caps remain shared across +both instances; separate settings do not increase the account's scheduler limits. + The resident checks its successor every tick and before handover, requeues vanished or terminal jobs with the same dependency, and keeps ticking through the walltime margin if recovery fails. Experiments run wherever your `compute` backend says. Slurm is the first diff --git a/scripts/tick_chain.sbatch b/scripts/tick_chain.sbatch index e867bf1c..19f62923 100755 --- a/scripts/tick_chain.sbatch +++ b/scripts/tick_chain.sbatch @@ -42,6 +42,15 @@ CADENCE_MIN="${OUTERLOOP_CADENCE_MIN:-30}" JOB_NAME="outerloop-tick" RESIDENT_JOB_NAME="outerloop-resident" +# Derive both names from the settings file before any scheduler query/cancellation. +# The default settings path retains both old names, regardless of state root. +# Use stdlib Python before deploy has synced dependencies. +if [ -n "${OUTERLOOP_HOME:-}" ]; then + instance_python="$OUTERLOOP_HOME/.venv/bin/python" + [ -x "$instance_python" ] || instance_python=python3 + RESIDENT_JOB_NAME=$("$instance_python" "$OUTERLOOP_HOME/src/outerloop/instance.py") || exit 1 + JOB_NAME="outerloop-tick${RESIDENT_JOB_NAME#outerloop-resident}" +fi missing="" for var in OUTERLOOP_HOME OUTERLOOP_ROOT; do eval "val=\${$var:-}" @@ -105,7 +114,7 @@ while [ "$i" -le "$need" ]; do begin=$(date -d "@$begin_epoch" +%Y-%m-%dT%H:%M:%S 2>/dev/null \ || date -r "$begin_epoch" +%Y-%m-%dT%H:%M:%S) for attempt in 1 2 3; do - if sbatch --dependency=singleton --begin="$begin" \ + if sbatch --dependency=singleton --begin="$begin" --job-name="$JOB_NAME" --export=ALL \ ${acct_arg:+"$acct_arg"} ${part_arg:+"$part_arg"} ${qos_arg:+"$qos_arg"} \ "${OUTERLOOP_HOME:-}/scripts/tick_chain.sbatch"; then break diff --git a/scripts/tick_deploy.sh b/scripts/tick_deploy.sh index 702555bc..9acac2e0 100755 --- a/scripts/tick_deploy.sh +++ b/scripts/tick_deploy.sh @@ -14,7 +14,16 @@ # Instead single allowlisted keys are read from it, and only when the file is # ours and not group/world-writable (a writable one could still inject a # malicious VALUE, e.g. a bad codex binary path). -ENV_FILE="$HOME/.config/outerloop/.env" +# the selector, trimmed (Python trims it too); empty or blank means the default file +_sel="${OUTERLOOP_ENV_FILE:-}" +_sel="${_sel#"${_sel%%[![:space:]]*}"}"; _sel="${_sel%"${_sel##*[![:space:]]}"}" +ENV_FILE="${_sel:-$HOME/.config/outerloop/.env}" +case "$ENV_FILE" in + /*) ;; + *) echo "deploy: OUTERLOOP_ENV_FILE must be an absolute path" >&2 + export OUTERLOOP_DEPLOY_BROKEN=1 + return 1 ;; +esac ENV_TRUSTED="" if [ -r "$ENV_FILE" ]; then # GNU stat first, BSD stat second (a developer's Mac runs this too) diff --git a/src/outerloop/cli.py b/src/outerloop/cli.py index df99a163..36aeddec 100644 --- a/src/outerloop/cli.py +++ b/src/outerloop/cli.py @@ -28,6 +28,7 @@ from outerloop import paths from outerloop.endpoints import author_model_setting, endpoint_config_key from outerloop.harness import HARNESS_INSTALL, default_binary +from outerloop.instance import job_name if TYPE_CHECKING: from outerloop.init import AppPermissionGaps @@ -96,18 +97,26 @@ ) -class StartError(Exception): +class StartError(paths.ConfigError): """A start that cannot proceed; the message is the whole diagnosis.""" +def operator_env_file(default: Path | None = None) -> Path: + try: + return paths.env_file(ENV_FILE if default is None else default) + except paths.ConfigError as exc: + raise StartError(str(exc)) from exc + + def env_file_values( - path: Path = ENV_FILE, keys: tuple[str, ...] | None = START_KEYS + path: Path | None = None, keys: tuple[str, ...] | None = START_KEYS ) -> dict[str, str]: """`keys` from the operator's .env under the deploy step's trust rule: the file must be ours and not group/world-writable, or it is refused. Last assignment wins; surrounding quotes and a CR are stripped; a key set to an empty value is present (an off-switch), an absent key is absent. `keys=None` reads all assignments. No file: nothing.""" + path = operator_env_file() if path is None else path try: st = path.stat() except OSError: @@ -127,6 +136,8 @@ def env_file_values( continue key, value = line.split("=", 1) key = key.strip() + if key == "OUTERLOOP_ENV_FILE": + continue if ( keys is not None and key not in keys @@ -184,7 +195,7 @@ def command(self) -> list[str]: "--parsable", "--dependency=singleton", # two starts can both submit; only one ever runs f"--time={self.resident_minutes}", - f"--job-name={RESIDENT_JOB_NAME}", + f"--job-name={job_name()}", ] if self.account: # unset bills the caller's default Slurm association argv.append(f"--account={self.account}") @@ -280,7 +291,7 @@ def plan_start( raise StartError( "Slurm mode needs the state root on the shared filesystem: " "--root, OUTERLOOP_ROOT in the environment, or OUTERLOOP_ROOT= in " - "~/.config/outerloop/.env" + f"{operator_env_file()}" ) acc = _setting("OUTERLOOP_ACCOUNT", account, environ, from_file) part = _setting("OUTERLOOP_PARTITION", partition, environ, from_file) @@ -315,7 +326,7 @@ def plan_start( return StartPlan( mode=mode, qos=qos, - root=Path(root_s).expanduser(), + root=Path(root_s).expanduser().resolve(), home=home, account=acc, partition=part, @@ -341,7 +352,7 @@ def _resident_jobs() -> list[str] | None: "squeue", "-u", os.environ.get("USER", ""), - f"--name={RESIDENT_JOB_NAME}", + f"--name={job_name()}", "-h", "-o", "%i", @@ -522,7 +533,7 @@ def missing_claude_model(values: Mapping[str, str], environ: Mapping[str, str]) return "" return ( "OUTERLOOP_CLAUDE_MODEL is not set, but this deployment runs Claude roles: " - f"{'; '.join(roles)}. Add the line OUTERLOOP_CLAUDE_MODEL= to {ENV_FILE} " + f"{'; '.join(roles)}. Add the line OUTERLOOP_CLAUDE_MODEL= to {operator_env_file()} " "(or export it in the shell) and start again" ) @@ -550,7 +561,7 @@ def permissions(args: argparse.Namespace) -> int: from outerloop.appmanifest import DEFAULT_PERMISSIONS try: - values = {**env_file_values(ENV_FILE, APP_PERMISSION_KEYS), **os.environ} + values = {**env_file_values(operator_env_file(ENV_FILE), APP_PERMISSION_KEYS), **os.environ} gaps = _app_gaps_from_env(values) if gaps is None: if values.get("OUTERLOOP_PAT_FILE", "").strip(): @@ -593,7 +604,9 @@ def permissions(args: argparse.Namespace) -> int: def start(args: argparse.Namespace) -> int: try: - values = env_file_values(ENV_FILE, START_KEYS + TICK_ENV_KEYS) # one read for everything + values = env_file_values( + operator_env_file(ENV_FILE), START_KEYS + TICK_ENV_KEYS + ) # one read for everything from outerloop.author_overrides import validate_overrides try: @@ -628,10 +641,14 @@ def start(args: argparse.Namespace) -> int: sbatch_on_path=shutil.which("sbatch") is not None, cwd=Path.cwd(), ) - except StartError as e: + except (StartError, ValueError, OSError) as e: print(f"outerloop start: {e}", file=sys.stderr) return 2 - cmd = plan.command() + try: + cmd = plan.command() + except (ValueError, OSError) as exc: + print(f"outerloop start: {exc}", file=sys.stderr) + return 2 if args.dry_run: print(shlex.join(cmd)) return 0 @@ -710,7 +727,7 @@ def start(args: argparse.Namespace) -> int: print( "outerloop start: could not ask the scheduler whether a resident tick " "exists (squeue failed); nothing submitted. Retry, or check " - f"`squeue --name {RESIDENT_JOB_NAME}`.", + f"`squeue --name {job_name()}`.", file=sys.stderr, ) return 1 @@ -767,7 +784,7 @@ def start(args: argparse.Namespace) -> int: f"resident tick submitted: job {job} on {plan.partition}, " f"{plan.resident_minutes} min walltime, hands over to itself. " f"Logs: {plan.root}/logs. Pause: touch {plan.root}/PAUSE. " - f"Stop: scancel --name {RESIDENT_JOB_NAME}." + f"Stop: scancel --name {job_name()}." ) return 0 diff --git a/src/outerloop/climbboard.py b/src/outerloop/climbboard.py index 24fd92c8..24251356 100644 --- a/src/outerloop/climbboard.py +++ b/src/outerloop/climbboard.py @@ -890,7 +890,7 @@ def render_html( FIXED_JOB_PATTERNS = tuple( re.compile(p) for p in ( - r"^outerloop-(resident|tick)$", + r"^outerloop-(resident|tick)(-[0-9a-f]{12})?$", r"^climb-[\w.-]+-agent-\d+$", r"^(steward|climb)-issue-\d+$", ) diff --git a/src/outerloop/harness_cli.py b/src/outerloop/harness_cli.py index d3f27608..0bbae729 100644 --- a/src/outerloop/harness_cli.py +++ b/src/outerloop/harness_cli.py @@ -302,7 +302,7 @@ def main(argv: list[str] | None = None) -> int: if args.command == "upgrade" and any(name not in NAMES for name in args.names): parser.error("harness names must be claude, codex or hermes") try: - env_file = paths.ENV_FILE + env_file = paths.env_file(paths.ENV_FILE) env = {**env_file_values(env_file, keys=CONFIG_KEYS), **os.environ} if args.command == "status": return status(env) diff --git a/src/outerloop/init.py b/src/outerloop/init.py index 40a35678..a7b39619 100644 --- a/src/outerloop/init.py +++ b/src/outerloop/init.py @@ -31,7 +31,7 @@ from pathlib import Path from typing import Any -from outerloop.cli import ENV_FILE, StartError, env_file_values +from outerloop.cli import ENV_FILE, StartError, env_file_values, operator_env_file from outerloop.harness import HARNESS_INSTALL, default_binary from outerloop.hermes_install import hermes_ready from outerloop.image import ensure_image @@ -134,10 +134,16 @@ def render_env( return "\n".join(out).rstrip("\n") + "\n" +def _env_path() -> Path: + return operator_env_file(CONFIG_DIR / ENV_FILE.name) + + def _existing_env() -> str: """The current `.env` text, or "" when there is none (a fresh setup).""" try: - return (CONFIG_DIR / ENV_FILE.name).read_text() + path = _env_path() + env_file_values(path, keys=None) + return path.read_text() except OSError: return "" @@ -271,6 +277,7 @@ def write_config( """Write the PAT file (only when a token is pasted) and the `.env`, both owner-only (0600) — `start`/`tick_deploy` refuse a group/world-readable `.env`, and a token file must never be wider. Returns (env_path, pat_path).""" + config_dir = operator_env_file(config_dir / ENV_FILE.name).parent config_dir.mkdir(parents=True, exist_ok=True) written_pat: Path | None = None if token: @@ -280,7 +287,7 @@ def write_config( write_private(pat_path, token) written_pat = pat_path pat_file = str(pat_path) - env_path = config_dir / ENV_FILE.name + env_path = operator_env_file(config_dir / ENV_FILE.name) write_private(env_path, render_env(a, pat_file)) return env_path, written_pat @@ -477,7 +484,7 @@ def _collect(args: argparse.Namespace, interactive: bool) -> tuple[InitAnswers, pasted token is returned separately to be written 0600.""" preserved: dict[str, str] = {} if args.github_app: - preserved = env_file_values(CONFIG_DIR / ENV_FILE.name, keys=None) + preserved = env_file_values(_env_path(), keys=None) existing = preserved.copy() explicit_image = args.image # Flags, then shell, then the working deployment. Keep unknown keys too. @@ -649,7 +656,7 @@ def _github_app_recheck(answers: InitAnswers, app_json: Path) -> int: problem, fatal = _app_verdict(app_provider_from_file(app_json), answers.target) except Exception as exc: problem, fatal = f"could not read the App credentials: {exc}", False - env_path = CONFIG_DIR / ENV_FILE.name + env_path = _env_path() write_private( env_path, render_env( @@ -677,7 +684,7 @@ def _github_app_setup( Interactive by nature (a browser click + install), so no `--yes` variant.""" from outerloop import appmanifest - existing = sorted(CONFIG_DIR.glob("github_app.*.json")) + existing = sorted(_env_path().parent.glob("github_app.*.json")) if len(existing) == 1: # a re-run after fixing the installation: re-check the App this machine # already has rather than creating a second one @@ -700,7 +707,7 @@ def _github_app_setup( except ValueError as exc: print(f"outerloop init: {exc}", file=sys.stderr) return 1 - pem_path, app_json = appmanifest.save_app_creds(conversion, CONFIG_DIR) + pem_path, app_json = appmanifest.save_app_creds(conversion, _env_path().parent) repo = answers.target.split("/", 1)[-1] print(f" created App '{conversion['slug']}'; credentials in {app_json} and {pem_path} (0600)") mismatch = _app_owner_mismatch_note(conversion, owner, answers.target) @@ -737,7 +744,7 @@ def _github_app_setup( problem, fatal = f"could not read the App credentials: {exc}", False if fatal: write_private( - CONFIG_DIR / ENV_FILE.name, + _env_path(), render_env( answers, app_file=str(app_json), @@ -752,7 +759,7 @@ def _github_app_setup( else: # the credentials are kept; nothing can run until the App is installed write_private( - CONFIG_DIR / ENV_FILE.name, + _env_path(), render_env( answers, app_file=str(app_json), @@ -763,7 +770,7 @@ def _github_app_setup( return _app_failure( answers, str(conversion["slug"]), f"the App is not installed on {answers.target}" ) - env_path = CONFIG_DIR / ENV_FILE.name + env_path = _env_path() write_private( env_path, render_env( @@ -782,6 +789,11 @@ def _github_app_setup( def main(argv: list[str] | None = None) -> int: + try: + env_file_values(_env_path(), keys=None) + except StartError as exc: + print(f"outerloop init: {exc}", file=sys.stderr) + return 2 parser = argparse.ArgumentParser( prog="outerloop init", description="Guided setup: asks for anything not given as a flag, checks the " @@ -859,7 +871,7 @@ def main(argv: list[str] | None = None) -> int: try: effective_overrides = { - **env_file_values(CONFIG_DIR / ENV_FILE.name, keys=None), + **env_file_values(_env_path(), keys=None), **answers.preserved_env, **os.environ, } @@ -906,7 +918,7 @@ def main(argv: list[str] | None = None) -> int: # Never clobber a working setup silently: a re-run of init on a configured # machine must ask (or be told --force). Checked before any App is created. - env_path = CONFIG_DIR / ENV_FILE.name + env_path = _env_path() if env_path.exists() and not args.force: if not interactive: print(f"outerloop init: {env_path} exists; pass --force to overwrite", file=sys.stderr) @@ -938,7 +950,11 @@ def main(argv: list[str] | None = None) -> int: "(hidden; blank to set later): " ).strip() if pasted: - key_path = write_author_key(answers.author_backend, pasted, config_dir=CONFIG_DIR) + key_path = write_author_key( + answers.author_backend, + pasted, + config_dir=_env_path().parent, + ) answers.author_key_file = str(key_path) print(f"wrote {key_path} (0600)") @@ -1088,7 +1104,7 @@ def main(argv: list[str] | None = None) -> int: "Paste a GitHub PAT with write access to the target (hidden; blank to skip): " ).strip() - env_path, pat_path = write_config(answers, token, pat_file, config_dir=CONFIG_DIR) + env_path, pat_path = write_config(answers, token, pat_file, config_dir=_env_path().parent) print(f"wrote {env_path}") if pat_path: print(f"wrote {pat_path} (0600)") @@ -1141,7 +1157,7 @@ def _claude_model_hint(answers: InitAnswers) -> None: if not answers.claude_model: print( " OUTERLOOP_CLAUDE_MODEL not recorded — add OUTERLOOP_CLAUDE_MODEL= to " - f"{CONFIG_DIR / ENV_FILE.name} before `outerloop start` (every Claude role reads it)" + f"{_env_path()} before `outerloop start` (every Claude role reads it)" ) @@ -1149,6 +1165,6 @@ def _author_key_hint(answers: InitAnswers) -> None: if not answers.author_key_file: print( " no author key set — put it in " - f"{author_key_path(answers.author_backend, config_dir=CONFIG_DIR)} " + f"{author_key_path(answers.author_backend, config_dir=_env_path().parent)} " "before the first climb" ) diff --git a/src/outerloop/instance.py b/src/outerloop/instance.py new file mode 100644 index 00000000..efbae82f --- /dev/null +++ b/src/outerloop/instance.py @@ -0,0 +1,30 @@ +"""Stable scheduler identity, also runnable by the batch shim before deploy. + +Only the standard library is used: the checkout need not have been synced yet. +""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + + +def job_name(base: str = "outerloop-resident") -> str: + """Key scheduler identity on the resolved settings path, never the state root.""" + default = Path.home() / ".config/outerloop/.env" + selected = os.environ.get("OUTERLOOP_ENV_FILE", "").strip() + if not selected: # unset or empty: the default instance + return base + path = Path(selected) + if not path.is_absolute(): + raise ValueError("OUTERLOOP_ENV_FILE must be an absolute path") + canonical = path.resolve() + if canonical == default.resolve(): + return base + digest = hashlib.sha256(os.fsencode(canonical)).hexdigest()[:12] + return f"{base}-{digest}" + + +if __name__ == "__main__": + print(job_name()) diff --git a/src/outerloop/paths.py b/src/outerloop/paths.py index 15a2d2a6..82527cbf 100644 --- a/src/outerloop/paths.py +++ b/src/outerloop/paths.py @@ -20,6 +20,21 @@ def config_dir(home: Path | None = None) -> Path: ENV_FILE = CONFIG_DIR / ".env" +class ConfigError(ValueError): + """Invalid operator settings path.""" + + +def env_file(default: Path = ENV_FILE) -> Path: + """Resolve the process-only selector; never read it from settings.""" + value = os.environ.get("OUTERLOOP_ENV_FILE", "").strip() + if not value: # unset or empty: the default settings file + return default + path = Path(value) + if not path.is_absolute(): + raise ConfigError("OUTERLOOP_ENV_FILE must be an absolute path") + return path + + def write_private(path: Path, text: str) -> None: """Write `text` to `path` so no other user can read it at any moment: the file is created (or truncated) with mode 0600 in the same call, and a file diff --git a/tests/test_climbboard.py b/tests/test_climbboard.py index 3eb42592..b3c7e6e2 100644 --- a/tests/test_climbboard.py +++ b/tests/test_climbboard.py @@ -1234,10 +1234,12 @@ def job(id_: str, name: str, state: str, elapsed: str, partition: str) -> dict[s "cpu", ), job("786", "wake-speedrun-20260905-063328-agent-09", "PENDING", "0:00", "cpu"), # not ours + job("787", "outerloop-tick-0123456789ab", "PENDING", "0:00", "cpu"), # a second instance + job("788", "outerloop-resident-mine", "RUNNING", "0:01", "cpu"), # not an instance suffix ] body = collect_status(tmp_path, "org/repo", 3.0, queue=snap) q = {j["id"]: j for j in body["queue"]} - assert set(q) == {"777", "778", "780", "783", "784", "785"} + assert set(q) == {"777", "778", "780", "783", "784", "785", "787"} assert q["777"]["agent"] == "agent-01" and q["777"]["run_id"] == record.run_id assert q["778"]["agent"] == "agent-01" and q["778"]["run_id"] == record.run_id assert q["780"]["agent"] == "" and q["785"]["agent"] == "agent-04" diff --git a/tests/test_init.py b/tests/test_init.py index de1ddf2e..62644e6f 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -1229,3 +1229,33 @@ def test_init_bad_override_json_fails_before_provisioning(tmp_path, monkeypatch, monkeypatch.setattr(init, "ensure_image", lambda **k: pytest.fail("downloaded")) assert init.main(["--yes", "--compute", "local", "--target", "owner/repo"]) == 2 assert capsys.readouterr().err.count("outerloop init:") == 1 + + +def test_init_selects_separate_settings_and_credentials(tmp_path, monkeypatch): + selected = tmp_path / "sandbox" / "operator.env" + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(selected)) + monkeypatch.setattr(init, "CONFIG_DIR", tmp_path / "production") + monkeypatch.setattr(init, "validate_pat", lambda pf, t: "") + assert ( + init.main( + [ + "--yes", + "--compute", + "local", + "--target", + "owner/sandbox", + "--pat-file", + "/token", + "--claude-model", + "claude-test", + ] + ) + == 0 + ) + assert "OUTERLOOP_TARGET=owner/sandbox" in selected.read_text() + assert not (tmp_path / "production").exists() + # Direct writes put pasted credentials beside the selected settings too. + _, pat = write_config(InitAnswers(compute="local", target="owner/sandbox"), "fake", "") + assert pat == selected.parent / "bot_pat" + selected.chmod(0o622) + assert init.main(["--yes", "--force"]) == 2 diff --git a/tests/test_instance.py b/tests/test_instance.py new file mode 100644 index 00000000..dd06b9ad --- /dev/null +++ b/tests/test_instance.py @@ -0,0 +1,96 @@ +"""Scheduler identity and legacy production compatibility.""" + +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from outerloop import cli +from outerloop.instance import job_name +from outerloop.runstate import acquire_tick_lease, release_tick_lease, tick_lease_holder +from outerloop.tick import write_heartbeat + + +@pytest.mark.parametrize("base", ["outerloop-resident", "outerloop-tick"]) +def test_legacy_settings_without_root_keep_names(tmp_path, monkeypatch, base): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.delenv("OUTERLOOP_ENV_FILE", raising=False) + config = tmp_path / ".config/outerloop/.env" + config.parent.mkdir(parents=True) + # Legacy settings: the root was supplied at start and inherited by the job. + config.write_text("OUTERLOOP_TARGET=owner/repo\n") + config.chmod(0o600) + monkeypatch.setenv("OUTERLOOP_ROOT", str(tmp_path / "production")) + for _ in range(3): + assert job_name(base) == base + run = Mock(return_value=Mock(returncode=0, stdout="12\n")) + monkeypatch.setattr(cli.subprocess, "run", run) + assert cli._resident_jobs() == ["12"] + assert "--name=outerloop-resident" in run.call_args.args[0] + + +def test_selected_settings_have_stable_names(tmp_path, monkeypatch): + import hashlib + import os + + monkeypatch.setenv("HOME", str(tmp_path)) + selected = tmp_path / "sandbox.env" + selected.touch() + alias = tmp_path / "alias.env" + alias.symlink_to(selected) + digest = hashlib.sha256(os.fsencode(selected.resolve())).hexdigest()[:12] + for path in [selected, alias, selected]: + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(path)) + for root in ["first", "second"]: + monkeypatch.setenv("OUTERLOOP_ROOT", str(tmp_path / root)) + for base in ["outerloop-resident", "outerloop-tick"]: + assert job_name(base) == f"{base}-{digest}" + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(tmp_path / "other.env")) + assert job_name() != f"outerloop-resident-{digest}" + + +@pytest.mark.parametrize("alias", [False, True]) +def test_resolved_default_settings_keep_names(tmp_path, monkeypatch, alias): + monkeypatch.setenv("HOME", str(tmp_path)) + default = tmp_path / ".config/outerloop/.env" + default.parent.mkdir(parents=True) + default.touch() + selected = default.parent / ".." / "outerloop" / ".env" + if alias: + selected = tmp_path / "alias.env" + selected.symlink_to(default) + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(selected)) + monkeypatch.setenv("OUTERLOOP_ROOT", str(tmp_path / "other-root")) + assert job_name() == "outerloop-resident" + assert job_name("outerloop-tick") == "outerloop-tick" + + +def test_lease_and_heartbeat_are_per_root(tmp_path: Path): + first, second = tmp_path / "first", tmp_path / "second" + a = acquire_tick_lease(first, "host:1", 100, 30, settle_s=0) + b = acquire_tick_lease(second, "host:2", 100, 30, settle_s=0) + try: + assert tick_lease_holder(first, 100, 30, "host") == "host:1" + assert tick_lease_holder(second, 100, 30, "host") == "host:2" + write_heartbeat(first, 100) + assert not (second / "heartbeat.json").exists() + with pytest.raises(RuntimeError): + acquire_tick_lease(first, "host:3", 100, 30, settle_s=0) + finally: + release_tick_lease(a) + release_tick_lease(b) + + +@pytest.mark.parametrize("value", ["", " "]) +def test_empty_settings_selector_is_default_instance(monkeypatch, value): + # An empty selector must never fail the chain: it means the default instance. + monkeypatch.setenv("OUTERLOOP_ENV_FILE", value) + assert job_name() == "outerloop-resident" + assert job_name("outerloop-tick") == "outerloop-tick" + + +def test_empty_settings_selector_reads_default_file(monkeypatch, tmp_path): + from outerloop import paths + + monkeypatch.setenv("OUTERLOOP_ENV_FILE", "") + assert paths.env_file(tmp_path / "default.env") == tmp_path / "default.env" diff --git a/tests/test_paths.py b/tests/test_paths.py index 95ecdc1f..898f9c87 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -4,6 +4,8 @@ from pathlib import Path +import pytest + from outerloop.paths import CONFIG_DIR_NAME, config_dir @@ -20,4 +22,55 @@ def test_existing_config_dir(tmp_path: Path) -> None: def test_deploy_script_config_dir() -> None: sh = (Path(__file__).resolve().parents[1] / "scripts" / "tick_deploy.sh").read_text() - assert 'ENV_FILE="$HOME/.config/outerloop/.env"' in sh + assert 'ENV_FILE="${_sel:-$HOME/.config/outerloop/.env}"' in sh + + +def test_env_file_default_and_process_override(monkeypatch, tmp_path): + from outerloop import paths + + monkeypatch.delenv("OUTERLOOP_ENV_FILE", raising=False) + assert paths.env_file() == paths.CONFIG_DIR / ".env" + selected = tmp_path / "settings.env" + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(selected)) + assert paths.env_file() == selected + + +def test_harness_status_reads_selected_settings(monkeypatch, tmp_path): + from outerloop import harness_cli + + selected = tmp_path / "settings.env" + selected.write_text("OUTERLOOP_CLAUDE_BIN=/sandbox/claude\n") + selected.chmod(0o600) + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(selected)) + monkeypatch.delenv("OUTERLOOP_CLAUDE_BIN", raising=False) + seen = {} + + def status(env): + seen.update(env) + return 0 + + monkeypatch.setattr(harness_cli, "status", status) + assert harness_cli.main(["status"]) == 0 + assert seen["OUTERLOOP_CLAUDE_BIN"] == "/sandbox/claude" + + +@pytest.mark.parametrize( + ("value", "expected"), + [("", "DEFAULT"), (" ", "DEFAULT"), ("/x/y.env", "/x/y.env"), (" /x/y.env ", "/x/y.env")], +) +def test_deploy_selector_is_trimmed_like_python(value, expected): + # Run the deploy script's own selector lines: a blank selector must mean the default + # file (as in Python), never a failed deploy. + import subprocess + + sh = (Path(__file__).parents[1] / "scripts" / "tick_deploy.sh").read_text().splitlines() + start = next(i for i, line in enumerate(sh) if line.startswith("_sel=")) + snippet = "\n".join(sh[start : start + 2]) + '\necho "${_sel:-DEFAULT}"\n' + out = subprocess.run( + ["bash", "-c", snippet], + env={"OUTERLOOP_ENV_FILE": value, "PATH": "/usr/bin:/bin"}, + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.strip() == expected diff --git a/tests/test_start.py b/tests/test_start.py index e64224c8..bde41090 100644 --- a/tests/test_start.py +++ b/tests/test_start.py @@ -17,7 +17,6 @@ from outerloop.cli import ( DEFAULT_LOCAL_ROOT, DEFAULT_RESIDENT_MINUTES, - RESIDENT_JOB_NAME, START_KEYS, TICK_ENV_KEYS, StartError, @@ -26,6 +25,7 @@ main, plan_start, ) +from outerloop.instance import job_name REPO = Path(__file__).resolve().parents[1] @@ -194,7 +194,7 @@ def test_slurm_composes_the_resident_submit(tmp_path: Path) -> None: "--parsable", "--dependency=singleton", f"--time={DEFAULT_RESIDENT_MINUTES}", - f"--job-name={RESIDENT_JOB_NAME}", + f"--job-name={job_name()}", "--account=pr_1_general", "--partition=cpu_short", "--export=ALL", @@ -442,7 +442,7 @@ def test_slurm_start_submits_once_and_reports( out = capsys.readouterr().out assert "job 4242" in out and "cpu_short" in out and "PAUSE" in out argv = log.read_text().split("\n") - assert argv[0] == "--parsable" and f"--job-name={RESIDENT_JOB_NAME}" in argv + assert argv[0] == "--parsable" and f"--job-name={job_name()}" in argv assert "--dependency=singleton" in argv assert "--export=ALL" in argv # the knobs ride the inherited env, asserted next assert envlog.read_text() == f"1:{home}" # export_env reached sbatch's environment @@ -1122,3 +1122,71 @@ def capture_exec(cmd, env): path.write_text(path.read_text() + "OUTERLOOP_CLAUDE_MODEL=claude-x\n") assert main(["start", "--local"]) == 0 assert exports[-1]["OUTERLOOP_STEWARD_KEY_FILE"] == "/keys/steward" + + +def test_start_reads_process_env_file_and_exports_it(clean_env, monkeypatch): + selected = env_file(clean_env, "OUTERLOOP_ROOT=/sandbox\nOUTERLOOP_TARGET=owner/repo\n") + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(selected)) + monkeypatch.chdir(checkout(clean_env)) + captured = {} + + def execute(cmd, env): + captured.update(env) + return 0 + + monkeypatch.setattr(cli, "_exec", execute) + assert main(["start", "--local"]) == 0 + assert captured["OUTERLOOP_ENV_FILE"] == str(selected) + assert captured["OUTERLOOP_ROOT"] == "/sandbox" + assert captured["OUTERLOOP_TARGET"] == "owner/repo" + + +@pytest.mark.parametrize("mode", [0o620, 0o602]) +def test_start_override_keeps_trust_checks(clean_env, monkeypatch, capsys, mode): + selected = env_file(clean_env, "OUTERLOOP_ROOT=/sandbox\n", mode) + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(selected)) + assert main(["start", "--local", "--dry-run"]) == 2 + assert "refusing to read" in capsys.readouterr().err + + +def test_env_file_selector_is_process_only(clean_env, monkeypatch): + selected = env_file(clean_env, "OUTERLOOP_ENV_FILE=/other\nOUTERLOOP_ROOT=/sandbox\n") + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(selected)) + assert env_file_values(keys=None) == {"OUTERLOOP_ROOT": "/sandbox"} + monkeypatch.setenv("OUTERLOOP_ENV_FILE", "relative.env") + assert main(["start", "--local", "--dry-run"]) == 2 + + +def test_start_slurm_scopes_lookup_hints_and_inherited_selector(clean_env, monkeypatch, capsys): + from types import SimpleNamespace + + selected = env_file(clean_env, f"OUTERLOOP_ROOT={clean_env}/sandbox\n") + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(selected)) + monkeypatch.chdir(checkout(clean_env)) + monkeypatch.setattr(cli.shutil, "which", lambda name: "/bin/" + name) + calls = [] + + def run(cmd, **kwargs): + calls.append((cmd, kwargs)) + return SimpleNamespace(returncode=0, stdout="42" if cmd[0] == "sbatch" else "", stderr="") + + monkeypatch.setattr(cli.subprocess, "run", run) + assert main(["start"]) == 0 + name = job_name() + assert f"Stop: scancel --name {name}." in capsys.readouterr().out + for cmd, kwargs in calls: + if cmd[0] == "squeue": + assert f"--name={name}" in cmd + if cmd[0] == "sbatch": + assert f"--job-name={name}" in cmd + assert kwargs["env"]["OUTERLOOP_ENV_FILE"] == str(selected) + + +def test_override_rejects_another_owner(clean_env, monkeypatch): + from types import SimpleNamespace + + selected = env_file(clean_env, "OUTERLOOP_ROOT=/sandbox\n") + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(selected)) + monkeypatch.setattr(Path, "stat", lambda self: SimpleNamespace(st_uid=os.getuid() + 1)) + with pytest.raises(StartError, match="owned by you"): + env_file_values() diff --git a/tests/test_tick_resident.py b/tests/test_tick_resident.py index c3e2cddc..91279ff7 100644 --- a/tests/test_tick_resident.py +++ b/tests/test_tick_resident.py @@ -25,6 +25,12 @@ def _install(tmp_path: Path) -> tuple[Path, Path, Path, Path]: shutil.copy(f, home / "scripts" / f.name) root = tmp_path / "root" root.mkdir() + (home / "src/outerloop").mkdir(parents=True) + shutil.copy(ROOT / "src/outerloop/instance.py", home / "src/outerloop/instance.py") + config = home / ".config/outerloop/.env" + config.parent.mkdir(parents=True, exist_ok=True) + config.write_text(f"OUTERLOOP_ROOT={root}\n") + config.chmod(0o600) bindir = tmp_path / "bin" bindir.mkdir() shimlog = tmp_path / "shimlog" @@ -76,6 +82,7 @@ def _env(home: Path, root: Path, bindir: Path, **extra: str) -> dict[str, str]: } env.pop("OUTERLOOP_PAT_FILE", None) env.pop("OUTERLOOP_RESIDENT", None) + env.pop("OUTERLOOP_ENV_FILE", None) env.update(extra) return env @@ -821,7 +828,7 @@ def test_resident_recovery_at_margin(tmp_path: Path, submit_fails: bool) -> None def test_deploy_harness_upgrade_is_best_effort(tmp_path, mode): home, root, bindir, shimlog = _install(tmp_path) config = home / ".config/outerloop/.env" - config.parent.mkdir(parents=True) + config.parent.mkdir(parents=True, exist_ok=True) config.write_text("OUTERLOOP_CLAUDE_BIN=/old/claude\nOUTERLOOP_PANEL=\n") config.chmod(0o600) (bindir / "uv").write_text(f'''#!/bin/sh @@ -870,3 +877,105 @@ def test_deploy_configured_cache_precedes_sync(tmp_path, explicit, inherited_def ) assert proc.returncode == 0, proc.stderr assert set((tmp_path / "uv-caches").read_text().splitlines()) == {str(chosen)} + + +@pytest.mark.parametrize("mode", [0o600, 0o620, 0o602]) +def test_deploy_env_override_trust(tmp_path, mode): + selected = tmp_path / "sandbox.env" + selected.write_text("OUTERLOOP_TARGET=owner/sandbox\n") + selected.chmod(mode) + header = (ROOT / "scripts/tick_deploy.sh").read_text().split("# --- 2. deploy:")[0] + proc = subprocess.run( + ["bash", "-c", header + '\n_k=OUTERLOOP_TARGET; env_line; env_value; echo "VALUE=$_v"'], + env={**os.environ, "OUTERLOOP_ENV_FILE": str(selected)}, + capture_output=True, + text=True, + ) + assert proc.returncode == 0 + assert ("VALUE=owner/sandbox" in proc.stdout) == (mode == 0o600) + assert ("refusing to read" in proc.stdout) == (mode != 0o600) + + +def test_instance_resubmit_preserves_settings_and_does_not_drain_production(tmp_path, monkeypatch): + from outerloop.instance import job_name + + home, root, bindir, shimlog = _install(tmp_path) + # Production's legacy root differs from this sandbox. + (home / ".config/outerloop/.env").write_text("OUTERLOOP_ROOT=/production\n") + selected = tmp_path / "sandbox.env" + selected.write_text("OUTERLOOP_TARGET=owner/sandbox\nOUTERLOOP_ENV_FILE=/ignored\n") + selected.chmod(0o600) + sbatch = bindir / "sbatch" + sbatch.write_text( + sbatch.read_text().replace( + 'echo "$@"', f'echo "$OUTERLOOP_ENV_FILE" >> "{shimlog}/env-files"\necho "$@"' + ) + ) + (bindir / "squeue").write_text(f'''#!/bin/sh +echo "$*" >> "{shimlog}/squeue" +case "$*" in + *"--name=outerloop-tick "*) echo 999 ;; + *"-j "*) echo PENDING ;; +esac +''') + uv = bindir / "uv" + uv.write_text( + uv.read_text().replace( + 'echo "tick $(date +%s)"', + f'echo "$OUTERLOOP_TARGET" >> "{shimlog}/targets"\necho "tick $(date +%s)"', + ) + ) + proc = _run_chain( + home, + _env( + home, + root, + bindir, + OUTERLOOP_ENV_FILE=str(selected), + OUTERLOOP_RESIDENT="1", + OUTERLOOP_RESIDENT_CADENCE_S="1", + SLURM_JOB_ID="42", + ), + ) + assert proc.returncode == 0, proc.stderr + submissions = (shimlog / "sbatch").read_text().splitlines() + assert len(submissions) == 2 # initial successor, then shim-change recovery + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(selected)) + assert all(f"--job-name={job_name()}" in line for line in submissions) + assert all("--export=ALL" in line for line in submissions) + assert (shimlog / "env-files").read_text().splitlines() == [str(selected)] * 2 + assert (shimlog / "targets").read_text().splitlines() == ["owner/sandbox"] * 3 + assert "999" not in (shimlog / "scancel").read_text().splitlines() + assert "--name=outerloop-tick-" in (shimlog / "squeue").read_text() + + +def test_legacy_settings_without_root_chain_keeps_names(tmp_path): + home, root, bindir, shimlog = _install(tmp_path) + (home / ".config/outerloop/.env").write_text("OUTERLOOP_TARGET=owner/repo\n") + (bindir / "squeue").write_text(f'#!/bin/sh\necho "$*" >> "{shimlog}/squeue"\n') + proc = _run_chain(home, _env(home, root, bindir)) + assert proc.returncode == 0, proc.stderr + queries = (shimlog / "squeue").read_text().splitlines() + assert any("--name=outerloop-resident" in line.split() for line in queries) + assert any("--name=outerloop-tick" in line.split() for line in queries) + submissions = (shimlog / "sbatch").read_text().splitlines() + assert len(submissions) == 2 + assert all("--job-name=outerloop-tick" in line.split() for line in submissions) + + +def test_per_cadence_resubmit_preserves_settings(tmp_path, monkeypatch): + from outerloop.instance import job_name + + home, root, bindir, shimlog = _install(tmp_path) + selected = tmp_path / "selected.env" + selected.write_text("OUTERLOOP_TARGET=owner/repo\n") + sbatch = bindir / "sbatch" + sbatch.write_text(sbatch.read_text() + f'echo "$OUTERLOOP_ENV_FILE" >> "{shimlog}/env-files"\n') + proc = _run_chain(home, _env(home, root, bindir, OUTERLOOP_ENV_FILE=str(selected))) + assert proc.returncode == 0, proc.stderr + monkeypatch.setenv("OUTERLOOP_ENV_FILE", str(selected)) + submissions = (shimlog / "sbatch").read_text().splitlines() + assert len(submissions) == 2 + assert all(f"--job-name={job_name('outerloop-tick')}" in line.split() for line in submissions) + assert all("--export=ALL" in line.split() for line in submissions) + assert (shimlog / "env-files").read_text().splitlines() == [str(selected)] * 2