diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b74e267..21e9f009 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,21 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] +- Deployment author overrides select a backend/model per target or agent slot, + bind it to each run, and leave panel and CI reviewer inheritance on the fleet + author. Per-run board details identify the effective author and overrides. +- Endpoint profiles accept an absolute `URL_FILE` instead of `URL`, reading bare + URLs or JSON addresses on every session. Missing files, failed bounded health + checks, and interrupted checks defer sessions without spending wake retries. + Init provisions fleet and override backends before validating prerequisites. +- Upgrading: no action or backfill needed; records without `author_overridden` + retain their existing author and panel behavior, including ended runs. New + overridden records reuse the saved author route and add this optional flag. + Rollback reads the records but loses fleet-only panel inheritance for overrides; + finish overridden runs and fresh endpoint capacity parks before rolling back. + Fresh endpoint deferrals reuse author-sleep capacity parks without a session ID; + older kernels cannot resume those parks. + - Launch ledger submissions require dispatched launch job IDs, preventing stale checkpoints from attributing discarded launches to a commit. Gate capacity waits still record any dispatched sibling launches. Existing ledger rows and diff --git a/docs/endpoints.md b/docs/endpoints.md index ebd72a64..0410f026 100644 --- a/docs/endpoints.md +++ b/docs/endpoints.md @@ -163,3 +163,26 @@ fallback. That requires the pinned clients under network observation. Legacy records with no model use native model defaults only. A missing native Codex model fails preflight; it never adopts an endpoint. No backfill is needed. + +## Server addresses that change + +Instead of `OUTERLOOP_ENDPOINT_ONPREM_URL`, set +`OUTERLOOP_ENDPOINT_ONPREM_URL_FILE=/absolute/path/server-address`. +Exactly one of `URL` and `URL_FILE` must be configured for a profile. The file +contains either a bare URL (a trailing newline is fine) or a JSON object: + +```json +{"url": "http://localhost:8000/v1"} +``` + +Both forms use the same HTTP(S) base URL validation as `URL`: no credentials, +query string or fragment. Publish updates by atomic rename. The kernel reads +this file outside the session sandbox each time a session starts, including +resumes and wakes; an already constructed harness does not cache its value. + +A missing or unreadable address defers intake and parked wakes without spending +wake retries. At session time the file is read once, then a single models-list +request checks the server with a three-second timeout and the profile key in an +Authorization header. Missing files, dead servers, and interrupted checks park +fresh runs and defer wakes without consuming a wake retry. There is no polling. Malformed contents are configuration errors. Profile names, served model, +API capabilities and credential paths retain their existing semantics. diff --git a/docs/install.md b/docs/install.md index 11009867..e55daab4 100644 --- a/docs/install.md +++ b/docs/install.md @@ -541,6 +541,31 @@ default, and a lens that names no model runs the author's model when it shares the author's backend, and must name an explicit model on any other backend); the author backend is `OUTERLOOP_AUTHOR_BACKEND`/`OUTERLOOP_AUTHOR_MODEL`. + +`OUTERLOOP_AUTHOR_OVERRIDES` optionally selects an author for individual targets +and agent slots, without changing judges or other targets: + +```sh +OUTERLOOP_AUTHOR_OVERRIDES='{"owner/repo":{"backend":"claude","model":"served-model[endpoint=onprem]","slots":["agent-05"]}}' +``` + +Each entry requires `backend` (`claude`, `codex`, or `hermes`) and `model`. +Omit `slots` to cover every author slot on that target; otherwise use the existing +`agent-01`, `agent-02`, … identities allocated by the contract's authors-abreast +width. This is deployment configuration, not a contract setting. The setting is +parsed and validated at startup. Endpoint overrides select their own profile in +`model`; they do not inherit `OUTERLOOP_AUTHOR_ENDPOINT`. Native overrides use +the selected backend's author credential. Normal author/judge credential +separation still applies to the effective override credential. + +Queued climbs bind the selection when submitted (before an intake claim is +launched); direct climbs bind at startup. Backend, model and key path are saved +in the run record. Changing overrides cannot switch an existing run, even at a +resume or wake. Panel inheritance and CI reviewers still use the fleet author, +exactly as for a run without an override. Per-run board details show the author +backend/model and mark overrides. Remove the setting (or use `{}`) to stop +selecting overrides for new work. + `OUTERLOOP_CLAUDE_MODEL` names the model for every Claude role (author, panel judges, steward) when no explicit or inherited model covers that role: there is no built-in default, and `start` refuses when a role needs it, naming diff --git a/scripts/tick_deploy.sh b/scripts/tick_deploy.sh index c7ed1b2c..702555bc 100755 --- a/scripts/tick_deploy.sh +++ b/scripts/tick_deploy.sh @@ -139,13 +139,14 @@ if [ -n "$ENV_TRUSTED" ]; then env_line env_value export "$_k=$_v" - done < <(sed -nE 's/^(OUTERLOOP_ENDPOINT_[A-Z][A-Z0-9_]*_(URL|KEY_FILE|MODEL|API))=.*/\1/p' "$ENV_FILE" | sort -u) + done < <(sed -nE 's/^(OUTERLOOP_ENDPOINT_[A-Z][A-Z0-9_]*_(URL|URL_FILE|KEY_FILE|MODEL|API))=.*/\1/p' "$ENV_FILE" | sort -u) for _k in OUTERLOOP_CLAUDE_VERSION OUTERLOOP_CODEX_VERSION \ OUTERLOOP_CLAUDE_SHA256 OUTERLOOP_CODEX_SHA256 \ OUTERLOOP_HERMES_REF OUTERLOOP_HERMES_SHA \ OUTERLOOP_CACHE_ROOT REVIEW_BACKEND \ OUTERLOOP_AUTHOR_ENDPOINT REVIEW_ENDPOINT REVIEW_MODEL \ - OUTERLOOP_AUTHOR_BACKEND OUTERLOOP_AUTHOR_MODEL OUTERLOOP_CLAUDE_MODEL \ + OUTERLOOP_AUTHOR_BACKEND OUTERLOOP_AUTHOR_MODEL OUTERLOOP_AUTHOR_OVERRIDES \ + OUTERLOOP_CLAUDE_MODEL \ OUTERLOOP_CLAUDE_BIN OUTERLOOP_CODEX_BIN OUTERLOOP_CODEX_KEY_FILE \ OUTERLOOP_HERMES_KEY_FILE OUTERLOOP_HERMES_RESUME_MAX_CHARS \ OUTERLOOP_CLAUDE_KEY_FILE OUTERLOOP_STEWARD_KEY_FILE \ diff --git a/src/outerloop/attempt.py b/src/outerloop/attempt.py index 0f182aa9..0231ed39 100644 --- a/src/outerloop/attempt.py +++ b/src/outerloop/attempt.py @@ -45,7 +45,13 @@ should_dispatch, snapshot_tree, ) -from outerloop.endpoints import author_model_setting, model_key, resolve_endpoint, split_endpoint +from outerloop.endpoints import ( + EndpointUnavailable, + author_model_setting, + model_key, + resolve_endpoint, + split_endpoint, +) from outerloop.evalcache import seed_dir from outerloop.github import ( GitError, @@ -273,6 +279,27 @@ def fleet_author_model(backend: str) -> str: return model +def defer_endpoint_wake(root: Path, record: RunRecord) -> bool: + """Keep the existing park and refund this delivery when its server is down.""" + _, profile = resolve_endpoint(record.author_model, record.author_backend or "claude") + try: + if profile: + _ = profile.url + except EndpointUnavailable as exc: + _defer_endpoint(root, record, exc) + return True + return False + + +def _defer_endpoint(root: Path, record: RunRecord, exc: EndpointUnavailable) -> None: + save_record( + root, + dc_replace(record, state=PARKED, wake_attempts=max(0, record.wake_attempts - 1)), + time.time(), + ) + log.warning("run %s: %s", record.run_id, exc) + + def resume_author( record: object, fleet_model: str, @@ -1328,6 +1355,8 @@ def post_leg_replies(messages: tuple[dict, ...]) -> None: kwargs.setdefault("scope_validator", steward_out_of_scope) kwargs.setdefault("ruler", RULER) + if not record.resume_session_id: + kwargs.setdefault("task_hypothesis", str(record.stage.get("hypothesis") or "")) result = attempt_once( config, contract_text, @@ -1502,14 +1531,15 @@ def _end(result: AttemptResult, drop_refs: list[str]) -> AttemptOutcome: drop_snapshot(ws, Snapshot(commit="", tree="", ref=ref)) return outcome - # The wake NEEDS the author harness (it resumes the session). Fail as a + # A capacity park before the first session starts with a fresh brief. + # Other wakes NEED the author harness and saved session. Fail as a # named ending, not a crash: the run cannot proceed and re-waking will not # help without the harness, so leaving it PARKED would just hit the stuck # cap slowly. if ( harness is None or spec is None - or not record.resume_session_id + or (not record.resume_session_id and not record.stage.get("capacity_wait")) or not getattr(harness, "supports_resume", True) ): return _end( @@ -1740,6 +1770,10 @@ def changed_paths() -> list[str]: if sleep_ref: drop_snapshot(ws, Snapshot(commit="", tree="", ref=sleep_ref)) return AttemptOutcome(run_id=run_id, outcome="parked") + except EndpointUnavailable as exc: + latest = load_record(run_root, run_id) + _defer_endpoint(run_root, latest, exc) + return AttemptOutcome(run_id=run_id, outcome="parked", pr_url=latest.pr_url) except ResumeContextBlocked as exc: latest = load_record(run_root, run_id) save_record( @@ -3238,7 +3272,11 @@ def _panel_lenses_from_args( author_backend = getattr(args, "author_backend", "") or "claude" if author_model is None: author_model = getattr(args, "model", "") or "" - parsed = resolve_lenses(args.panel, author_backend, author_model) + panel_backend, panel_model = author_backend, author_model + if getattr(args, "author_overridden", False): + panel_backend = os.environ.get("OUTERLOOP_AUTHOR_BACKEND") or "claude" + panel_model = fleet_author_model(panel_backend) + parsed = resolve_lenses(args.panel, panel_backend, panel_model) author_credential = effective_author_credential( author_backend, author_model, getattr(args, "key_file", "") ) @@ -4264,6 +4302,7 @@ def live_attempt( author_backend: str = "claude", author_model: str = "", author_key_file: str = "", + author_overridden: bool = False, task_hypothesis: str = "", spec: RoleSpec | None = None, panel_lenses: tuple[PanelLens, ...] = (), @@ -4293,8 +4332,10 @@ def live_attempt( issue_number=issue_number, author_backend=author_backend, author_model=author_model, + author_overridden=author_overridden, author_key_file=author_key_file, run_job_id=_os.environ.get("SLURM_JOB_ID", ""), + stage={"hypothesis": redact(task_hypothesis, secrets)} if task_hypothesis else {}, ) try: save_record(run_root, record, now) @@ -4632,6 +4673,32 @@ def acknowledge(seq: int) -> None: ), tree_of=lambda sha: ws.git("rev-parse", f"{sha}^{{tree}}").strip(), ) + except EndpointUnavailable as exc: + # Reuse a jobless capacity park; its first wake starts the author. + sha = snapshot() + kept_ref = snapshots[-1].ref + p = RunParked( + phase="author-sleep", + afterany="", + base_sha=pre_session_sha, + seed=0, + suite_seed=0, + candidate_sha=sha, + capacity_wait=True, + ) + _park_run( + run_root, + record, + p, + kept_ref, + eval_minutes, + time.time(), + secrets, + base_branch=base_branch, + ) + parked = p + log.warning("run %s: %s", run_id, exc) + return AttemptOutcome(run_id=run_id, outcome="parked") except RunParked as p: # The climb dispatched its measures and hibernated. Persist the # re-entry stage as a PARKED record (not an error), keep the @@ -5001,7 +5068,19 @@ def _run_id(value: str) -> str: parser.add_argument( "--hypothesis-b64", default="", help="base64 task hypothesis (issue text, fenced)" ) + parser.add_argument("--author-bound", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--author-overridden", action="store_true", help=argparse.SUPPRESS) args = parser.parse_args() + from outerloop.author_overrides import select_override + + try: + if not args.resume and not args.author_bound: + selected = select_override(args.target, args.agent_id) + if selected: + args.author_backend, args.model = selected.backend, selected.resolved_model() + args.author_overridden = True + except ValueError as exc: + parser.error(str(exc)) logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") if not args.model and not args.resume: # resolved after parsing, never at parser build: a deployment without @@ -5079,6 +5158,17 @@ def _run_id(value: str) -> str: # reap (the resume_run finally below only runs once we reach it) _release_own_lease(args.run_root, args.resume) parser.error(f"parked run {args.resume}: {_err}") + try: + deferred = isinstance(_wake_record, RunRecord) and defer_endpoint_wake( + args.run_root, _wake_record + ) + except ValueError as exc: + _release_own_lease(args.run_root, args.resume) + parser.error(f"parked run {args.resume}: {exc}") + if deferred: + _release_own_lease(args.run_root, args.resume) + return 0 + args.author_overridden = bool(getattr(_wake_record, "author_overridden", False)) args.key_file = wake_key_file # the wake runs the SAME verification panel as a fresh climb, so a # dispatched improvement is not published unverified. @@ -5182,10 +5272,11 @@ def _run_id(value: str) -> str: if not (args.target and args.benchmark): parser.error("--target and --benchmark are required for a fresh climb") - # a fresh climb authors on the FLEET's configured backend; validate it (codex - # writes+executes, so --image + a non-claude model) before any spend. + # Validate the selected author before spending; an override's endpoint + # selector is independent of the fleet endpoint. try: - args.model = author_model_setting(args.author_backend, args.model) + if not args.author_overridden and not args.author_bound: + args.model = author_model_setting(args.author_backend, args.model) except ValueError as exc: parser.error(str(exc)) _err = author_config_error(args.author_backend, args.model, args.image) @@ -5294,6 +5385,7 @@ def _run_id(value: str) -> str: issue_number=args.issue, author_backend=args.author_backend, author_model=args.model, + author_overridden=args.author_overridden, author_key_file=args.key_file, task_hypothesis=( __import__("base64").b64decode(args.hypothesis_b64).decode() diff --git a/src/outerloop/author_overrides.py b/src/outerloop/author_overrides.py new file mode 100644 index 00000000..e5c9ec6f --- /dev/null +++ b/src/outerloop/author_overrides.py @@ -0,0 +1,97 @@ +"""Deployment-owned author selection; contracts only allocate agent identities.""" + +from __future__ import annotations + +import json +import os +import re +from collections.abc import Mapping +from dataclasses import dataclass +from functools import lru_cache +from types import MappingProxyType + +SETTING = "OUTERLOOP_AUTHOR_OVERRIDES" + + +@dataclass(frozen=True) +class AuthorOverride: + backend: str + model: str + slots: tuple[str, ...] | None = None + + def resolved_model(self) -> str: + """Bind profile model defaults without inheriting the fleet endpoint.""" + from outerloop.endpoints import resolve_endpoint + + model, profile = resolve_endpoint(self.model, self.backend) + return f"{model}[endpoint={profile.name}]" if profile else model + + def matches(self, agent_id: str) -> bool: + return self.slots is None or agent_id in self.slots + + +@lru_cache(maxsize=16) +def parse_overrides(raw: str) -> Mapping[str, AuthorOverride]: + """Parse once per setting value, including in long-lived tick processes.""" + try: + data = json.loads(raw) if raw.strip() else {} + if not isinstance(data, dict): + raise ValueError("must be a JSON object") + result = {} + for target, value in data.items(): + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", target): + raise ValueError(f"invalid target {target!r}; expected owner/repo") + if not isinstance(value, dict) or set(value) - {"backend", "model", "slots"}: + raise ValueError(f"{target}: expected backend, model and optional slots") + backend, model, slots = value.get("backend"), value.get("model"), value.get("slots") + if backend not in ("claude", "codex", "hermes"): + raise ValueError(f"{target}: backend must be claude, codex or hermes") + if ( + not isinstance(model, str) + or not model.strip() + or model != model.strip() + or any(c in model for c in "\r\n\x00") + ): + raise ValueError( + f"{target}: model must be a nonempty string without outer whitespace" + ) + from outerloop.endpoints import split_endpoint + + split_endpoint(model) + if "slots" in value and ( + not isinstance(slots, list) + or not slots + or any( + not isinstance(s, str) + or not re.fullmatch(r"agent-\d{2,}", s) + or int(s[6:]) < 1 + or s != f"agent-{int(s[6:]):02d}" + for s in slots + ) + or len(set(slots)) != len(slots) + ): + raise ValueError( + f"{target}: slots must be distinct agent identities (agent-01, ...)" + ) + result[target] = AuthorOverride(backend, model, None if slots is None else tuple(slots)) + return MappingProxyType(result) + except (ValueError, TypeError) as exc: + raise ValueError(f"{SETTING}: {exc}") from exc + + +def overrides(environ: Mapping[str, str] | None = None) -> Mapping[str, AuthorOverride]: + return parse_overrides((os.environ if environ is None else environ).get(SETTING, "")) + + +def select_override(target: str, agent_id: str) -> AuthorOverride | None: + selected = overrides().get(target) + return selected if selected and selected.matches(agent_id) else None + + +def validate_overrides(environ: Mapping[str, str], image: str) -> None: + from outerloop.attempt import author_config_error + + for target, selected in overrides(environ).items(): + error = author_config_error(selected.backend, selected.model, image, environ=environ) + if error: + raise ValueError(f"{SETTING}: {target}: {error}") diff --git a/src/outerloop/cli.py b/src/outerloop/cli.py index 9199f244..df99a163 100644 --- a/src/outerloop/cli.py +++ b/src/outerloop/cli.py @@ -65,6 +65,7 @@ "REVIEW_MODEL", "OUTERLOOP_AUTHOR_BACKEND", "OUTERLOOP_AUTHOR_MODEL", + "OUTERLOOP_AUTHOR_OVERRIDES", "OUTERLOOP_CLAUDE_MODEL", "OUTERLOOP_CLAUDE_BIN", "OUTERLOOP_CODEX_BIN", @@ -404,13 +405,25 @@ def _setting_of(key: str, values: Mapping[str, str], environ: Mapping[str, str]) def missing_harness_binary(values: Mapping[str, str], environ: Mapping[str, str]) -> str: - """Check only the configured author's host CLI, using the harness's lookup.""" + """Check all configured authors' host CLIs using the harness's lookup.""" + from outerloop.author_overrides import overrides + + env = {**values, **environ} backend = _setting_of("OUTERLOOP_AUTHOR_BACKEND", values, environ).lower() or "claude" + backends = dict.fromkeys([backend, *(value.backend for value in overrides(env).values())]) + for selected in backends: + problem = _missing_author_binary(selected, env) + if problem: + return problem + return "" + + +def _missing_author_binary(backend: str, env: Mapping[str, str]) -> str: key = f"OUTERLOOP_{backend.upper()}_BIN" if backend == "hermes": from outerloop.hermes_install import hermes_ready - repo = _setting_of("REVIEW_HERMES_REPO", values, environ) + repo = env.get("REVIEW_HERMES_REPO", "") return ( "" if repo and hermes_ready(Path(repo).expanduser()) @@ -418,7 +431,6 @@ def missing_harness_binary(values: Mapping[str, str], environ: Mapping[str, str] ) if key not in HARNESS_BIN_KEYS: return f"unsupported author backend {backend!r}; choose claude, codex or hermes." - env = {**values, **environ} recorded = env.get(key, "") binary = default_binary(backend, env) if (not recorded and not os.path.isabs(binary)) or not ( @@ -582,6 +594,14 @@ def permissions(args: argparse.Namespace) -> int: def start(args: argparse.Namespace) -> int: try: values = env_file_values(ENV_FILE, START_KEYS + TICK_ENV_KEYS) # one read for everything + from outerloop.author_overrides import validate_overrides + + try: + validate_overrides( + {**values, **os.environ}, _setting_of("OUTERLOOP_IMAGE", values, os.environ) + ) + except ValueError as exc: + raise StartError(str(exc)) from exc problem = "" if args.dry_run else missing_harness_binary(values, os.environ) try: author_model_setting( diff --git a/src/outerloop/climbboard.py b/src/outerloop/climbboard.py index 75fe094a..24fd92c8 100644 --- a/src/outerloop/climbboard.py +++ b/src/outerloop/climbboard.py @@ -741,6 +741,12 @@ def render_html( " a.textContent = 'PR'; top.append(a);\n" " }\n" " card.append(top);\n" + " if (r.author_model) {\n" + " const a = document.createElement('span');\n" + " a.textContent = r.author_backend + ' / ' + r.author_model\n" + " + (r.author_overridden ? ' (override)' : '');\n" + " a.style.display = 'block'; card.append(a);\n" + " }\n" " if (r.direction) {\n" " const d = document.createElement('span'); d.className = 'dir';\n" " d.textContent = r.direction; d.style.display = 'block';\n" @@ -1105,6 +1111,9 @@ def collect_status( { "run_id": record.run_id, "agent": record.agent_id, + "author_backend": record.author_backend or "claude", + "author_model": record.author_model, + "author_overridden": record.author_overridden, "benchmark": record.benchmark, "state": record.state, "phase": "configuration-blocked" if blocked else stage.get("phase", ""), @@ -1181,6 +1190,9 @@ def service_status( # edits the contract — all real transitions the strip must show keys = ( "run_id", + "author_backend", + "author_model", + "author_overridden", "state", "phase", "waiting", diff --git a/src/outerloop/endpoints.py b/src/outerloop/endpoints.py index 75182549..834c7caf 100644 --- a/src/outerloop/endpoints.py +++ b/src/outerloop/endpoints.py @@ -2,16 +2,18 @@ from __future__ import annotations +import json import os import re from collections.abc import Mapping from dataclasses import dataclass +from http.client import HTTPConnection, HTTPException, HTTPSConnection from pathlib import Path from urllib.parse import urlsplit from outerloop.github import FileTokenProvider -PROFILE_KEY = re.compile(r"OUTERLOOP_ENDPOINT_[A-Z][A-Z0-9_]*_(URL|KEY_FILE|MODEL|API)\Z") +PROFILE_KEY = re.compile(r"OUTERLOOP_ENDPOINT_[A-Z][A-Z0-9_]*_(URL|URL_FILE|KEY_FILE|MODEL|API)\Z") PROFILE_NAME = re.compile(r"[A-Za-z][A-Za-z0-9_]*\Z") @@ -29,14 +31,92 @@ def split_endpoint(model: str) -> tuple[str, str]: return match[1], match[2].lower() +class EndpointUnavailable(Exception): + """A configured server address is temporarily unavailable.""" + + +def validate_url(value: str, name: str) -> str: + url = urlsplit(value) + if ( + url.scheme not in ("http", "https") + or not url.hostname + or url.username + or url.password + or url.query + or url.fragment + or any(c.isspace() for c in value) + ): + raise ValueError( + f"endpoint {name!r}: URL must be an HTTP(S) base URL " + "without credentials, query or fragment" + ) + try: + url.port # noqa: B018 -- raises on a malformed or out-of-range port + except ValueError as exc: + raise ValueError(f"endpoint {name!r}: URL has an invalid port") from exc + return value + + @dataclass(frozen=True) class EndpointProfile: name: str - url: str + fixed_url: str key_file: Path model: str apis: tuple[str, ...] + url_file: Path | None = None + + @property + def url(self) -> str: + if self.url_file is None: + return self.fixed_url + try: + value = self.url_file.read_text().strip() + except OSError as exc: + raise EndpointUnavailable( + f"endpoint {self.name!r}: URL_FILE {self.url_file} unavailable; " + "waiting for server address" + ) from exc + if value.startswith("{"): + try: + value = json.loads(value)["url"] + except (ValueError, KeyError, TypeError) as exc: + raise ValueError( + f"endpoint {self.name!r}: URL_FILE must contain a URL or JSON with a url key" + ) from exc + if not isinstance(value, str): + raise ValueError(f"endpoint {self.name!r}: URL_FILE url must be a string") + return validate_url(value, self.name) + + def session_url(self) -> str: + """Resolve once and probe a dynamic server once, without session retries.""" + from outerloop.attempt import Terminated + + connection: HTTPConnection | None = None + try: + value = self.url + if self.url_file is None: + return value + url = urlsplit(value) + connection_type = HTTPSConnection if url.scheme == "https" else HTTPConnection + connection = connection_type(url.hostname or "", url.port, timeout=3) + path = url.path.rstrip("/") + if not path.endswith("/v1"): + path += "/v1" + connection.request( + "GET", path + "/models", headers={"Authorization": f"Bearer {self.key()}"} + ) + response = connection.getresponse() + if response.status != 200: + raise EndpointUnavailable(f"endpoint {self.name!r}: models request failed") + return value + except (OSError, HTTPException, Terminated, KeyboardInterrupt) as exc: + raise EndpointUnavailable(f"endpoint {self.name!r}: server unavailable") from exc + finally: + if connection is not None: + connection.close() + def key(self) -> str: try: return FileTokenProvider(self.key_file).token() @@ -55,12 +135,20 @@ def endpoint_profile( prefix = f"OUTERLOOP_ENDPOINT_{name.upper()}_" values = { suffix: env.get(prefix + suffix, "").strip() - for suffix in ("URL", "KEY_FILE", "MODEL", "API") + for suffix in ("URL", "URL_FILE", "KEY_FILE", "MODEL", "API") } if not any(values.values()): raise ValueError(f"unknown endpoint profile {name!r}") + if bool(values["URL"]) == bool(values["URL_FILE"]): + raise ValueError( + f"endpoint {name!r}: missing or conflicting URL; " + f"exactly one of {prefix}URL and {prefix}URL_FILE is required" + ) + url_file = Path(values["URL_FILE"]) if values["URL_FILE"] else None + if url_file is not None and not url_file.is_absolute(): + raise ValueError(f"endpoint {name!r}: URL_FILE must be absolute") for suffix, value in values.items(): - if not value: + if suffix not in ("URL", "URL_FILE") and not value: raise ValueError(f"endpoint {name!r}: missing {prefix}{suffix}") apis = tuple(part.strip() for part in values["API"].split(",")) required = {"claude": "anthropic", "codex": "responses", "hermes": "chat"}[backend] @@ -68,19 +156,8 @@ def endpoint_profile( raise ValueError(f"endpoint {name!r}: API must list anthropic, responses, or chat") if required not in apis: raise ValueError(f"endpoint {name!r}: {backend} requires API {required}") - url = urlsplit(values["URL"]) - if ( - url.scheme not in ("http", "https") - or not url.hostname - or url.username - or url.password - or url.query - or url.fragment - ): - raise ValueError( - f"endpoint {name!r}: URL must be an HTTP(S) base URL " - "without credentials, query or fragment" - ) + if values["URL"]: + validate_url(values["URL"], name) path = Path(values["KEY_FILE"]).expanduser() if not path.is_absolute(): raise ValueError(f"endpoint {name!r}: KEY_FILE must be absolute") @@ -94,7 +171,7 @@ def endpoint_profile( raise ValueError( f"endpoint {name!r}: model {model!r} does not match served model {values['MODEL']!r}" ) - profile = EndpointProfile(name.lower(), values["URL"], path, values["MODEL"], apis) + profile = EndpointProfile(name.lower(), values["URL"], path, values["MODEL"], apis, url_file) profile.key() # Validate before any session or intake claim. return profile diff --git a/src/outerloop/harness.py b/src/outerloop/harness.py index 97a702f6..8e732943 100644 --- a/src/outerloop/harness.py +++ b/src/outerloop/harness.py @@ -731,7 +731,9 @@ def run( { # Claude Code appends /v1/messages itself; profiles use the # OpenAI-style base, so drop a trailing /v1 - "ANTHROPIC_BASE_URL": self.endpoint.url.rstrip("/").removesuffix("/v1"), + "ANTHROPIC_BASE_URL": self.endpoint.session_url() + .rstrip("/") + .removesuffix("/v1"), "CLAUDE_CODE_USE_VERTEX": "0", "CLAUDE_CODE_USE_BEDROCK": "0", "CLAUDE_CODE_USE_FOUNDRY": "0", @@ -1193,7 +1195,7 @@ def run( 'model_provider = "outerloop_endpoint"\n' "[model_providers.outerloop_endpoint]\n" 'name = "Outerloop endpoint"\n' - f"base_url = {json.dumps(self.endpoint.url)}\n" + f"base_url = {json.dumps(self.endpoint.session_url())}\n" 'env_key = "OUTERLOOP_SESSION_KEY"\n' 'wire_api = "responses"\n' "requires_openai_auth = false\n" @@ -1509,7 +1511,7 @@ def run( config_lines.append( "custom_providers:\n" f" - name: {json.dumps(self.provider)}\n" - f" base_url: {json.dumps(self.endpoint.url)}\n" + f" base_url: {json.dumps(self.endpoint.session_url())}\n" f" key_env: {json.dumps(self.key_env)}\n" " api_mode: chat_completions\n" ) diff --git a/src/outerloop/harness_cli.py b/src/outerloop/harness_cli.py index 53cdb113..d3f27608 100644 --- a/src/outerloop/harness_cli.py +++ b/src/outerloop/harness_cli.py @@ -85,6 +85,7 @@ def identity(name: str, desired: Mapping[str, str]) -> str: "OUTERLOOP_CODEX_BIN", "REVIEW_HERMES_REPO", "OUTERLOOP_AUTHOR_BACKEND", + "OUTERLOOP_AUTHOR_OVERRIDES", "OUTERLOOP_STEWARD_KEY_FILE", "REVIEW_BACKEND", "OUTERLOOP_PANEL", @@ -165,7 +166,9 @@ def status(env: Mapping[str, str]) -> int: def used_harnesses(env: Mapping[str, str]) -> list[str]: author = env.get("OUTERLOOP_AUTHOR_BACKEND") or "claude" - used = {author} + from outerloop.author_overrides import overrides + + used = {author, *(selected.backend for selected in overrides(env).values())} if env.get("OUTERLOOP_STEWARD_KEY_FILE"): used.add("claude") if env.get("REVIEW_BACKEND"): diff --git a/src/outerloop/init.py b/src/outerloop/init.py index e49afb56..40a35678 100644 --- a/src/outerloop/init.py +++ b/src/outerloop/init.py @@ -855,6 +855,19 @@ def main(argv: list[str] | None = None) -> int: except StartError as exc: print(f"outerloop init: {exc}", file=sys.stderr) return 2 + from outerloop.author_overrides import overrides, validate_overrides + + try: + effective_overrides = { + **env_file_values(CONFIG_DIR / ENV_FILE.name, keys=None), + **answers.preserved_env, + **os.environ, + } + selected_authors = overrides(effective_overrides) + except (ValueError, StartError) as exc: + print(f"outerloop init: {exc}", file=sys.stderr) + return 2 + if answers.author_backend == "hermes": from outerloop.harness import hermes_resume_max_chars @@ -940,12 +953,26 @@ def main(argv: list[str] | None = None) -> int: settings = env_file_values(env_path, keys=None) judge_keys = ( "OUTERLOOP_PANEL", + "OUTERLOOP_AUTHOR_OVERRIDES", "REVIEW_BACKEND", "REVIEW_MODEL", "REVIEW_HERMES_PROVIDER", "REVIEW_HERMES_REPO", ) - for key in judge_keys: + from outerloop.endpoints import endpoint_config_key + + deployment_env = effective_overrides + override_keys = { + key + for selected in selected_authors.values() + for key in ( + f"OUTERLOOP_{selected.backend.upper()}_KEY_FILE", + f"OUTERLOOP_{selected.backend.upper()}_BIN", + ) + } + if selected_authors: + override_keys.update(key for key in deployment_env if endpoint_config_key(key)) + for key in (*judge_keys, *sorted(override_keys)): if key in os.environ: settings[key] = os.environ[key] if key in settings: @@ -955,11 +982,25 @@ def main(argv: list[str] | None = None) -> int: try: panel = settings.get("OUTERLOOP_PANEL", "") lenses = parse_lenses(panel, answers.author_backend) if panel.strip() else () - needs_hermes = settings.get("REVIEW_BACKEND", "").lower() == "hermes" or any( - backend == "hermes" for _, backend, _ in lenses - ) + for backend in sorted({a.backend for a in selected_authors.values()}): + if backend in (answers.author_backend or AUTHOR_BACKENDS[0], "hermes"): + continue + binary = locate_harness(backend) + if cli_install_wanted(binary, args): + binary = install_harness(backend) + if binary: + answers.preserved_env[author_bin_env(backend)] = binary + needs_hermes = ( + answers.author_backend == "hermes" + or any(a.backend == "hermes" for a in selected_authors.values()) + or settings.get("REVIEW_BACKEND", "").lower() == "hermes" + ) or any(backend == "hermes" for _, backend, _ in lenses) if needs_hermes: - repo = Path(settings.get("REVIEW_HERMES_REPO") or Path.home() / "hermes-agent") + repo = Path( + (answers.author_bin if answers.author_backend == "hermes" else "") + or settings.get("REVIEW_HERMES_REPO") + or Path.home() / "hermes-agent" + ) repo = repo.expanduser().resolve() if cli_install_wanted(str(repo) if hermes_ready(repo) else "", args): install_harness("hermes", target=repo) @@ -968,8 +1009,8 @@ def main(argv: list[str] | None = None) -> int: print(f"outerloop init: {exc}", file=sys.stderr) return 1 - # The image, only now: every check that could still end the run has passed - # and the overwrite question is answered, so a download is never wasted. + # Provision the image before checking backend prerequisites; the override + # syntax and overwrite checks have already passed. # An existing image is used, else the published one is fetched on a machine # that can run it (asked first when interactive); --no-image opts out. if not answers.image and not args.no_image: @@ -977,6 +1018,15 @@ def main(argv: list[str] | None = None) -> int: # compute nodes, which the login node cannot speak for answers.image = ensure_image(interactive=interactive, probe=(answers.compute == "local")) + try: + validate_overrides( + {**effective_overrides, **answers.preserved_env}, + answers.image or effective_overrides.get("OUTERLOOP_IMAGE", ""), + ) + except ValueError as exc: + print(f"outerloop init: {exc}", file=sys.stderr) + return 2 + if answers.author_backend == "hermes": from outerloop.attempt import author_config_error from outerloop.endpoints import author_model_setting diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index 41af2e07..a2a57d13 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -1510,10 +1510,30 @@ def _resume(message: Message) -> AttemptResult | None: # session started with (a wake refreshed it too; this covers a refusal) with contextlib.suppress(Exception): refresh_tool(workspace) - with _watched(): - wake_result = run_role( - spec, harness, prompt, workspace, resume_session_id=session.session_id - ) + from outerloop.endpoints import EndpointUnavailable + + try: + with _watched(): + wake_result = run_role( + spec, harness, prompt, workspace, resume_session_id=session.session_id + ) + except EndpointUnavailable: + # A prior leg already ran: keep its native context and budget meters. + raise RunParked( + phase="author-sleep", + afterany="", + base_sha=base_sha, + seed=run_seed, + suite_seed=suite_seed, + candidate_sha=snapshot(), + session=session, + syscall=SyscallRequest(launches=()), + launches_used=launches_used, + sleeps_used=sleeps_used, + gpu_hours_used=gpu_hours_used, + judged=failed_gate, + capacity_wait=True, + ) from None session = wake_result.session if wake_result.ok: _ack(messages) diff --git a/src/outerloop/runstate.py b/src/outerloop/runstate.py index 7e5203ad..086c2e2a 100644 --- a/src/outerloop/runstate.py +++ b/src/outerloop/runstate.py @@ -137,6 +137,7 @@ class RunRecord: # the harness strips it only when constructing the backend session. author_backend: str = "" author_model: str = "" + author_overridden: bool = False # judges inherit the fleet author for this run # The resolved author key FILE PATH (not the key) this run used, so a wake or # follow-up reproduces the exact key — an explicit --key-file survives, and an # in-flight run is immune to a later env change. "" = resolve per backend @@ -264,6 +265,8 @@ def _save_record(root: Path, record: RunRecord, now: float) -> None: # same tmp file before the atomic replace tmp = directory / f".{RECORD_NAME}.{os.getpid()}.tmp" payload = asdict(stamped) + if not stamped.author_overridden: + payload.pop("author_overridden") # No-setting records retain their exact wire shape. path = directory / RECORD_NAME if path.exists(): old = json.loads(path.read_text()) diff --git a/src/outerloop/tick.py b/src/outerloop/tick.py index f6e7f581..b6a75038 100644 --- a/src/outerloop/tick.py +++ b/src/outerloop/tick.py @@ -1679,6 +1679,19 @@ def _sweep_one( deferred.append(record.run_id) return + from outerloop.endpoints import EndpointUnavailable, resolve_endpoint + + try: + _, profile = resolve_endpoint(record.author_model, record.author_backend or "claude") + if profile: + _ = profile.url + except EndpointUnavailable as exc: + log.warning("run %s: %s", record.run_id, exc) + deferred.append(record.run_id) + return + except ValueError: + pass # Existing configuration validation reports permanent errors. + # Layer 5: too many failed attempts is a terminal, reported state. if record.wake_attempts >= MAX_WAKE_ATTEMPTS: if not dry_run: @@ -2415,23 +2428,54 @@ def _climb_panel_argv(spec: ServiceSpec) -> list[str]: return argv -def _author_config_error(spec: ServiceSpec) -> str: - """Why the config-driven author would die at the climb's startup ("" when it - won't), checked on the tick host BEFORE a claim/submit so a codex misconfig - (e.g. OUTERLOOP_AUTHOR_BACKEND=codex with no non-claude model) never - strands a claimed intake issue. Reads the fleet author config from env — the - same source the climb defaults from — and the image the tick already knows.""" - from outerloop.attempt import author_config_error, fleet_author_model +def _selected_author(spec: ServiceSpec, agent_id: str = "agent-01") -> tuple[str, str]: + from outerloop.attempt import fleet_author_model + from outerloop.author_overrides import select_override + selected = select_override(spec.target, agent_id) + if selected: + return selected.backend, selected.resolved_model() backend = os.environ.get("OUTERLOOP_AUTHOR_BACKEND") or "claude" + return backend, fleet_author_model(backend) + + +def _climb_author_argv(spec: ServiceSpec, agent_id: str = "agent-01") -> list[str]: + from outerloop.attempt import effective_author_credential + from outerloop.author_overrides import overrides, select_override + + if not overrides(): + return [] + backend, model = _selected_author(spec, agent_id) + credential = effective_author_credential(backend, model) + return [ + "--author-bound", + "--author-backend", + backend, + "--model", + model, + "--key-file", + str(credential.key_file), + *(["--author-overridden"] if select_override(spec.target, agent_id) else []), + ] + + +def _author_config_error(spec: ServiceSpec, agent_id: str = "agent-01") -> str: + from outerloop.attempt import author_config_error + from outerloop.endpoints import EndpointUnavailable, resolve_endpoint + try: - model = fleet_author_model(backend) - except (ClaudeModelUnset, ValueError) as exc: + backend, model = _selected_author(spec, agent_id) + _, profile = resolve_endpoint(model, backend) + if profile: + _ = profile.url # Readiness before claim: a missing address never spends an attempt. + return author_config_error(backend, model, spec.image) + except (ClaudeModelUnset, ValueError, EndpointUnavailable) as exc: return str(exc) - return author_config_error(backend, model, spec.image) -def _panel_preflight_error(spec: ServiceSpec) -> str: +def _panel_preflight_error( + spec: ServiceSpec, agent_id: str = "agent-01", *, record: RunRecord | None = None +) -> str: """Why the climb would die at startup on this panel config ("" when it won't): the lens spec, then the key file — each checked with the climb's OWN rules (resolve_lenses for grammar and author/model inheritance; @@ -2465,15 +2509,23 @@ def _panel_preflight_error(spec: ServiceSpec) -> str: ) except ValueError as exc: return str(exc) - from outerloop.attempt import fleet_author_model from outerloop.endpoints import resolve_endpoint from outerloop.harness import hermes_resume_max_chars if any(backend == "hermes" for _, backend, _ in lenses): hermes_resume_max_chars() - author_backend = os.environ.get("OUTERLOOP_AUTHOR_BACKEND") or "claude" + author_key_file = "" + if record is None: + author_backend, author_model = _selected_author(spec, agent_id) + else: + from outerloop.attempt import fleet_author_model, resume_author + + fleet_backend = os.environ.get("OUTERLOOP_AUTHOR_BACKEND") or "claude" + author_backend, author_model, author_key_file = resume_author( + record, fleet_author_model(fleet_backend), fleet_backend + ) author_credential = effective_author_credential( - author_backend, fleet_author_model(author_backend) + author_backend, author_model, author_key_file ) author_path = author_credential.key_file traditional = [] @@ -2484,7 +2536,6 @@ def _panel_preflight_error(spec: ServiceSpec) -> str: continue if not spec.image or not Path(spec.image).is_file(): return f"a {backend} endpoint panel lens requires a real container image" - author_backend = os.environ.get("OUTERLOOP_AUTHOR_BACKEND") or "claude" from outerloop.endpoints import validate_judge_key_file validate_judge_key_file( @@ -2778,7 +2829,7 @@ def service_self_initiated( if lane_error := _gpu_lane_error(contract, benchmark, spec): log.error("attempt on %s not launched: %s", benchmark, lane_error) return None - author_error = _author_config_error(spec) + author_error = _author_config_error(spec, slot_agent) if author_error: log.error( "climb on %s not launched: author misconfigured — %s " @@ -2787,7 +2838,7 @@ def service_self_initiated( author_error, ) return None - panel_error = _panel_preflight_error(spec) + panel_error = _panel_preflight_error(spec, slot_agent) if panel_error: log.error( "climb on %s not launched: panel misconfigured — %s " @@ -2814,12 +2865,12 @@ def service_self_initiated( slot_agent, *_climb_limit_argv(limits, job_minutes), *_climb_panel_argv(spec), + *_climb_author_argv(spec, slot_agent), ] if spec.pat_file: argv += ["--pat-file", spec.pat_file] - # config-driven author: climb resolves the author backend/model/key from - # OUTERLOOP_AUTHOR_* env (inherited by the job), so the tick threads - # neither the backend nor its key — a new backend needs zero tick change. + # With overrides configured the launch args bind the effective author; + # otherwise preserve the existing fleet-driven job command. job_id = submit( root, spec.target, @@ -3077,6 +3128,7 @@ def service_intake( if dry_run: return (f"issue-{task.number}", "dry-run") job_minutes = _attempt_job_minutes(spec, limits) + author_argv = _climb_author_argv(spec) # claim BEFORE submit: Slurm queueing can take minutes, and the next # tick must not re-claim the same issue in that window from outerloop.intake import CLAIM_MARKER, MAX_INTAKE_ATTEMPTS, RELEASE_MARKER @@ -3107,11 +3159,11 @@ def service_intake( hypothesis_b64, *_climb_limit_argv(limits, job_minutes), *_climb_panel_argv(spec), + *author_argv, ] if spec.pat_file: argv += ["--pat-file", spec.pat_file] - # config-driven author: climb resolves the author key from the - # OUTERLOOP_AUTHOR_* env by backend; the tick does not thread it. + # The selected author was bound before claiming the issue. try: job_id = submit( root, @@ -3201,7 +3253,9 @@ def dispatch(self, record: RunRecord, reason: str) -> str: "--max-turns", str(self.spec.max_turns), ] - panel_skip = _panel_preflight_error(self.spec) if self.spec.panel.strip() else "" + panel_skip = ( + _panel_preflight_error(self.spec, record=record) if self.spec.panel.strip() else "" + ) if panel_skip: argv += ["--panel-skip", panel_skip] # An AUTHOR-SLEEP wake resumes a FULL author session (not the short @@ -3518,6 +3572,12 @@ def main() -> int: "OUTERLOOP_CADENCE_MIN via the chain's own parser (default 30)", ) args = parser.parse_args() + from outerloop.author_overrides import validate_overrides + + try: + validate_overrides(os.environ, os.environ.get("OUTERLOOP_IMAGE", "")) + except ValueError as exc: + parser.error(str(exc)) logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") args.root.mkdir(parents=True, exist_ok=True) diff --git a/tests/test_attempt.py b/tests/test_attempt.py index c5ad91b9..6db48cfc 100644 --- a/tests/test_attempt.py +++ b/tests/test_attempt.py @@ -874,6 +874,7 @@ def run_live( author_key_file="", eval_image="", submit=False, + task_hypothesis="", ) -> tuple: github = FakeGitHub() queue = list(values) @@ -892,6 +893,7 @@ def run_live( author_backend=author_backend, author_model=author_model, author_key_file=author_key_file, + task_hypothesis=task_hypothesis, eval_image=eval_image, ) return outcome, github @@ -7357,3 +7359,103 @@ def successful_resume(self, brief, workspace, resume_session_id=None): for _ in range(2): sweep() assert not woke + + +@pytest.mark.parametrize("wake", [False, True]) +@pytest.mark.parametrize("failure", ["missing", "dead", "term", "interrupt"]) +def test_endpoint_unavailable_parks_and_recovers( + tmp_path, target_repo_syscalls, monkeypatch, wake, failure +): + from dataclasses import replace + from unittest.mock import Mock + + from outerloop.endpoints import EndpointProfile + from outerloop.roles import author_spec + + root = tmp_path / "state" + dispatch = _fake_dispatch() + if wake: + outcome, _ = run_live( + tmp_path, + target_repo_syscalls, + edits={".outerloop/syscall.json": json.dumps({"type": "sleep", "launches": []})}, + values=[], + dispatch=dispatch, + ) + assert outcome.outcome == "parked" + before = load_record(root, "tsp-1") + save_record(root, replace(before, wake_attempts=3), 1_000_001) + + address = tmp_path / "address" + key = tmp_path / "key" + key.write_text("secret") + key.chmod(0o600) + endpoint = EndpointProfile("local", "", key, "model", ("anthropic",), address) + if failure != "missing": + address.write_text("http://localhost:8000/v1") + connection = Mock() + connection.request.side_effect = { + "dead": ConnectionRefusedError(), + "term": climb_mod.Terminated(), + "interrupt": KeyboardInterrupt(), + "missing": None, + }[failure] + monkeypatch.setattr("outerloop.endpoints.HTTPConnection", Mock(return_value=connection)) + original = ScriptedHarness.run + seen_briefs = [] + + def unavailable(self, *args, **kwargs): + endpoint.session_url() + pytest.fail("author started") + + monkeypatch.setattr(ScriptedHarness, "run", unavailable) + + def resume(): + return resume_run( + root, + "tsp-1", + dispatch=dispatch, + github=CommentingGitHub(), # type: ignore[arg-type] + bot_auth=NoAuth(), + now=1_000_100, + harness=ScriptedHarness(edits={".outerloop/syscall.json": json.dumps({"type": "end"})}), + spec=author_spec(), + ) + + if wake: + outcome = resume() + else: + outcome, _ = run_live( + tmp_path, + target_repo_syscalls, + edits={}, + values=[], + dispatch=dispatch, + task_hypothesis="try a new move", + ) + assert outcome.outcome == "parked" + waiting = load_record(root, "tsp-1") + assert waiting.state == "parked" and not waiting.ending + assert waiting.wake_attempts == (2 if wake else 0) + if wake: + assert waiting.stage == before.stage + assert waiting.resume_session_id == before.resume_session_id + else: + assert waiting.stage["capacity_wait"] and not waiting.resume_session_id + assert _git(root / "runs/tsp-1/ws", "rev-parse", str(waiting.stage["candidate_ref"])).strip() + + # A repeated unavailable wake refunds exactly its own delivery, including + # the fresh run which has never had a native session id. + save_record(root, replace(waiting, wake_attempts=waiting.wake_attempts + 1), 1_000_101) + assert resume().outcome == "parked" + assert load_record(root, "tsp-1").wake_attempts == waiting.wake_attempts + + def recovered(self, brief, *args, **kwargs): + seen_briefs.append(brief) + return original(self, brief, *args, **kwargs) + + monkeypatch.setattr(ScriptedHarness, "run", recovered) + assert resume().outcome != "parked" + if not wake: + assert "try a new move" in seen_briefs[0] + assert load_record(root, "tsp-1").ending != "aborted" diff --git a/tests/test_author_overrides.py b/tests/test_author_overrides.py new file mode 100644 index 00000000..2605d7fa --- /dev/null +++ b/tests/test_author_overrides.py @@ -0,0 +1,397 @@ +"""Deployment author experiments retain their route without changing judges.""" + +import json +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from typing import Any, cast + +import pytest + +from outerloop import attempt +from outerloop.author_overrides import parse_overrides, select_override +from outerloop.runstate import RunRecord, load_record, save_record +from outerloop.tick import ServiceSpec, _climb_author_argv, _panel_preflight_error + + +@pytest.mark.parametrize("backend", ["claude", "codex", "hermes"]) +def test_slot_matching(monkeypatch, backend): + monkeypatch.setenv( + "OUTERLOOP_AUTHOR_OVERRIDES", + json.dumps( + { + "owner/repo": { + "backend": backend, + "model": "served-model[endpoint=onprem]", + "slots": ["agent-05"], + }, + "owner/all": {"backend": backend, "model": "served-model"}, + } + ), + ) + selected = select_override("owner/repo", "agent-05") + assert selected is not None and selected.backend == backend + assert select_override("owner/repo", "agent-04") is None + assert select_override("other/repo", "agent-05") is None + selected = select_override("owner/all", "agent-99") + assert selected is not None and selected.backend == backend + + +@pytest.mark.parametrize( + "value", + [ + "oops", + "[]", + "null", + '{"repo": {}}', + '{"owner/repo": []}', + '{"owner/repo": {"backend": "other", "model": "x"}}', + '{"owner/repo": {"backend": "claude"}}', + *[ + json.dumps({"owner/repo": {"backend": "claude", "model": "x", "slots": slots}}) + for slots in cast( + list[Any], + [ + None, + [], + "agent-01", + ["agent-00"], + ["agent-1"], + ["agent-001"], + ["agent-01", "agent-01"], + [1], + [{}], + ], + ) + ], + '{"owner/repo": {"backend": "claude", "model": "x", "endpoint": "onprem"}}', + ], +) +def test_invalid(value): + with pytest.raises(ValueError, match="OUTERLOOP_AUTHOR_OVERRIDES"): + parse_overrides(value) + + +@pytest.fixture +def deployment(tmp_path, monkeypatch): + def key(name, value): + path = tmp_path / name + path.write_text(value) + path.chmod(0o600) + return str(path) + + image = tmp_path / "image.sif" + image.touch() + monkeypatch.setenv("OUTERLOOP_AUTHOR_BACKEND", "claude") + monkeypatch.setenv("OUTERLOOP_AUTHOR_MODEL", "claude-fleet") + monkeypatch.delenv("OUTERLOOP_AUTHOR_ENDPOINT", raising=False) + monkeypatch.setenv("OUTERLOOP_CLAUDE_KEY_FILE", key("fleet-key", "fleet-secret")) + monkeypatch.setenv("OUTERLOOP_CODEX_KEY_FILE", key("codex-key", "codex-secret")) + monkeypatch.setenv("OUTERLOOP_HERMES_KEY_FILE", key("hermes-key", "hermes-secret")) + monkeypatch.setenv("OUTERLOOP_ENDPOINT_ONPREM_URL", "http://localhost:8000/v1") + monkeypatch.setenv("OUTERLOOP_ENDPOINT_ONPREM_MODEL", "served-model") + monkeypatch.setenv("OUTERLOOP_ENDPOINT_ONPREM_API", "anthropic,responses,chat") + monkeypatch.setenv("OUTERLOOP_ENDPOINT_ONPREM_KEY_FILE", key("endpoint-key", "endpoint-secret")) + monkeypatch.setenv( + "OUTERLOOP_AUTHOR_OVERRIDES", + json.dumps( + { + "owner/repo": { + "backend": "codex", + "model": "served-model[endpoint=onprem]", + "slots": ["agent-05"], + } + } + ), + ) + return ServiceSpec( + account="", + partition="", + run_root=tmp_path, + home=tmp_path, + target="owner/repo", + image=str(image), + panel="verify,review", + panel_key_file=key("panel-key", "judge-secret"), + ) + + +def panel_args(spec, **kwargs): + return SimpleNamespace( + panel=spec.panel, + panel_key_file=spec.panel_key_file, + image=spec.image, + claude_bin="claude", + codex_bin="codex", + **kwargs, + ) + + +def test_panel_independence(deployment): + ordinary, secrets = attempt._panel_lenses_from_args( + panel_args(deployment, author_backend="claude", model="claude-fleet") + ) + overridden, override_secrets = attempt._panel_lenses_from_args( + panel_args( + deployment, + author_backend="codex", + model="served-model[endpoint=onprem]", + author_overridden=True, + ) + ) + assert [(x.kind, x.harness) for x in ordinary] == [(x.kind, x.harness) for x in overridden] + assert secrets == override_secrets + assert _panel_preflight_error(deployment, "agent-05") == "" + + +@pytest.mark.parametrize("same_path", [False, True]) +def test_override_credential_separation(deployment, same_path): + endpoint_key = deployment.home / "endpoint-key" + if same_path: + deployment = replace(deployment, panel_key_file=str(endpoint_key)) + else: + Path(deployment.panel_key_file).write_text(endpoint_key.read_text()) + assert "role separation" in _panel_preflight_error(deployment, "agent-05") + assert _panel_preflight_error(deployment, "agent-04") == "" + with pytest.raises(ValueError, match="role separation"): + attempt._panel_lenses_from_args( + panel_args( + deployment, + author_backend="codex", + model="served-model[endpoint=onprem]", + author_overridden=True, + ) + ) + + +@pytest.mark.parametrize("backend", ["claude", "codex", "hermes"]) +@pytest.mark.parametrize("queued", [False, True]) +def test_binding_survives_setting_change(deployment, monkeypatch, backend, queued): + monkeypatch.setenv( + "OUTERLOOP_AUTHOR_OVERRIDES", + json.dumps( + { + "owner/repo": { + "backend": backend, + "model": "served-model[endpoint=onprem]", + "slots": ["agent-05"], + } + } + ), + ) + monkeypatch.setenv("REVIEW_HERMES_REPO", str(deployment.home / "hermes")) + monkeypatch.setattr("outerloop.hermes_install.hermes_ready", lambda path: True) + argv = _climb_author_argv(deployment, "agent-05") if queued else [] + if queued: + monkeypatch.setenv( + "OUTERLOOP_AUTHOR_OVERRIDES", '{"owner/repo":{"backend":"claude","model":"claude-new"}}' + ) + seen = {} + monkeypatch.setattr( + attempt, "resolve_bot_auth", lambda *a: SimpleNamespace(token=lambda: "bot") + ) + monkeypatch.setattr(attempt, "_dispatch_settings", lambda *a: None) + monkeypatch.setattr(attempt, "build_harness", lambda *a, **kw: seen.update(kw) or object()) + + real_launch = attempt.live_attempt + + class AfterRecord(BaseException): + pass + + def stop_before_network(*args, **kwargs): + raise AfterRecord + + monkeypatch.setattr(attempt.Workspace, "clone", stop_before_network) + + def launch(**kw): + with pytest.raises(AfterRecord): + real_launch(**kw) + record = load_record(deployment.run_root, kw["run_id"]) + record = replace(record, state="parked", stage={"phase": "author-sleep"}) + save_record(deployment.run_root, record, 1) + seen["record"] = record + return attempt.AttemptOutcome(run_id=record.run_id, outcome="parked") + + monkeypatch.setattr(attempt, "live_attempt", launch) + monkeypatch.setattr( + "sys.argv", + [ + "attempt", + "--target", + "owner/repo", + "--benchmark", + "bench", + "--agent-id", + "agent-05", + "--run-root", + str(deployment.run_root), + "--image", + deployment.image, + "--min-free-gb", + "0", + *argv, + ], + ) + assert attempt.main() == 0 + assert (seen["backend"], seen["model"]) == (backend, "served-model[endpoint=onprem]") + record = load_record(deployment.run_root, seen["record"].run_id) + assert record.author_overridden + assert attempt.resume_author(record, "claude-new", "claude")[:2] == ( + backend, + "served-model[endpoint=onprem]", + ) + monkeypatch.setenv("OUTERLOOP_AUTHOR_OVERRIDES", "{}") + # Exercise the actual dispatched wake entrypoint, including its harness. + monkeypatch.setattr(attempt, "_lease_held_by_another_job", lambda *a: "") + monkeypatch.setattr("outerloop.tick.dispatch_wake_armed", lambda *a: True) + monkeypatch.setattr(attempt, "_release_own_lease", lambda *a: None) + monkeypatch.setattr( + attempt, + "resume_run", + lambda *a, **kw: attempt.AttemptOutcome(run_id=record.run_id, outcome="parked"), + ) + monkeypatch.setattr( + "sys.argv", + [ + "attempt", + "--resume", + record.run_id, + "--run-root", + str(deployment.run_root), + "--image", + deployment.image, + ], + ) + seen.pop("backend") + assert attempt.main() == 0 + assert (seen["backend"], seen["model"]) == (backend, "served-model[endpoint=onprem]") + + +def test_no_setting_keeps_launch_bytes(deployment, monkeypatch): + monkeypatch.delenv("OUTERLOOP_AUTHOR_OVERRIDES") + assert _climb_author_argv(deployment, "agent-05") == [] + monkeypatch.setenv("OUTERLOOP_AUTHOR_OVERRIDES", "{}") + assert _climb_author_argv(deployment, "agent-05") == [] + + +@pytest.mark.parametrize("state", ["parked", "ended"]) +def test_legacy_record_tolerated_idempotently(tmp_path, state): + data = json.loads(Path("tests/fixtures/author_route_legacy.json").read_text()) + data["state"] = state + if state == "ended": + data["ending"] = "stuck" + directory = tmp_path / "runs" / data["run_id"] + directory.mkdir(parents=True) + (directory / "state.json").write_text(json.dumps(data)) + for _ in range(3): # first read, idempotent pass, interrupted writer retry + record = load_record(tmp_path, data["run_id"]) + assert not record.author_overridden + assert attempt.resume_author(record, "other", "claude") == ( + "codex", + "gpt-5.6-terra", + "/keys/author", + ) + (directory / ".state.json.interrupted.tmp").write_text('{"author_overridden":') + save_record(tmp_path, record, 2) + + +def test_wake_preflight_uses_bound_credential(deployment, monkeypatch): + record = RunRecord( + run_id="bound", + target=deployment.target, + task_title="trial", + state="parked", + agent_id="agent-05", + author_backend="codex", + author_model="served-model[endpoint=onprem]", + author_overridden=True, + ) + monkeypatch.delenv("OUTERLOOP_AUTHOR_OVERRIDES") + Path(deployment.panel_key_file).write_text("endpoint-secret") + assert _panel_preflight_error(deployment) == "" + assert "role separation" in _panel_preflight_error(deployment, record=record) + + +def test_unmatched_submission_is_also_bound(deployment): + argv = _climb_author_argv(deployment, "agent-04") + assert "--author-bound" in argv and "--author-overridden" not in argv + assert argv[argv.index("--model") + 1] == "claude-fleet" + assert argv[argv.index("--author-backend") + 1] == "claude" + + +def test_no_override_record_keeps_wire_bytes(tmp_path): + original = Path("tests/fixtures/author_route_legacy.json").read_text() + data = json.loads(original) + record = RunRecord(**data) + save_record(tmp_path, record, 1.0) + assert (tmp_path / "runs" / record.run_id / "state.json").read_text() == original + + +def test_status_exposes_author(deployment): + from outerloop.climbboard import collect_status + + record = RunRecord( + run_id="bound", + target=deployment.target, + task_title="trial", + state="parked", + author_backend="codex", + author_model="served-model[endpoint=onprem]", + author_overridden=True, + ) + status = collect_status(deployment.run_root, deployment.target, 1, records=[record]) + assert status["runs"][0]["author_backend"] == "codex" + assert status["runs"][0]["author_model"] == "served-model[endpoint=onprem]" + assert status["runs"][0]["author_overridden"] is True + + +def test_ci_reviewer_unchanged(deployment, monkeypatch): + from outerloop.review_agent_cli import resolve_reviewer_harness + from outerloop.roles import reviewer_spec + + monkeypatch.setenv("REVIEW_BACKEND", "claude") + monkeypatch.setenv("REVIEW_MODEL", "claude-fleet") + monkeypatch.setenv("ANTHROPIC_REVIEWER_KEY", "judge-secret") + overridden = resolve_reviewer_harness(reviewer_spec()) + monkeypatch.delenv("OUTERLOOP_AUTHOR_OVERRIDES") + assert overridden == resolve_reviewer_harness(reviewer_spec()) + + +def test_parse_once(): + raw = '{"owner/repo":{"backend":"claude","model":"claude-trial"}}' + first = parse_overrides(raw) + assert parse_overrides(raw) is first + with pytest.raises(TypeError): + first["owner/other"] = first["owner/repo"] # type: ignore[index] + + +def test_native_override_ignores_fleet_endpoint(deployment, monkeypatch): + from outerloop.tick import _author_config_error, _selected_author + + monkeypatch.setenv("OUTERLOOP_AUTHOR_ENDPOINT", "onprem") + monkeypatch.setenv( + "OUTERLOOP_AUTHOR_OVERRIDES", '{"owner/repo":{"backend":"codex","model":"gpt-trial"}}' + ) + assert _selected_author(deployment, "agent-05") == ("codex", "gpt-trial") + assert _author_config_error(deployment, "agent-05") == "" + + +def test_validation_names_setting(deployment, monkeypatch): + import os + + from outerloop.author_overrides import validate_overrides + + monkeypatch.setenv( + "OUTERLOOP_AUTHOR_OVERRIDES", '{"owner/repo":{"backend":"codex","model":"claude-wrong"}}' + ) + with pytest.raises(ValueError, match="OUTERLOOP_AUTHOR_OVERRIDES: owner/repo"): + validate_overrides(os.environ, deployment.image) + + +def test_endpoint_model_default_is_bound(deployment, monkeypatch): + monkeypatch.setenv( + "OUTERLOOP_AUTHOR_OVERRIDES", + '{"owner/repo":{"backend":"claude","model":"[endpoint=onprem]"}}', + ) + argv = _climb_author_argv(deployment, "agent-05") + assert argv[argv.index("--model") + 1] == "served-model[endpoint=onprem]" diff --git a/tests/test_endpoints.py b/tests/test_endpoints.py index 9c15b77d..269c9370 100644 --- a/tests/test_endpoints.py +++ b/tests/test_endpoints.py @@ -9,6 +9,7 @@ from pathlib import Path from types import SimpleNamespace from typing import Any +from unittest.mock import Mock import pytest import yaml @@ -26,6 +27,10 @@ @pytest.fixture def profile(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: + connection = Mock() + connection.getresponse.return_value.status = 200 + monkeypatch.setattr("outerloop.endpoints.HTTPConnection", Mock(return_value=connection)) + monkeypatch.setattr("outerloop.endpoints.HTTPSConnection", Mock(return_value=connection)) key = tmp_path / "endpoint-key" key.write_text("endpoint-secret") key.chmod(0o600) @@ -132,6 +137,7 @@ def test_author_and_start_preflight( @pytest.mark.parametrize("backend", ["claude", "codex", "hermes"]) @pytest.mark.parametrize("contained", [False, True]) @pytest.mark.parametrize("failed", [False, True]) +@pytest.mark.parametrize("url_file", [False, True]) def test_exact_client_configuration_at_process_boundary( profile: dict[str, str], tmp_path: Path, @@ -139,6 +145,7 @@ def test_exact_client_configuration_at_process_boundary( backend: str, contained: bool, failed: bool, + url_file: bool, ) -> None: seen: dict[str, Any] = {} workspace = tmp_path / "workspace" @@ -171,6 +178,11 @@ def communicate(self, **kwargs: Any) -> tuple[str, str]: monkeypatch.setattr( CodexHarness, "_login", lambda *_: pytest.fail("custom provider must not log in to OpenAI") ) + address = tmp_path / "server-address" + if url_file: + monkeypatch.delenv("OUTERLOOP_ENDPOINT_LOCAL_URL") + monkeypatch.setenv("OUTERLOOP_ENDPOINT_LOCAL_URL_FILE", str(address)) + address.write_text(profile["OUTERLOOP_ENDPOINT_LOCAL_URL"]) harness = build_harness( "ignored-key", author_spec(), @@ -181,6 +193,10 @@ def communicate(self, **kwargs: Any) -> tuple[str, str]: container_image="/opt/image.sif" if contained else "", ) result = harness.run("brief", workspace) + if url_file and not failed: + profile["OUTERLOOP_ENDPOINT_LOCAL_URL"] = "https://llm.example.internal:8443/v1" + address.write_text(json.dumps({"url": profile["OUTERLOOP_ENDPOINT_LOCAL_URL"]})) + result = harness.run("wake", workspace, resume_session_id=result.session_id or "existing") if failed: assert result.is_error assert "endpoint-secret" not in repr(seen["argv"]) @@ -203,7 +219,9 @@ def communicate(self, **kwargs: Any) -> tuple[str, str]: assert env[f"APPTAINERENV_{key_env}"] == "endpoint-secret" if backend == "claude": # Claude Code appends /v1/messages itself - assert env["ANTHROPIC_BASE_URL"] == "https://llm.example.internal" + assert env["ANTHROPIC_BASE_URL"] == profile["OUTERLOOP_ENDPOINT_LOCAL_URL"].removesuffix( + "/v1" + ) assert "ANTHROPIC_API_KEY" not in env assert "GOOGLE_APPLICATION_CREDENTIALS" not in env assert env["CLAUDE_CODE_USE_VERTEX"] == "0" @@ -250,7 +268,13 @@ def test_reviewer_endpoint( def test_dynamic_deploy_allowlists(profile: dict[str, str], tmp_path: Path) -> None: path = tmp_path / ".env" - settings = {**profile, "REVIEW_ENDPOINT": "local", "OUTERLOOP_AUTHOR_ENDPOINT": "local"} + settings = { + **profile, + "REVIEW_ENDPOINT": "local", + "OUTERLOOP_AUTHOR_ENDPOINT": "local", + "OUTERLOOP_ENDPOINT_LOCAL_URL_FILE": "/tmp/server-address", + "OUTERLOOP_AUTHOR_OVERRIDES": '{"owner/repo":{"backend":"claude","model":"served-model"}}', + } path.write_text( "\n".join(f"{k}={v}" for k, v in settings.items()) + "\nOUTERLOOP_ENDPOINT_LOCAL_KEY=must-not-forward\n" @@ -566,3 +590,152 @@ def test_endpoint_author_native_panel_separation( ) with pytest.raises(ValueError, match="role separation"): _panel_lenses_from_args(args) + + +@pytest.mark.parametrize("json_value", [False, True]) +def test_url_file_reloaded_for_sessions(profile, tmp_path, json_value): + path = tmp_path / "address" + profile.pop("OUTERLOOP_ENDPOINT_LOCAL_URL") + profile["OUTERLOOP_ENDPOINT_LOCAL_URL_FILE"] = str(path) + endpoint = endpoint_profile("local", "claude", environ=profile) + for port in (8000, 8001): + url = f"http://localhost:{port}/v1" + path.write_text(json.dumps({"url": url}) if json_value else url + "\n") + # Same harness-held profile, another session/wake. + assert endpoint.session_url() == url + + +@pytest.mark.parametrize( + "value", + [ + '{"url": 3}', + "{}", + "{", + "file:///tmp/server", + "http://user:pass@localhost", + "http://localhost?key=secret", + "http://localhost:not-a-port", + "http://localhost:99999", + ], +) +def test_url_file_validates_values(profile, tmp_path, value): + path = tmp_path / "address" + path.write_text(value) + profile.pop("OUTERLOOP_ENDPOINT_LOCAL_URL") + profile["OUTERLOOP_ENDPOINT_LOCAL_URL_FILE"] = str(path) + with pytest.raises(ValueError, match="endpoint"): + _ = endpoint_profile("local", "codex", environ=profile).url + + +def test_url_file_exclusive_absolute_and_missing(profile, tmp_path, monkeypatch, caplog): + from outerloop.endpoints import EndpointUnavailable, endpoint_config_key + + assert endpoint_config_key("OUTERLOOP_ENDPOINT_LOCAL_URL_FILE") + profile["OUTERLOOP_ENDPOINT_LOCAL_URL_FILE"] = "relative" + with pytest.raises(ValueError, match="exactly one"): + endpoint_profile("local", "claude", environ=profile) + profile.pop("OUTERLOOP_ENDPOINT_LOCAL_URL") + with pytest.raises(ValueError, match="absolute"): + endpoint_profile("local", "claude", environ=profile) + path = tmp_path / "address" + profile["OUTERLOOP_ENDPOINT_LOCAL_URL_FILE"] = str(path) + endpoint = endpoint_profile("local", "claude", environ=profile) + with pytest.raises(EndpointUnavailable, match="waiting for server"): + _ = endpoint.url + with pytest.raises(EndpointUnavailable): + endpoint.session_url() + + +def test_url_file_wake_keeps_park_and_refunds_retry(profile, tmp_path, monkeypatch): + from outerloop.attempt import defer_endpoint_wake + from outerloop.runstate import RunRecord, load_record, save_record + + monkeypatch.delenv("OUTERLOOP_ENDPOINT_LOCAL_URL") + path = tmp_path / "address" + monkeypatch.setenv("OUTERLOOP_ENDPOINT_LOCAL_URL_FILE", str(path)) + record = RunRecord( + run_id="waiting", + target="owner/repo", + task_title="trial", + state="parked", + author_backend="codex", + author_model="open-model[endpoint=local]", + wake_attempts=1, + resume_session_id="existing", + stage={"phase": "author-sleep", "candidate_ref": "keep"}, + ) + save_record(tmp_path, record, 1) + assert defer_endpoint_wake(tmp_path, record) + waiting = load_record(tmp_path, record.run_id) + assert waiting.state == "parked" and waiting.wake_attempts == 0 + assert waiting.resume_session_id == "existing" and waiting.stage == record.stage + path.write_text("http://localhost:8000/v1") + assert not defer_endpoint_wake(tmp_path, waiting) + + +def test_unreadable_url_file_is_retryable(profile, tmp_path, monkeypatch): + from outerloop.endpoints import EndpointUnavailable + + path = tmp_path / "address" + profile.pop("OUTERLOOP_ENDPOINT_LOCAL_URL") + profile["OUTERLOOP_ENDPOINT_LOCAL_URL_FILE"] = str(path) + endpoint = endpoint_profile("local", "claude", environ=profile) + original = Path.read_text + + def unreadable(self, *args, **kwargs): + if self == path: + raise PermissionError("unreadable") + return original(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", unreadable) + with pytest.raises(EndpointUnavailable, match=r"URL_FILE.*unavailable"): + _ = endpoint.url + + +@pytest.mark.parametrize("backend", ["claude", "codex", "hermes"]) +@pytest.mark.parametrize("failure", ["missing", "dead", "http", "term", "interrupt"]) +def test_session_probe_propagates_without_starting_model( + profile, tmp_path, monkeypatch, backend, failure +): + from outerloop.attempt import Terminated + from outerloop.endpoints import EndpointUnavailable + + address = tmp_path / "address" + monkeypatch.delenv("OUTERLOOP_ENDPOINT_LOCAL_URL") + monkeypatch.setenv("OUTERLOOP_ENDPOINT_LOCAL_URL_FILE", str(address)) + if failure != "missing": + address.write_text("http://localhost:8000/v1") + connection = Mock() + connection.getresponse.return_value.status = 503 if failure == "http" else 200 + errors = { + "dead": ConnectionRefusedError(), + "term": Terminated(), + "interrupt": KeyboardInterrupt(), + } + if failure in errors: + connection.request.side_effect = errors[failure] + factory = Mock(return_value=connection) + monkeypatch.setattr("outerloop.endpoints.HTTPConnection", factory) + monkeypatch.setattr(harness_mod, "hermes_ready", lambda _: True) + monkeypatch.setattr(harness_mod.subprocess, "Popen", lambda *a, **k: pytest.fail("spawned")) + harness = build_harness( + "ignored", + author_spec(), + backend=backend, + endpoint="local", + binary="/opt/cli", + hermes_repo=tmp_path / "hermes", + container_image="/opt/image.sif", + ) + workspace = tmp_path / "ws" + workspace.mkdir() + with pytest.raises(EndpointUnavailable): + harness.run("brief", workspace) + if failure == "missing": + factory.assert_not_called() + else: + factory.assert_called_once_with("localhost", 8000, timeout=3) + connection.request.assert_called_once_with( + "GET", "/v1/models", headers={"Authorization": "Bearer endpoint-secret"} + ) + connection.close.assert_called_once() diff --git a/tests/test_init.py b/tests/test_init.py index 19f7b83e..de1ddf2e 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -1183,3 +1183,49 @@ def test_init_ignores_hermes_budget_for_other_authors(tmp_path, monkeypatch, bac ) == 0 ) + + +@pytest.mark.parametrize("backend", ["codex", "hermes"]) +def test_fresh_init_provisions_override_before_validation(tmp_path, monkeypatch, backend): + monkeypatch.setattr(init, "CONFIG_DIR", tmp_path) + monkeypatch.setenv( + "OUTERLOOP_AUTHOR_OVERRIDES", + json.dumps({"owner/repo": {"backend": backend, "model": "org/model"}}), + ) + monkeypatch.setenv("REVIEW_HERMES_PROVIDER", "openrouter") + monkeypatch.setenv("REVIEW_HERMES_REPO", str(tmp_path / "hermes")) + installed = [] + + def install(name, target=None): + name = name or "claude" + installed.append(name) + return str(tmp_path / name) + + def ready(repo): + return "hermes" in installed + + monkeypatch.setattr(init, "install_harness", install) + monkeypatch.setattr(init, "hermes_ready", ready) + monkeypatch.setattr("outerloop.hermes_install.hermes_ready", ready) + image = tmp_path / "image.sif" + + def download(**kwargs): + assert "claude" in installed and backend in installed + image.touch() + return str(image) + + monkeypatch.setattr(init, "ensure_image", download) + assert init.main(["--yes", "--compute", "local", "--target", "owner/repo"]) == 0 + env = (tmp_path / ".env").read_text() + assert f"OUTERLOOP_IMAGE={image}" in env + key = "REVIEW_HERMES_REPO" if backend == "hermes" else "OUTERLOOP_CODEX_BIN" + assert f"{key}={tmp_path / backend}" in env + + +def test_init_bad_override_json_fails_before_provisioning(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(init, "CONFIG_DIR", tmp_path) + monkeypatch.setenv("OUTERLOOP_AUTHOR_OVERRIDES", "{") + monkeypatch.setattr(init, "install_harness", lambda *a, **k: pytest.fail("installed")) + monkeypatch.setattr(init, "ensure_image", lambda **k: pytest.fail("downloaded")) + assert init.main(["--yes", "--compute", "local", "--target", "owner/repo"]) == 2 + assert capsys.readouterr().err.count("outerloop init:") == 1 diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 5855bfdb..e161fe81 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -2410,3 +2410,27 @@ def wait(*args, **kwargs): "REFUSED" in str(m.payload) and "operator GPU limit" in str(m.payload) for m in pending(directory, 0) ) + + +def test_endpoint_loss_between_legs_preserves_session(tmp_path, monkeypatch): + from outerloop.endpoints import EndpointUnavailable + from outerloop.orchestrator import RunParked + + _write_syscall( + tmp_path, {"launches": [{"name": "a", "command": "x"}, {"name": "b", "command": "y"}]} + ) + original = FakeHarness.run + + def run(self, brief_text, workspace, resume_session_id=None): + if resume_session_id: + raise EndpointUnavailable("down") + return original(self, brief_text, workspace, resume_session_id) + + monkeypatch.setattr(FakeHarness, "run", run) + tight = CONTRACT.replace(" direction: min\n", " direction: min\n depth_k: 1\n", 1) + with pytest.raises(RunParked) as raised: + run_climb(tmp_path, [], contract=tight, launcher=_fake_launcher([])) + park = raised.value + assert park.capacity_wait and park.phase == "author-sleep" + assert park.session and park.session.session_id == "s1" + assert park.candidate_sha and park.sleeps_used == 0 and park.launches_used == 0