Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ Versions follow [SemVer](https://semver.org).

## [Unreleased]

- `OUTERLOOP_AUTHOR_OVERRIDES` accepts a list of entries per target, so different
slots of one target can use different authors (each listed entry names its slots;
a slot may appear only once). The single-object form is unchanged.

- Reject Codex authors and judges on chat-only endpoints during preflight when
the bridge runtime is missing or stale, with `outerloop harness upgrade --used`
as the fix, before spending the author budget.
Expand Down
10 changes: 9 additions & 1 deletion docs/install.md
Original file line number Diff line number Diff line change
Expand Up @@ -599,7 +599,15 @@ OUTERLOOP_AUTHOR_OVERRIDES='{"owner/repo":{"backend":"claude","model":"served-mo
Each entry requires `backend` (`claude`, `codex`, or `hermes`) and `model`.
Omit `slots` to cover every author slot on that target; otherwise use the existing
`agent-01`, `agent-02`, … identities allocated by the contract's authors-abreast
width. This is deployment configuration, not a contract setting. The setting is
width. To give different slots of one target different authors, map the target
to a list of entries instead; each entry then names its `slots`, and no slot may
appear twice:

```sh
OUTERLOOP_AUTHOR_OVERRIDES='{"owner/repo":[{"backend":"claude","model":"served-model[endpoint=onprem]","slots":["agent-04"]},{"backend":"codex","model":"served-model[endpoint=onprem]","slots":["agent-03"]}]}'
```

This is deployment configuration, not a contract setting. The setting is
parsed and validated at startup. Endpoint overrides select their own profile in
`model`; they do not inherit `OUTERLOOP_AUTHOR_ENDPOINT`. Native overrides use
the selected backend's author credential. Normal author/judge credential
Expand Down
98 changes: 59 additions & 39 deletions src/outerloop/author_overrides.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,67 +31,87 @@ def matches(self, agent_id: str) -> bool:


@lru_cache(maxsize=16)
def parse_overrides(raw: str) -> Mapping[str, AuthorOverride]:
def parse_overrides(raw: str) -> Mapping[str, tuple[AuthorOverride, ...]]:
"""Parse once per setting value, including in long-lived tick processes."""
try:
data = json.loads(raw) if raw.strip() else {}
if not isinstance(data, dict):
raise ValueError("must be a JSON object")
result = {}
for target, value in data.items():
for target, entries in data.items():
if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", target):
raise ValueError(f"invalid target {target!r}; expected owner/repo")
if not isinstance(value, dict) or set(value) - {"backend", "model", "slots"}:
raise ValueError(f"{target}: expected backend, model and optional slots")
backend, model, slots = value.get("backend"), value.get("model"), value.get("slots")
if backend not in ("claude", "codex", "hermes"):
raise ValueError(f"{target}: backend must be claude, codex or hermes")
if (
not isinstance(model, str)
or not model.strip()
or model != model.strip()
or any(c in model for c in "\r\n\x00")
):
raise ValueError(
f"{target}: model must be a nonempty string without outer whitespace"
)
from outerloop.endpoints import split_endpoint

split_endpoint(model)
if "slots" in value and (
not isinstance(slots, list)
or not slots
or any(
not isinstance(s, str)
or not re.fullmatch(r"agent-\d{2,}", s)
or int(s[6:]) < 1
or s != f"agent-{int(s[6:]):02d}"
for s in slots
)
or len(set(slots)) != len(slots)
):
raise ValueError(
f"{target}: slots must be distinct agent identities (agent-01, ...)"
)
result[target] = AuthorOverride(backend, model, None if slots is None else tuple(slots))
# one override, or a list of them for different slots of the same target
listed = isinstance(entries, list)
if listed and not entries:
raise ValueError(f"{target}: expected at least one override")
parsed: list[AuthorOverride] = []
for value in entries if listed else [entries]:
parsed.append(_parse_one(target, value, listed))
claimed = [s for o in parsed for s in (o.slots or ())]
if len(set(claimed)) != len(claimed):
raise ValueError(f"{target}: a slot is listed in more than one override")
result[target] = tuple(parsed)
return MappingProxyType(result)
except (ValueError, TypeError) as exc:
raise ValueError(f"{SETTING}: {exc}") from exc


def overrides(environ: Mapping[str, str] | None = None) -> Mapping[str, AuthorOverride]:
def _parse_one(target: str, value: object, listed: bool) -> AuthorOverride:
if not isinstance(value, dict) or set(value) - {"backend", "model", "slots"}:
raise ValueError(f"{target}: expected backend, model and optional slots")
backend, model, slots = value.get("backend"), value.get("model"), value.get("slots")
if backend not in ("claude", "codex", "hermes"):
raise ValueError(f"{target}: backend must be claude, codex or hermes")
if (
not isinstance(model, str)
or not model.strip()
or model != model.strip()
or any(c in model for c in "\r\n\x00")
):
raise ValueError(f"{target}: model must be a nonempty string without outer whitespace")
from outerloop.endpoints import split_endpoint

split_endpoint(model)
if "slots" in value and (
not isinstance(slots, list)
or not slots
or any(
not isinstance(s, str)
or not re.fullmatch(r"agent-\d{2,}", s)
or int(s[6:]) < 1
or s != f"agent-{int(s[6:]):02d}"
for s in slots
)
or len(set(slots)) != len(slots)
):
raise ValueError(f"{target}: slots must be distinct agent identities (agent-01, ...)")
if listed and slots is None:
raise ValueError(f"{target}: each override in a list must name its slots")
return AuthorOverride(backend, model, None if slots is None else tuple(slots))


def overrides(
environ: Mapping[str, str] | None = None,
) -> Mapping[str, tuple[AuthorOverride, ...]]:
return parse_overrides((os.environ if environ is None else environ).get(SETTING, ""))


def override_entries(
environ: Mapping[str, str] | None = None,
) -> tuple[tuple[str, AuthorOverride], ...]:
"""Every (target, override) pair, for callers that check or install each backend."""
return tuple((t, o) for t, group in overrides(environ).items() for o in group)


def select_override(target: str, agent_id: str) -> AuthorOverride | None:
selected = overrides().get(target)
return selected if selected and selected.matches(agent_id) else None
return next((o for o in overrides().get(target, ()) if o.matches(agent_id)), None)


def validate_overrides(environ: Mapping[str, str], image: str) -> None:
from outerloop.attempt import author_config_error

for target, selected in overrides(environ).items():
for target, selected in override_entries(environ):
error = author_config_error(selected.backend, selected.model, image, environ=environ)
if error:
raise ValueError(f"{SETTING}: {target}: {error}")
4 changes: 2 additions & 2 deletions src/outerloop/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -419,11 +419,11 @@ def _setting_of(key: str, values: Mapping[str, str], environ: Mapping[str, str])

def missing_harness_binary(values: Mapping[str, str], environ: Mapping[str, str]) -> str:
"""Check all configured authors' host CLIs using the harness's lookup."""
from outerloop.author_overrides import overrides
from outerloop.author_overrides import override_entries

env = {**values, **environ}
backend = _setting_of("OUTERLOOP_AUTHOR_BACKEND", values, environ).lower() or "claude"
backends = dict.fromkeys([backend, *(value.backend for value in overrides(env).values())])
backends = dict.fromkeys([backend, *(value.backend for _, value in override_entries(env))])
for selected in backends:
problem = _missing_author_binary(selected, env)
if problem:
Expand Down
6 changes: 3 additions & 3 deletions src/outerloop/harness_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,9 +185,9 @@ def status(env: Mapping[str, str]) -> int:

def used_harnesses(env: Mapping[str, str]) -> list[str]:
author = env.get("OUTERLOOP_AUTHOR_BACKEND") or "claude"
from outerloop.author_overrides import overrides
from outerloop.author_overrides import override_entries

used = {author, *(selected.backend for selected in overrides(env).values())}
used = {author, *(selected.backend for _, selected in override_entries(env))}
if env.get("OUTERLOOP_STEWARD_KEY_FILE"):
used.add("claude")
if env.get("REVIEW_BACKEND"):
Expand All @@ -203,7 +203,7 @@ def used_harnesses(env: Mapping[str, str]) -> list[str]:
model = env.get("OUTERLOOP_AUTHOR_MODEL", "")
roles = [("author", author, model, env.get("OUTERLOOP_AUTHOR_ENDPOINT", ""))]
roles.extend(
(f"author override {key}", s.backend, s.model, "") for key, s in overrides(env).items()
(f"author override {key}", s.backend, s.model, "") for key, s in override_entries(env)
)
if env.get("REVIEW_BACKEND"):
roles.append(
Expand Down
10 changes: 5 additions & 5 deletions src/outerloop/init.py
Original file line number Diff line number Diff line change
Expand Up @@ -867,15 +867,15 @@ def main(argv: list[str] | None = None) -> int:
except StartError as exc:
print(f"outerloop init: {exc}", file=sys.stderr)
return 2
from outerloop.author_overrides import overrides, validate_overrides
from outerloop.author_overrides import override_entries, validate_overrides

try:
effective_overrides = {
**env_file_values(_env_path(), keys=None),
**answers.preserved_env,
**os.environ,
}
selected_authors = overrides(effective_overrides)
selected_authors = [o for _, o in override_entries(effective_overrides)]
except (ValueError, StartError) as exc:
print(f"outerloop init: {exc}", file=sys.stderr)
return 2
Expand Down Expand Up @@ -980,7 +980,7 @@ def main(argv: list[str] | None = None) -> int:
deployment_env = effective_overrides
override_keys = {
key
for selected in selected_authors.values()
for selected in selected_authors
for key in (
f"OUTERLOOP_{selected.backend.upper()}_KEY_FILE",
f"OUTERLOOP_{selected.backend.upper()}_BIN",
Expand All @@ -998,7 +998,7 @@ def main(argv: list[str] | None = None) -> int:
try:
panel = settings.get("OUTERLOOP_PANEL", "")
lenses = parse_lenses(panel, answers.author_backend) if panel.strip() else ()
for backend in sorted({a.backend for a in selected_authors.values()}):
for backend in sorted({a.backend for a in selected_authors}):
if backend in (answers.author_backend or AUTHOR_BACKENDS[0], "hermes"):
continue
binary = locate_harness(backend)
Expand All @@ -1008,7 +1008,7 @@ def main(argv: list[str] | None = None) -> int:
answers.preserved_env[author_bin_env(backend)] = binary
needs_hermes = (
answers.author_backend == "hermes"
or any(a.backend == "hermes" for a in selected_authors.values())
or any(a.backend == "hermes" for a in selected_authors)
or settings.get("REVIEW_BACKEND", "").lower() == "hermes"
) or any(backend == "hermes" for _, backend, _ in lenses)
if needs_hermes:
Expand Down
38 changes: 38 additions & 0 deletions tests/test_author_overrides.py
Original file line number Diff line number Diff line change
Expand Up @@ -395,3 +395,41 @@ def test_endpoint_model_default_is_bound(deployment, monkeypatch):
)
argv = _climb_author_argv(deployment, "agent-05")
assert argv[argv.index("--model") + 1] == "served-model[endpoint=onprem]"


def test_a_target_may_list_overrides_for_different_slots(monkeypatch):
model = "served-model[endpoint=onprem]"
raw = json.dumps(
{
"owner/repo": [
{"backend": "claude", "model": model, "slots": ["agent-04"]},
{"backend": "codex", "model": model, "slots": ["agent-03"]},
]
}
)
parsed = parse_overrides(raw)["owner/repo"]
assert [o.backend for o in parsed] == ["claude", "codex"]
monkeypatch.setenv("OUTERLOOP_AUTHOR_OVERRIDES", raw)
four, three = (
select_override("owner/repo", "agent-04"),
select_override("owner/repo", "agent-03"),
)
assert four is not None and four.backend == "claude"
assert three is not None and three.backend == "codex"
assert select_override("owner/repo", "agent-01") is None


@pytest.mark.parametrize(
"entries",
[
[], # an empty list
[{"backend": "claude", "model": "m"}], # a list entry must name its slots
[
{"backend": "claude", "model": "m", "slots": ["agent-03"]},
{"backend": "codex", "model": "m", "slots": ["agent-03"]},
], # one slot claimed twice
],
)
def test_listed_overrides_are_validated(entries):
with pytest.raises(ValueError, match=r"^OUTERLOOP_AUTHOR_OVERRIDES:"):
parse_overrides(json.dumps({"owner/repo": entries}))
Loading