Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@ Versions follow [SemVer](https://semver.org).

## [Unreleased]

- Bad author overrides hold only affected fresh claims during ticks, with one log
per entry per tick; malformed settings hold named targets, or all fresh claims
when unreadable. Other tick services and bound runs continue; `outerloop start`
and `outerloop init` remain strict. No persisted state changes.

- Add read-only `outerloop status` (text/`--json`) for local runs and endpoint
outages. Endpoint waits stay out of the published board/status strip and never
trigger research-log commits; log one shared outage start and recovery with
Expand All @@ -16,7 +21,7 @@ Versions follow [SemVer](https://semver.org).
keys/journals are tolerated; ended runs and in-flight PRs are unchanged.
Rollback to the preceding kernel safely ignores the additive state.

- Startup validation of `OUTERLOOP_AUTHOR_OVERRIDES` (the tick and `outerloop start`) uses the
- Startup validation of `OUTERLOOP_AUTHOR_OVERRIDES` (`outerloop start`) uses the
image sessions actually run with, the default image when `OUTERLOOP_IMAGE` is unset. Before, a
codex override on a deployment without `OUTERLOOP_IMAGE` failed validation and stopped the tick.

Expand Down
7 changes: 5 additions & 2 deletions docs/install.md
Original file line number Diff line number Diff line change
Expand Up @@ -627,8 +627,11 @@ OUTERLOOP_AUTHOR_OVERRIDES='{"owner/repo":[{"backend":"claude","model":"served-m
```

This is deployment configuration, not a contract setting. The setting is
parsed and validated at startup. Endpoint overrides select their own profile in
`model`; they do not inherit `OUTERLOOP_AUTHOR_ENDPOINT`. Native overrides use
strictly validated by `outerloop start` and `outerloop init`; during ticks, an unusable
entry holds fresh claims only for its slots, without falling back to the fleet author.
A malformed setting holds fresh claims for every readable target key (or all targets
if unreadable), while existing runs and other tick services continue.
Endpoint overrides select their own profile in `model`; they do not inherit `OUTERLOOP_AUTHOR_ENDPOINT`. Native overrides use
the selected backend's author credential. Normal author/judge credential
separation still applies to the effective override credential.

Expand Down
124 changes: 93 additions & 31 deletions src/outerloop/tick.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,13 @@
import socket
import subprocess
import sys
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from dataclasses import asdict, dataclass, field, replace
from pathlib import Path
from typing import Any, Protocol
from uuid import uuid4

from outerloop.author_overrides import AuthorOverride
from outerloop.compute import (
GONE,
Compute,
Expand Down Expand Up @@ -1910,6 +1911,9 @@ def tick(
min_tick_s,
)
return TickReport(coalesced=True, launch_blocked=_launches_held(root))
author_errors: set[str] = set()
if service_spec is not None:
_preflight_claim_overrides(service_spec, author_errors)
report = sweep(
root,
compute,
Expand Down Expand Up @@ -2051,6 +2055,7 @@ def tick(
limits,
dry_run=service_dry_run,
records=tick_records,
author_errors=author_errors,
)
if launch_ok and contract is not None
else None
Expand Down Expand Up @@ -2082,6 +2087,7 @@ def tick(
limits=limits,
dry_run=service_dry_run,
records=tick_records,
author_errors=author_errors,
)
except Exception as exc:
log.warning("self-initiated selection failed: %s", exc)
Expand Down Expand Up @@ -2438,11 +2444,77 @@ def _climb_panel_argv(spec: ServiceSpec) -> list[str]:
return argv


def _malformed_override_targets() -> tuple[str, ...] | None:
"""Recover target scope only from a readable JSON object; otherwise hold all."""
from outerloop.author_overrides import SETTING

try:
data = json.loads(os.environ.get(SETTING, ""))
except ValueError:
return None
return tuple(data) if isinstance(data, dict) and data else None


def _claim_overrides(target: str) -> Mapping[str, tuple[AuthorOverride, ...]]:
from outerloop.author_overrides import overrides

try:
return overrides()
except ValueError:
targets = _malformed_override_targets()
if targets is not None and target not in targets:
return {}
raise


def _claim_author_error(spec: ServiceSpec, agent_id: str, reported: set[str] | None = None) -> str:
"""Use the author preflight for claims, logging each affected entry once per tick."""
error = _author_config_error(spec, agent_id)
if not error or isinstance(error, EndpointWaitReason):
return error
from outerloop.author_overrides import overrides

try:
selected = next((o for o in overrides().get(spec.target, ()) if o.matches(agent_id)), None)
slots = ", ".join(selected.slots) if selected and selected.slots else "all"
scope = f"target {spec.target}, slots {slots}"
except ValueError:
targets = _malformed_override_targets()
scope = f"targets {', '.join(targets)}, slots all" if targets else "all targets, slots all"
message = f"fresh claims held: {scope}: author misconfigured — {error}"
if reported is None or message not in reported:
log.error("%s", message)
if reported is not None:
reported.add(message)
return error


def _preflight_claim_overrides(spec: ServiceSpec, reported: set[str]) -> None:
from outerloop.author_overrides import overrides

try:
entries = overrides()
except ValueError:
targets = _malformed_override_targets()
_claim_author_error(
replace(spec, target=targets[0] if targets else spec.target), "agent-01", reported
)
return
for target, group in entries.items():
for entry in group:
_claim_author_error(
replace(spec, target=target),
entry.slots[0] if entry.slots else "agent-01",
reported,
)


def _selected_author(spec: ServiceSpec, agent_id: str = "agent-01") -> tuple[str, str]:
from outerloop.attempt import fleet_author_model
from outerloop.author_overrides import select_override

selected = select_override(spec.target, agent_id)
selected = next(
(o for o in _claim_overrides(spec.target).get(spec.target, ()) if o.matches(agent_id)), None
)
if selected:
return selected.backend, selected.resolved_model()
backend = os.environ.get("OUTERLOOP_AUTHOR_BACKEND") or "claude"
Expand All @@ -2451,10 +2523,13 @@ def _selected_author(spec: ServiceSpec, agent_id: str = "agent-01") -> tuple[str

def _climb_author_argv(spec: ServiceSpec, agent_id: str = "agent-01") -> list[str]:
from outerloop.attempt import effective_author_credential
from outerloop.author_overrides import overrides, select_override
from outerloop.author_overrides import overrides

if not overrides():
return []
try:
if not overrides():
return []
except ValueError:
_claim_overrides(spec.target) # Only unaffected targets may bind the fleet author.
backend, model = _selected_author(spec, agent_id)
credential = effective_author_credential(backend, model)
return [
Expand All @@ -2465,7 +2540,11 @@ def _climb_author_argv(spec: ServiceSpec, agent_id: str = "agent-01") -> list[st
model,
"--key-file",
str(credential.key_file),
*(["--author-overridden"] if select_override(spec.target, agent_id) else []),
*(
["--author-overridden"]
if any(o.matches(agent_id) for o in _claim_overrides(spec.target).get(spec.target, ()))
else []
),
]


Expand Down Expand Up @@ -2755,6 +2834,7 @@ def service_self_initiated(
limits: EffectiveLimits | None = None,
dry_run: bool = False,
records: list[RunRecord] | None = None,
author_errors: set[str] | None = None,
) -> tuple[str, str] | None:
"""The default background mode: when nothing else needs doing, climb the
least-recently-attempted benchmark.
Expand All @@ -2763,6 +2843,7 @@ def service_self_initiated(
`compute.submit` and the climb job writing its run record — without it,
every tick during Slurm queue latency would launch a duplicate climb.
"""
author_errors = author_errors if author_errors is not None else set()
limits = limits if limits is not None else effective_limits(getattr(contract, "budgets", None))
paused = outage_active(root, now, role="solver")
if paused:
Expand Down Expand Up @@ -2823,6 +2904,9 @@ def service_self_initiated(
if len(occupied) >= width:
return None
slot_agent = _free_agent_slot(occupied, width)
while slot_agent is not None and _claim_author_error(spec, slot_agent, author_errors):
occupied.add(slot_agent)
slot_agent = _free_agent_slot(occupied, width)
if slot_agent is None:
return None
dead_attempts = read_tombstones(root, spec.target, contract, now)
Expand All @@ -2845,17 +2929,6 @@ def service_self_initiated(
if lane_error := _gpu_lane_error(contract, benchmark, spec):
log.error("attempt on %s not launched: %s", benchmark, lane_error)
return None
author_error = _author_config_error(spec, slot_agent)
if author_error:
if isinstance(author_error, EndpointWaitReason):
return None
log.error(
"climb on %s not launched: author misconfigured — %s "
"(fix OUTERLOOP_AUTHOR_BACKEND/_MODEL)",
benchmark,
author_error,
)
return None
panel_error = _panel_preflight_error(spec, slot_agent)
if panel_error:
log.error(
Expand Down Expand Up @@ -3071,6 +3144,7 @@ def service_intake(
limits: EffectiveLimits | None = None,
dry_run: bool = False,
records: list[RunRecord] | None = None,
author_errors: set[str] | None = None,
) -> tuple[str, str] | None:
"""The requested lane: claim at most ONE qualifying issue per tick and
submit a climb job for it. The claim comment (posted by the climb job
Expand Down Expand Up @@ -3125,16 +3199,7 @@ def service_intake(
if lane_error := _gpu_lane_error(contract, task.benchmark, spec):
log.error("attempt on %s not launched: %s", task.benchmark, lane_error)
return None
author_error = _author_config_error(spec)
if author_error:
if isinstance(author_error, EndpointWaitReason):
return None
log.error(
"issue #%d not claimed: author misconfigured — %s "
"(fix OUTERLOOP_AUTHOR_BACKEND/_MODEL)",
task.number,
author_error,
)
if _claim_author_error(spec, "agent-01", author_errors):
return None
panel_error = _panel_preflight_error(spec)
if panel_error:
Expand Down Expand Up @@ -3602,11 +3667,8 @@ def main() -> int:
"OUTERLOOP_CADENCE_MIN via the chain's own parser (default 30)",
)
args = parser.parse_args()
from outerloop.author_overrides import validate_overrides

try:
gpu_lanes = gpu_lanes_from_env()
validate_overrides(os.environ, startup_image())
except ValueError as exc:
parser.error(str(exc))
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
Expand Down
Loading
Loading