diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c4ce680..732fd3ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] +- Include GPU count and resolved GPU type in dispatched eval and baseline cache identity, including budget discounts. Legacy eval slots and baseline entries are cache misses. Upgrading: An eval dispatched by the previous kernel and still in flight at upgrade is measured again once under the new cache key (no extra budget charge). To avoid the extra run, upgrade when no evals are in flight: `touch /PAUSE` (stops wakes, so no new evals are dispatched), wait until no eval jobs remain in the queue, upgrade, then `rm /PAUSE` and run `outerloop start`. + - Park launch capacity refusals in capacity wait when an immediate resume is unavailable or already refused, instead of ending the run. The next wake delivers the refusal and a retry note, resuming the same session when supported diff --git a/src/outerloop/measure.py b/src/outerloop/measure.py index ce69bfcd..beb79d4f 100644 --- a/src/outerloop/measure.py +++ b/src/outerloop/measure.py @@ -182,12 +182,13 @@ def read_baseline_cache( metric: str = "", seed_env: str = "", gpus: int = 0, + gpu_type: str = "", ) -> dict[str, Any] | None: """The cached base-tree measurement for (benchmark, base sha), or None. The entry must have been measured under the SAME determinants the candidate will be — eval image, contract command, metric key, seed - variable, GPU count: everything the measurer's own eval identity - carries except the tree sha (the key) and the seed VALUE (fresh per + variable, GPU count and resolved GPU type: everything the measurer's + own eval identity carries except the tree sha (the key) and the seed VALUE (fresh per attempt by design) — or it is stale (terra #178): a comparison across determinants is not a comparison. A cache entry is only ever written from an orchestrator-measured value (below), @@ -207,7 +208,8 @@ def read_baseline_cache( or data.get("command", "") != command or data.get("metric", "") != metric or data.get("seed_env", "") != seed_env - or int(data.get("gpus", 0) or 0) != gpus + or data.get("gpus") != gpus + or data.get("gpu_type") != gpu_type ): return None return data @@ -226,6 +228,7 @@ def write_baseline_cache( metric: str = "", seed_env: str = "", gpus: int = 0, + gpu_type: str = "", ) -> None: """Record an orchestrator-measured baseline for every later attempt on this base, with the determinants it was measured under. Atomic (a @@ -250,6 +253,7 @@ def write_baseline_cache( "metric": metric, "seed_env": seed_env, "gpus": gpus, + "gpu_type": gpu_type, }, fh, ) @@ -302,7 +306,8 @@ def _det(self, m: Measure) -> str: # Everything a measure's RESULT depends on and that can vary across a # PARK/RESUME (when a fresh measurer reads this run_dir): the container # image, the measure's logical role, the code (tree_sha), and the - # contract facts it is evaluated under (command, metric, seeded env). + # contract facts it is evaluated under (command, metric, seeded env), + # GPU count, and the resolved GPU type from the lane. # A cache key missing any of these would return a value computed under # DIFFERENT inputs — e.g. a resume that re-fetched the contract after # its command changed, or ran under a rebuilt image, reading the stale @@ -310,7 +315,10 @@ def _det(self, m: Measure) -> str: # only whether it completes.) NUL separators keep the parts unambiguous # (`a`+`bc` != `ab`+`c`). env = "".join(f"\0{k}={v}" for k, v in sorted(m.env().items())) - return f"{self.image}\0{m.name}\0{m.tree_sha}\0{m.command}\0{m.metric}{env}" + return ( + f"{self.image}\0{m.name}\0{m.tree_sha}\0{m.command}\0{m.metric}" + f"\0{m.gpus}\0{self.gpu_type}{env}" + ) def _slot(self, m: Measure) -> str: # Storage identity = the full determinant, with NOTHING truncated: the diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index 54dd0f14..0e8b7806 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -918,6 +918,7 @@ def measure_and_decide( metric=bench.metric, seed_env=bench.seed_env or "", gpus=bench.gpus, + gpu_type=str(getattr(measurer, "gpu_type", "")), ) if bench.baseline == "cached" and cache_dir is not None else None @@ -960,6 +961,7 @@ def measure_and_decide( metric=bench.metric, seed_env=bench.seed_env or "", gpus=bench.gpus, + gpu_type=str(getattr(measurer, "gpu_type", "")), ) candidate = main["candidate"] if not improved(baseline, candidate, bench.direction, min_relative_improvement): @@ -1796,6 +1798,7 @@ def _not_run_note(request: SyscallRequest | None) -> str: metric=bench.metric, seed_env=bench.seed_env or "", gpus=bench.gpus, + gpu_type=str(getattr(measurer, "gpu_type", "")), ): main_evals = 1 capacity_refused = False diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index c6986fb5..1fd752c4 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -1,5 +1,19 @@ # Endpoint compatibility fixtures +- `dispatched_pre_gpu_identity/`: run directories produced by + `DispatchedMeasurer._dispatch` from kernel `d4ad7529667de79aef299e2e0ad8b5fb6c9c9598`, + before GPU count/type entered the determinant. A fake Slurm submit returned + job `101`; the completed variant adds the job's exit-code and metric stdout. + Both use a synthetic repo/image, one GPU, and `SEED=7`. `identity.json` + records that kernel's slot and scheduler name. The new reader ignores and + preserves the directories (including when interrupted), so old jobs can + finish writing and + rollback can still read the old records. Legacy slots are cache misses; the + new kernel dispatches once under the GPU-aware key without charging the + resumed gate again. Newly dispatched GPU-aware slots + are not readable by the old kernel without re-measurement. Cross-run legacy + baseline entries remain misses. + - `author_route_legacy.json`: produced by `runstate.RunRecord` and `runstate.save_record` from kernel commit `5563c46` (the parent tree before the endpoint change), with synthetic `owner/repo` and key-file coordinates. diff --git a/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/command.txt b/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/command.txt new file mode 100644 index 00000000..911dc380 --- /dev/null +++ b/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/command.txt @@ -0,0 +1 @@ +cmd \ No newline at end of file diff --git a/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/exit-code b/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/exit-code new file mode 100644 index 00000000..573541ac --- /dev/null +++ b/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/exit-code @@ -0,0 +1 @@ +0 diff --git a/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/job.sh b/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/job.sh new file mode 100755 index 00000000..77adf793 --- /dev/null +++ b/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/job.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -u +EV=tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec +REPO=/fixture/repo +SCRATCH="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}/dispatch-eval-$$" +TREE="$SCRATCH/tree" +mkdir -p "$SCRATCH/cache" "$SCRATCH/home" "$SCRATCH/work" "$TREE" +cleanup() { rm -rf "$SCRATCH"; git -C "$REPO" -c core.hooksPath=/dev/null -c core.sshCommand=false -c credential.helper= -c protocol.allow=never -c protocol.https.allow=always -c protocol.file.allow=always -c core.fsmonitor= -c core.quotePath=false worktree prune >/dev/null 2>&1 || true; } +trap 'cleanup' EXIT +trap 'echo 143 > "$EV/exit-code"; cleanup; trap - EXIT; exit 0' TERM INT HUP +[ -s "$EV/command.txt" ] || { echo 96 > "$EV/exit-code"; exit 0; } +export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null +export GIT_CONFIG_COUNT=1 +export GIT_CONFIG_KEY_0=core.attributesFile +export GIT_CONFIG_VALUE_0=/dev/null +if git -C "$REPO" -c core.hooksPath=/dev/null -c core.sshCommand=false -c credential.helper= -c protocol.allow=never -c protocol.https.allow=always -c protocol.file.allow=always -c core.fsmonitor= -c core.quotePath=false worktree add --detach "$TREE" aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa >> "$EV/setup.log" 2>&1; then rm -f "$TREE/.git"; else echo 97 > "$EV/exit-code"; exit 0; fi +export UV_CACHE_DIR="$SCRATCH/cache" UV_LINK_MODE=copy UV_PROJECT_ENVIRONMENT="$SCRATCH/cache/venv" +export APPTAINERENV_UV_CACHE_DIR="$UV_CACHE_DIR" APPTAINERENV_UV_LINK_MODE=copy APPTAINERENV_UV_PROJECT_ENVIRONMENT="$UV_PROJECT_ENVIRONMENT" +export SEED=7 APPTAINERENV_SEED=7 +apptainer exec --containall --cleanenv --nv --bind "$TREE:$TREE" --home "$SCRATCH/home:$SCRATCH/home" --bind "$SCRATCH/cache:$SCRATCH/cache" --pwd "$TREE" --workdir "$SCRATCH/work" /img.sif sh -c "$(cat "$EV/command.txt")" > "$EV/stdout" 2> "$EV/stderr" +echo $? > "$EV/exit-code" +exit 0 diff --git a/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/provenance.json b/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/provenance.json new file mode 100644 index 00000000..d1b704d3 --- /dev/null +++ b/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/provenance.json @@ -0,0 +1 @@ +{"commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "author": {}} \ No newline at end of file diff --git a/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/stdout b/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/stdout new file mode 100644 index 00000000..5f822e56 --- /dev/null +++ b/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/stdout @@ -0,0 +1 @@ +{"metric": "r2", "value": 0.42} diff --git a/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/submitted b/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/submitted new file mode 100644 index 00000000..97a55e1d --- /dev/null +++ b/tests/fixtures/dispatched_pre_gpu_identity/completed/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/submitted @@ -0,0 +1 @@ +101 \ No newline at end of file diff --git a/tests/fixtures/dispatched_pre_gpu_identity/identity.json b/tests/fixtures/dispatched_pre_gpu_identity/identity.json new file mode 100644 index 00000000..56e6c2d2 --- /dev/null +++ b/tests/fixtures/dispatched_pre_gpu_identity/identity.json @@ -0,0 +1,4 @@ +{ + "slot": "candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec", + "job_name": "eval-r1-candidate-b874c69af4063bd5" +} diff --git a/tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/command.txt b/tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/command.txt new file mode 100644 index 00000000..911dc380 --- /dev/null +++ b/tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/command.txt @@ -0,0 +1 @@ +cmd \ No newline at end of file diff --git a/tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/job.sh b/tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/job.sh new file mode 100755 index 00000000..775b3fcd --- /dev/null +++ b/tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/job.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -u +EV=tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec +REPO=/fixture/repo +SCRATCH="${SLURM_TMPDIR:-${TMPDIR:-/tmp}}/dispatch-eval-$$" +TREE="$SCRATCH/tree" +mkdir -p "$SCRATCH/cache" "$SCRATCH/home" "$SCRATCH/work" "$TREE" +cleanup() { rm -rf "$SCRATCH"; git -C "$REPO" -c core.hooksPath=/dev/null -c core.sshCommand=false -c credential.helper= -c protocol.allow=never -c protocol.https.allow=always -c protocol.file.allow=always -c core.fsmonitor= -c core.quotePath=false worktree prune >/dev/null 2>&1 || true; } +trap 'cleanup' EXIT +trap 'echo 143 > "$EV/exit-code"; cleanup; trap - EXIT; exit 0' TERM INT HUP +[ -s "$EV/command.txt" ] || { echo 96 > "$EV/exit-code"; exit 0; } +export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null +export GIT_CONFIG_COUNT=1 +export GIT_CONFIG_KEY_0=core.attributesFile +export GIT_CONFIG_VALUE_0=/dev/null +if git -C "$REPO" -c core.hooksPath=/dev/null -c core.sshCommand=false -c credential.helper= -c protocol.allow=never -c protocol.https.allow=always -c protocol.file.allow=always -c core.fsmonitor= -c core.quotePath=false worktree add --detach "$TREE" aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa >> "$EV/setup.log" 2>&1; then rm -f "$TREE/.git"; else echo 97 > "$EV/exit-code"; exit 0; fi +export UV_CACHE_DIR="$SCRATCH/cache" UV_LINK_MODE=copy UV_PROJECT_ENVIRONMENT="$SCRATCH/cache/venv" +export APPTAINERENV_UV_CACHE_DIR="$UV_CACHE_DIR" APPTAINERENV_UV_LINK_MODE=copy APPTAINERENV_UV_PROJECT_ENVIRONMENT="$UV_PROJECT_ENVIRONMENT" +export SEED=7 APPTAINERENV_SEED=7 +apptainer exec --containall --cleanenv --nv --bind "$TREE:$TREE" --home "$SCRATCH/home:$SCRATCH/home" --bind "$SCRATCH/cache:$SCRATCH/cache" --pwd "$TREE" --workdir "$SCRATCH/work" /img.sif sh -c "$(cat "$EV/command.txt")" > "$EV/stdout" 2> "$EV/stderr" +echo $? > "$EV/exit-code" +exit 0 diff --git a/tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/provenance.json b/tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/provenance.json new file mode 100644 index 00000000..d1b704d3 --- /dev/null +++ b/tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/provenance.json @@ -0,0 +1 @@ +{"commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "author": {}} \ No newline at end of file diff --git a/tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/submitted b/tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/submitted new file mode 100644 index 00000000..97a55e1d --- /dev/null +++ b/tests/fixtures/dispatched_pre_gpu_identity/inflight/eval-candidate-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-f69684777d523fe6502ee4ed4d09f98410a56aec/submitted @@ -0,0 +1 @@ +101 \ No newline at end of file diff --git a/tests/test_attempt.py b/tests/test_attempt.py index cb85e108..f9e8b4af 100644 --- a/tests/test_attempt.py +++ b/tests/test_attempt.py @@ -7960,3 +7960,74 @@ def test_line_snapshot_restores_protected_paths_only_in_seal(tmp_path, target_re assert not logs # Only ancestry was dropped; the final content was already restored. else: assert len(logs) == 1 and protected in logs[0] + + +def test_legacy_eval_redispatch_preserves_run_gpu_meter(tmp_path, monkeypatch): + import hashlib + + from outerloop.compute import CommandResult, SlurmCompute + from outerloop.measure import DispatchSettings, plan_measures + + real_measurer = DispatchSettings.measurer + state, run_id = _write_parked_candidate(tmp_path, monkeypatch, contract=CONTRACT_GPU) + monkeypatch.setattr(DispatchSettings, "measurer", real_measurer) + record = load_record(state, run_id) + record.stage.update(submitted=True, gpu_hours_used=1.0, sleeps_used=1, launches_used=0) + save_record(state, record, 1_000_000.0) + run_dir = state / "runs" / run_id + bench = load_contract(CONTRACT_GPU, "org/pilot").benchmarks[0] + measures = plan_measures( + bench.command, + bench.metric, + str(record.stage["base_sha"]), + str(record.stage["candidate_sha"]), + gpus=1, + ) + # Old-format completed slots exist, but cannot satisfy the upgraded gate. + for measure in measures: + legacy_det = ( + f"/img.sif\0{measure.name}\0{measure.tree_sha}\0{measure.command}\0{measure.metric}" + ) + slot = run_dir / ( + f"eval-{measure.name}-{measure.tree_sha}-{hashlib.sha1(legacy_det.encode()).hexdigest()}" + ) + slot.mkdir() + (slot / "submitted").write_text("501") + (slot / "exit-code").write_text("0\n") + (slot / "stdout").write_text('{"metric":"mean_tour_length","value":13.0}\n') + submitted = [] + live = {} + + def runner(argv, timeout_s): + if argv[0] == "sbatch": + submitted.append(argv) + job = str(600 + len(submitted)) + name = next(a.split("=", 1)[1] for a in argv if a.startswith("--job-name=")) + live[name] = job + return CommandResult(0, job + "\n", "") + if argv[0] == "squeue" and "--name" in argv: + return CommandResult(0, live.get(argv[argv.index("--name") + 1], ""), "") + return CommandResult(0, "PENDING\n" if argv[0] == "sacct" else "", "") + + dispatch = DispatchSettings( + compute=SlurmCompute(runner=runner), + image="/img.sif", + account="acct", + partition="cpu", + gpu_partition="gpu", + ) + for now in (1_000_100.0, 1_000_200.0): + outcome = resume_run( + state, + run_id, + dispatch=dispatch, + github=CommentingGitHub(), # type: ignore[arg-type] + bot_auth=NoAuth(), + now=now, + ) + assert outcome.outcome == "parked" + assert len(submitted) == 2 + saved = load_record(state, run_id) + assert saved.stage["afterany"] == "afterany:601:602" + assert saved.stage["gpu_hours_used"] == 1.0 + assert saved.stage["sleeps_used"] == 1 diff --git a/tests/test_measure.py b/tests/test_measure.py index 5f9bffe2..01faa4ed 100644 --- a/tests/test_measure.py +++ b/tests/test_measure.py @@ -551,3 +551,101 @@ def runner(argv, timeout_s): scripts = {n: Path(a[-1]).read_text() for n, a in by_job.items()} for n, text in scripts.items(): assert ("--nv" in text) == ("-sib-speedru" in n) + + +@pytest.mark.parametrize( + "gpus,gpu_type,reused", [(1, "type-a", True), (2, "type-a", False), (1, "type-b", False)] +) +def test_dispatched_cache_resource_identity(tmp_path, gpus, gpu_type, reused): + from dataclasses import replace + + submitted: list = [] + m = _measurer(tmp_path, submitted) + m.gpu_partition = "gpu" + m.gpu_type = "type-a" + original = Measure("candidate", "a" * 40, "cmd", "r2", gpus=1) + _land(m, original, 0.42) + resumed = replace(m, gpu_type=gpu_type) + measure = replace(original, gpus=gpus) + if reused: + assert resumed.results([measure]) == {"candidate": 0.42} + assert not submitted + else: + assert resumed._job_name(measure) != m._job_name(original) + with pytest.raises(MeasurementPending): + resumed.results([measure]) + assert len(submitted) == 1 + + +@pytest.fixture +def legacy_eval_run(tmp_path): + """Copy durable output from the previous kernel, without new-code writers.""" + import shutil + + source = Path(__file__).parent / "fixtures" / "dispatched_pre_gpu_identity" + identity = json.loads((source / "identity.json").read_text()) + + def copy(state): + run_dir = tmp_path / state + shutil.copytree(source / state, run_dir) + return run_dir, identity + + return copy + + +@pytest.mark.parametrize("state", ["completed", "inflight"]) +@pytest.mark.parametrize("interrupt", [False, True]) +def test_legacy_slot_is_miss_and_redispatch_is_idempotent( + legacy_eval_run, state, interrupt, monkeypatch +): + from dataclasses import replace + + run_dir, identity = legacy_eval_run(state) + legacy = run_dir / ("eval-" + identity["slot"]) + before = {p.name: p.read_bytes() for p in legacy.iterdir() if p.is_file()} + assert b"#SBATCH" not in before["job.sh"] + submitted: list = [] + live = {identity["job_name"]: "old-job"} if state == "inflight" else {} + m = _measurer(run_dir, submitted, live=live) + m.gpu_partition = "gpu" + m.gpu_type = "type-a" + measure = Measure("candidate", "a" * 40, "cmd", "r2", (("SEED", "7"),), 1) + assert m._slot(measure) != identity["slot"] + assert m._job_name(measure) != identity["job_name"] + + if interrupt: + # Die after sbatch accepts the new job, before its marker is written. + write_text = Path.write_text + + def interrupted_write(path, *args, **kwargs): + if path == m._ev(measure) / "submitted": + raise KeyboardInterrupt + return write_text(path, *args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(Path, "write_text", interrupted_write) + with pytest.raises(KeyboardInterrupt): + m.results([measure]) + assert not (m._ev(measure) / "submitted").exists() + else: + with pytest.raises(MeasurementPending) as caught: + m.results([measure]) + assert caught.value.job_ids == ("101",) + assert len(submitted) == 1 + assert Path(submitted[0][-1]) == m._ev(measure) / "job.sh" + + live[m._job_name(measure)] = "101" + resumed = _measurer(run_dir, submitted, live=live) + resumed.gpu_partition = "gpu" + resumed.gpu_type = "type-a" + for _ in range(2): + with pytest.raises(MeasurementPending) as caught: + replace(resumed).results([measure]) + assert caught.value.job_ids == ("101",) + assert len(submitted) == 1 + assert before == {p.name: p.read_bytes() for p in legacy.iterdir() if p.is_file()} + + # Only the new slot supplies a result, even alongside a completed old slot. + _land(resumed, measure, 0.9) + assert replace(resumed).results([measure]) == {"candidate": 0.9} + assert len(submitted) == 1 diff --git a/tests/test_measure_and_decide.py b/tests/test_measure_and_decide.py index 35b17474..d62c4c21 100644 --- a/tests/test_measure_and_decide.py +++ b/tests/test_measure_and_decide.py @@ -439,3 +439,18 @@ def test_cached_baseline_is_keyed_by_image_and_command(tmp_path): ) got = read_baseline_cache(d, "main", BASE, image="/a.sif", command="run main") assert got and got["value"] == 0.6 and not list(d.glob("*.tmp")) + + +@pytest.mark.parametrize("missing", [("gpus",), ("gpu_type",), ("gpus", "gpu_type")]) +def test_baseline_cache_missing_resource_fields_is_a_miss(tmp_path, missing): + import json + + from outerloop.measure import read_baseline_cache, write_baseline_cache + + write_baseline_cache(tmp_path, "main", BASE, value=0.5, seed=0, run_tag="old") + path = tmp_path / f"main@{BASE}.json" + data = json.loads(path.read_text()) + for key in missing: + del data[key] + path.write_text(json.dumps(data)) + assert read_baseline_cache(tmp_path, "main", BASE) is None diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 76de320c..090f207d 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -2927,3 +2927,90 @@ def preflight(): ) assert caught.value.base_sha == "tip" assert caught.value.sleeps_used == 1 and caught.value.launches_used == 0 + + +@pytest.mark.parametrize( + "change,main_evals", [("same", 1), ("count", 2), ("type", 2), ("legacy", 2)] +) +def test_cached_baseline_resource_identity_and_budget(tmp_path, monkeypatch, change, main_evals): + import json + + from outerloop.contract import load_contract + from outerloop.measure import write_baseline_cache + from outerloop.orchestrator import MeasureOK, measure_and_decide + from outerloop.syscall import budget_error + + contract_text = CONTRACT.replace( + " direction: min\n", + " direction: min\n baseline: cached\n min_delta: 0.01\n" + " gpus: 2\n eval_minutes: 30\n", + ) + contract = load_contract(contract_text, CONFIG.target) + bench = contract.benchmarks[0] + + class Measurer: + baseline_cache = tmp_path / "baselines" + image = "/eval.sif" + gpu_type = "type-a" + + def results(self, measures): + assert len(measures) == main_evals + return {m.name: 10.0 if m.name == "baseline" else 9.0 for m in measures} + + measurer = Measurer() + write_baseline_cache( + measurer.baseline_cache, + bench.name, + "base", + value=10.0, + seed=0, + run_tag="old", + image=measurer.image, + command=bench.command, + metric=bench.metric, + gpus=1 if change == "count" else 2, + gpu_type="type-b" if change == "type" else measurer.gpu_type, + ) + if change == "legacy": + path = measurer.baseline_cache / f"{bench.name}@base.json" + data = json.loads(path.read_text()) + del data["gpus"], data["gpu_type"] + path.write_text(json.dumps(data)) + + checked = [] + + def check_budget(request, **kwargs): + assert kwargs["main_evals"] == main_evals + error = budget_error(request, **kwargs) + assert bool(error) == (main_evals == 2) + checked.append(True) + return error + + monkeypatch.setattr("outerloop.orchestrator.syscall_budget_error", check_budget) + _write_syscall(tmp_path, {"submit": True, "report": "H: candidate"}) + attempt_once( + CONFIG, + contract_text, + tmp_path, + FakeHarness(result=ok_session()), + measurer, + "base", + lambda: "candidate", + inbox_dir=tmp_path / "inbox", + ruler="score", + changed_paths=lambda: ["src/pilot/solvers/tsp.py"], + launcher=_fake_launcher([]), + ) + assert checked + outcome = measure_and_decide( + contract, + bench, + base_sha="base", + candidate_sha="candidate", + seed=0, + suite_seed=0, + measured_paths=("src/pilot/solvers/tsp.py",), + measurer=measurer, + min_relative_improvement=0.005, + ) + assert isinstance(outcome, MeasureOK)