diff --git a/docs/code/configuration.md b/docs/code/configuration.md index 32612560..d562e758 100644 --- a/docs/code/configuration.md +++ b/docs/code/configuration.md @@ -392,6 +392,32 @@ Common `AGENT_HARNESS` values include `claude-code`, `opencode`, `codex`, `curso **Qwen note:** Qwen Code accepts a model via its `--model` flag (e.g. `qwen3-coder-plus`) — set it with `AGENT_MODEL`; you can also keep the model in `~/.qwen/settings.json`. +### Failover across multiple harnesses + +`AGENT_HARNESS` accepts a comma-separated, priority-ordered list so the unattended worker keeps processing when an agent hits its usage limit: + +```bash +# .devintern-code/.env +AGENT_HARNESS=claude-code,codex +``` + +The first entry is your preferred harness; later entries are fallbacks in priority order. A single value behaves exactly as before. + +**Failover behavior (worker mode):** + +Applies to every unattended worker surface — fleet task polling, PR review addressing, `@mention` runs, conflict resolution, scheduled automations, estimations, dashboard retries, relay-driven tasks, and `devintern webhook serve` — not only the webhook queue. + +- At startup every entry is checked against the harness registry and your machine: unknown or not-installed entries produce a clear warning and are skipped, and the effective chain is logged (e.g. `Agent harness: claude-code → codex (failover enabled)`). +- When the active harness reports a usage/rate limit, the worker records its reset window (parsed from the limit output; a 1-hour cooldown applies when no timer is parseable, e.g. monthly spend limits) and immediately retries the same work on the highest-priority harness that still has capacity. +- When the primary harness's window elapses, the worker automatically fails back to it and logs the switch. Fallback agents hitting their own limits mid-run advance the chain again. +- If every harness in the chain is limited at once, new agent work is deferred until the earliest window ends (the webhook queue pauses; polling/review/automation runs return to their next tick). +- Failover state (active harness + per-harness windows) persists in the queue database, so restarting the worker resumes on the right harness instead of retrying a still-limited agent. +- Which harness executed each run is recorded in run records, and `/health` on the webhook server reports the active harness, the chain, and open limit windows. + +Interactive one-shot runs you start yourself (`devintern TASK-123` in a terminal) always use the first (priority) entry; the worker pins each subprocess to the active harness so failover can switch the next attempt. + +**Per-harness overrides inside a list:** `_CLI_PATH` (e.g. `CODEX_CLI_PATH`) resolves per active harness at spawn time. The global `AGENT_CLI_PATH` applies to the first entry only, so a stale global override cannot leak onto a fallback agent. `AGENT_MODEL` applies to whichever harness is active (the string is harness-specific). + ### Model selection Set the model the agent harness runs with using `AGENT_MODEL` in `.devintern-code/.env`: diff --git a/docs/code/worker.md b/docs/code/worker.md index d4b644bf..40aa7f6d 100644 --- a/docs/code/worker.md +++ b/docs/code/worker.md @@ -38,6 +38,10 @@ devintern worker devintern webhook serve ``` +## Agent failover + +Set `AGENT_HARNESS=codex,grok` (comma-separated, priority first) in the workspace `.env` so the worker keeps going when one agent hits a usage limit. Failover applies to every worker job: tracker tasks, PR review addressing, `@mention` runs, conflict resolution, scheduled automations, estimations, dashboard retries, and relay-driven work. Details: [Failover across multiple harnesses](./configuration.md#failover-across-multiple-harnesses). + ## Recurring automations Put recurring work in `workspace.toml`. Set `repo` when the workspace has multiple repositories; it is optional for a one-repo workspace: diff --git a/packages/agent-harness/src/detect-usage-limit.ts b/packages/agent-harness/src/detect-usage-limit.ts index 7d662c4d..728fefe9 100644 --- a/packages/agent-harness/src/detect-usage-limit.ts +++ b/packages/agent-harness/src/detect-usage-limit.ts @@ -20,10 +20,14 @@ * AI_RetryError: Failed after 4 attempts. Last error: Too Many Requests * Too Many Requests: {"error":{"code":"1302","message":"Rate limit reached for req..."}} * rate_limit_error / quota exceeded + * Free usage exceeded, subscribe to Go + * 5 hour usage limit reached. It will reset in 5 hours 23 minutes. To continue + * using this model now, enable usage from your available balance * - * which the provider-rate-limit patterns below cover. + * which the subscription and provider-rate-limit patterns below cover. * Ref (opencode API rate-limit reporting): * https://github.com/sst/opencode/issues/2398 (AI_RetryError: ... Too Many Requests) + * https://github.com/anomalyco/opencode/blob/dev/packages/opencode/src/session/retry.ts * * OpenCode may keep `run` alive after a provider error and only expose the * diagnostic through `--print-logs`, as a timestamped line containing an @@ -43,19 +47,30 @@ const USAGE_LIMIT_PATTERNS = [ /^(?:(?:AI_(?:APICall|Retry)Error|error):\s*)?(?:(?:\d+[- ]hour\s+)?(?:usage|session|account|fast|opus|sonnet|fable 5|usage credit) limit reached|claude (?:ai )?usage limit(?: reached)?)(?:\s*(?:[.·—-]\s*)?(?:resets?|try again|available again|retry[- ]after)\b[^\n]*)?[.!]?$/i, // Claude Code 2.1.218 exports these as USAGE_LIMIT_ERROR_PREFIXES. /^(?:error:\s*)?(?:you(?:'|’)re out of (?:usage credits|extra usage)|your org is out of usage\s*·\s*(?:add funds to continue|contact your admin)|your seat type doesn(?:'|’)t include usage credits|your usage allocation has been disabled by your admin|your group(?:'|’)s usage limit is set to \$0)(?:\s*(?:[.·—-]\s*)?(?:resets?|try again|available again|retry[- ]after)\b[^\n]*)?[.!]?$/i, - // Codex UsageLimitReachedError variants. Keep these whole-line anchored: - // Codex writes its complete tool transcript to stderr. - /^you(?:'|’)ve hit your usage limit\.\s+(?:(?:upgrade to (?:pro|plus)|visit https:\/\/chatgpt\.com\/codex\/settings\/usage|contact your admin)\b[^\n]*?)(?:or\s+)?try again at\s+[^\n.]+[.]?$/i, - /^you(?:'|’)ve hit your usage limit for [^.]+\.\s+switch to another model now, or try again at\s+[^\n.]+[.]?$/i, - /^your workspace is out of credits\.\s+(?:add credits to continue|ask your workspace owner to add credits)[.]?$/i, - /^you hit your spend cap set in your workspace\.\s+(?:increase your spend cap to continue|ask your workspace owner to increase the spend cap)[.]?$/i, + // Codex UsageLimitReachedError Display (openai/codex protocol/src/error.rs). + // Plan promo copy and the reset suffix (`try again at 4:27 PM` vs `later`) + // vary; keep the distinctive prefix and allow the rest of the line. Codex + // writes its complete tool transcript to stderr, often with an `ERROR:` prefix. + /^(?:error:\s*)?you(?:'|’)ve hit your usage limit(?:\s+for [^.]+)?\.(?:\s+[^\n]*)?$/i, + /^your workspace is out of credits\.\s+(?:add credits to continue|ask your workspace owner to (?:add credits|refill in order to continue))[.]?$/i, + /^you hit your spend cap set (?:in your workspace|by the owner of your workspace)\.\s+(?:increase your spend cap to continue|ask (?:your workspace owner|an owner) to increase (?:your |the )?spend cap(?: to continue)?)[.]?$/i, /^quota exceeded\.\s+check your plan and billing details[.]?$/i, /^to use codex with your chatgpt plan, upgrade to plus\b[^\n]*$/i, + // OpenCode Go (anomalyco/opencode session/retry.ts): FreeUsageLimitError and + // GoUsageLimitError are rewritten into these headless retry messages. + /^(?:error:\s*)?free usage exceeded(?:,\s*subscribe to go)?[.]?$/i, + /^(?:(?:AI_(?:APICall|Retry)Error|error):\s*)?(?:(?:\d+[-\s]?hour|weekly|daily|monthly)\s+)?usage limit reached\.(?:\s+it will reset in [^\n.]+(?:\.|$))?(?:\s+to continue using this model now, enable usage from your available balance\b[^\n]*)?$/i, + /^subscription quota exceeded\.\s+you can continue using free models[.]?$/i, // Grok Build translates account/team exhaustion to these headless messages. /^you(?:'|’)ve hit the rate limit for your plan\.\s+upgrade your account or try again later[.]?$/i, /^you(?:'|’)ve hit your team(?:'|’)s api rate limit\.\s+ask a team admin to purchase more credits for higher limits, or try again later\.\s+see https:\/\/docs\.x\.ai\/developers\/rate-limits#rate-limit-tiers$/i, /^you(?:'|’)ve reached your free grok build usage limit for now\.\s+get supergrok for much higher limits, or try again later:\s+https:\/\/grok\.com\/supergrok\?referrer=grok-build$/i, /^resource-exhausted:\s+too many requests for team [^.]+\.\s+see https:\/\/console\.x\.ai\/team\/default\/rate-limits[.]?$/i, + // Grok Build paid-balance exhaustion: headless CLI prints a pretty-printed + // JSON Internal error (`"message": "API error (status 402 Payment Required): ..."`). + // Allow a JSON `"...": "` prefix so the diagnostic still matches inside that object. + /(?:^|:\s*")(?:api error \(status 402 payment required\):\s*)?grok build usage balance exhausted\b/i, + /(?:^|:\s*")api error \(status 402 payment required\)/i, // Goose maps provider HTTP 402 responses to this error and can exit zero. /^(?:error:\s*)?credits exhausted:\s+[^\n]+$/i, /^(?:⚠\s*)?individual quota reached\.\s+please upgrade your subscription to increase your limits[.]?$/i, @@ -138,7 +153,10 @@ function isLikelyProviderDiagnostic(line: OutputLine): boolean { /^\[API Error:\s*429\b[^\n]*\]$/i.test(trimmed) || /^(?:AI_RetryError|Too Many Requests)\b/i.test(trimmed) || /^HTTP\s*429\b/i.test(trimmed) || - /^\s*[{"[].*(?:rate_limit|quota|insufficient balance|add credits).*[}\]]\s*$/i.test(trimmed) || + /^(?:internal error|error:\s*internal error)\b/i.test(trimmed) || + /^\s*[{"[].*(?:rate_limit|quota|insufficient balance|add credits|payment required|usage balance exhausted).*[}\]]\s*$/i.test( + trimmed, + ) || /\b(?:last error|provider (?:error|response)|response status|returned (?:an? )?(?:error|status)|request failed|retrying)\b/i.test( trimmed, ) @@ -198,11 +216,17 @@ function findUsageLimitLine(stdout: string, stderr: string): OutputLine | undefi return lines.find((line) => { const normalized = line.normalized.trim(); - if (!normalized || isSourceOrDiffLine(normalized)) { + // Codex (and some other CLIs) prefix diagnostics with `ERROR:`. Strip it + // only for subscription-limit matching so a prefixed Codex message is + // still detected without treating arbitrary transcript lines as errors. + const diagnostic = normalized.replace(/^(?:error:\s*)/i, ""); + if (!normalized || isSourceOrDiffLine(normalized) || isSourceOrDiffLine(diagnostic)) { return false; } - if (USAGE_LIMIT_PATTERNS.some((pattern) => pattern.test(normalized))) { + if ( + USAGE_LIMIT_PATTERNS.some((pattern) => pattern.test(diagnostic) || pattern.test(normalized)) + ) { return true; } diff --git a/packages/agent-harness/src/harness-chain.ts b/packages/agent-harness/src/harness-chain.ts new file mode 100644 index 00000000..41ac8a93 --- /dev/null +++ b/packages/agent-harness/src/harness-chain.ts @@ -0,0 +1,242 @@ +/** + * Priority-ordered harness chain (`AGENT_HARNESS=claude-code,codex`). + * + * A comma-separated `AGENT_HARNESS` value names a failover chain: the first + * entry is the preferred harness, later entries are fallbacks used when an + * earlier one hits its usage/rate limit. Parsing and resolution live here so + * every consumer (worker failover, readiness probe, one-shot runs) sees the + * same list semantics: + * + * - Entries are split on commas and trimmed; empty entries are dropped. + * - Aliases (e.g. `agy`, deprecated `gemini`) resolve to canonical names. + * - Duplicate canonical names collapse to the first occurrence, preserving + * the requested priority. + * - An empty/unset value defaults to `["claude-code"]`. + * + * Resolution validates each name against the registry, resolves its CLI path + * per harness (so `AGENT_CLI_PATH` cannot leak across harnesses — harness + * specific `_CLI_PATH` overrides and defaults apply instead), and + * checks installability. Unknown or not-installed entries are reported as + * issues for the caller to warn about and skipped rather than fatal, unless + * that would leave an empty chain (then the full list is kept and the spawn + * itself surfaces the real error, matching single-harness behavior). + */ + +import { DEFAULT_HARNESS_NAME, getHarness, HARNESS_ALIASES, listHarnesses } from "./registry.js"; +import { getHarnessCliCommand, isHarnessCliAvailable } from "./resolver.js"; +import type { AgentHarness } from "./types.js"; + +/** + * Parse a raw (possibly comma-separated) harness list into canonical names. + * + * Applies registry aliases, drops empty entries, and de-duplicates canonical + * names keeping the first occurrence. Returns `[DEFAULT_HARNESS_NAME]` when + * nothing usable remains. + * + * @param raw - Raw `AGENT_HARNESS` value (already env-resolved), may be undefined. + * @returns Ordered canonical harness names (priority first). + */ +export function parseHarnessList(raw: string | undefined): string[] { + const names: string[] = []; + const seen = new Set(); + for (const part of (raw ?? "").split(",")) { + const requested = part.trim(); + if (!requested) { + continue; + } + const canonical = getHarness(requested)?.name ?? requested; + if (seen.has(canonical)) { + continue; + } + seen.add(canonical); + names.push(canonical); + } + return names.length > 0 ? names : [DEFAULT_HARNESS_NAME]; +} + +/** One ordered, resolved entry of the harness chain. */ +export interface HarnessChainEntry { + /** Canonical registry name (after alias resolution). */ + readonly name: string; + readonly harness: AgentHarness; + /** Resolved CLI command/path for this harness. */ + readonly path: string; + /** Whether the resolved CLI is installed/reachable on this machine. */ + readonly installed: boolean; +} + +/** A chain entry that could not be used, with a warning message. */ +export interface HarnessChainIssue { + /** Name as written in the list (after alias resolution). */ + readonly requested: string; + readonly reason: "unknown" | "not-installed"; + readonly message: string; +} + +/** Result of resolving a harness chain. */ +export interface ResolvedHarnessChain { + /** Ordered usable entries (priority first). */ + readonly entries: HarnessChainEntry[]; + /** Entries dropped from the chain, with warning messages. */ + readonly issues: HarnessChainIssue[]; + /** All canonical names as parsed from the raw value (before dropping). */ + readonly parsed: string[]; + /** True when the raw value listed more than one harness. */ + readonly multiHarness: boolean; +} + +/** Options for {@link resolveHarnessChain}. */ +export interface HarnessChainOptions { + /** Raw list value; defaults to `AGENT_HARNESS` env (or the default harness). */ + raw?: string; + /** + * Probe installability and drop not-installed entries (default true). + * When false, every known entry is kept with `installed` reported as-is. + */ + checkInstalled?: boolean; + /** + * When false, suppress deprecation warnings for aliased harness names + * (e.g. `gemini` → `antigravity`). Defaults to true. + */ + warnDeprecated?: boolean; + /** Installability predicate override (tests). Defaults to PATH probing. */ + isInstalled?: (entry: { name: string; path: string }) => boolean; +} + +/** + * Resolve the CLI path for one parsed chain entry. + * + * Harness-specific `_CLI_PATH` overrides and the harness default + * apply; the global `AGENT_CLI_PATH` only applies to the first (priority) + * entry so a single-value configuration keeps resolving exactly like + * {@link resolveHarness} does today, while a stale global override cannot + * stick to a fallback harness selected during failover. + * + * @param harness - Registered harness for the entry. + * @param isPrimary - Whether this is the first (priority) chain entry. + * @param warnDeprecated - Whether legacy path env vars may warn. + * @returns The CLI command or path to spawn/probe for this entry. + */ +function resolveEntryPath( + harness: AgentHarness, + isPrimary: boolean, + warnDeprecated?: boolean, +): string { + if (isPrimary && process.env.AGENT_CLI_PATH) { + return process.env.AGENT_CLI_PATH; + } + return getHarnessCliCommand(harness, { warnDeprecated }); +} + +/** + * Resolve the priority-ordered harness chain from a comma-separated value. + * + * Unknown names and (when `checkInstalled`) not-installed CLIs are reported in + * `issues` and skipped; if nothing survives, the full parsed list is kept so + * spawning fails with the familiar actionable error instead of an empty chain. + * + * @param options - Raw value override, installability toggles, and test hooks. + * @returns The resolved chain with usable entries and dropped-entry issues. + */ +export function resolveHarnessChain(options: HarnessChainOptions = {}): ResolvedHarnessChain { + const raw = options.raw ?? process.env.AGENT_HARNESS; + const parsed = parseHarnessList(raw); + const checkInstalled = options.checkInstalled ?? true; + const warnDeprecated = options.warnDeprecated; + + const entries: HarnessChainEntry[] = []; + const issues: HarnessChainIssue[] = []; + + const buildEntry = ( + canonical: string, + isPrimary: boolean, + reportIssues: boolean, + enforceInstalled: boolean, + ): HarnessChainEntry | null => { + const harness = getHarness(canonical); + if (!harness) { + if (reportIssues) { + const availableNames = listHarnesses() + .map((h) => `"${h.name}"`) + .join(", "); + issues.push({ + requested: canonical, + reason: "unknown", + message: `Unknown agent harness "${canonical}" in AGENT_HARNESS; skipping it. Available harnesses: ${availableNames}.`, + }); + } + return null; + } + + const path = resolveEntryPath(harness, isPrimary, warnDeprecated); + const installed = options.isInstalled + ? options.isInstalled({ name: harness.name, path }) + : isHarnessCliAvailable(path); + + if (enforceInstalled && checkInstalled && !installed) { + if (reportIssues) { + const envKey = harness.name.toUpperCase().replace(/-/g, "_"); + issues.push({ + requested: canonical, + reason: "not-installed", + message: `Harness "${canonical}" is not installed (looked for "${path}"); skipping it. Install its CLI or set ${envKey}_CLI_PATH to the executable.`, + }); + } + return null; + } + + return { name: harness.name, harness, path, installed }; + }; + + // Emit deprecation warnings for requested alias names (e.g. `gemini`), + // once per distinct alias, mirroring resolveHarness's behavior. + if (warnDeprecated !== false) { + const warnedAliases = new Set(); + for (const token of (raw ?? "").split(",")) { + const requested = token.trim(); + const alias = requested ? HARNESS_ALIASES[requested] : undefined; + if (alias?.deprecated && alias.warning && !warnedAliases.has(requested)) { + warnedAliases.add(requested); + console.warn(`⚠️ ${alias.warning}`); + } + } + } + + parsed.forEach((canonical, index) => { + const entry = buildEntry(canonical, index === 0, true, true); + if (entry) { + entries.push(entry); + } + }); + + if (entries.length === 0) { + if (parsed.every((canonical) => !getHarness(canonical))) { + // Nothing but unknown names: a configuration error, same as handing + // resolveHarness a single invalid name. + const availableNames = listHarnesses() + .map((h) => `"${h.name}"`) + .join(", "); + throw new Error( + `Unknown agent harness "${parsed.join(",")}" in AGENT_HARNESS. ` + + `Available harnesses: ${availableNames}. ` + + `Set AGENT_HARNESS to a comma-separated list of registered harnesses.`, + ); + } + // Everything was dropped as not installed: keep the full parsed list so + // the spawn path reports the familiar "CLI not found" error instead of + // failing startup with an empty chain. + parsed.forEach((canonical, index) => { + const entry = buildEntry(canonical, index === 0, false, false); + if (entry) { + entries.push(entry); + } + }); + } + + return { + entries, + issues, + parsed, + multiHarness: parsed.length > 1, + }; +} diff --git a/packages/agent-harness/src/index.ts b/packages/agent-harness/src/index.ts index a959352c..e6580cc8 100644 --- a/packages/agent-harness/src/index.ts +++ b/packages/agent-harness/src/index.ts @@ -33,7 +33,23 @@ export { } from "./modes.js"; // Registry -export { registerHarness, getHarness, listHarnesses, HARNESS_ALIASES } from "./registry.js"; +export { + registerHarness, + getHarness, + listHarnesses, + HARNESS_ALIASES, + DEFAULT_HARNESS_NAME, +} from "./registry.js"; + +// Harness chain (comma-separated AGENT_HARNESS failover lists) +export { + parseHarnessList, + resolveHarnessChain, + type HarnessChainEntry, + type HarnessChainIssue, + type HarnessChainOptions, + type ResolvedHarnessChain, +} from "./harness-chain.js"; // Prompt argument construction export { buildPromptArgs } from "./prompt-args.js"; diff --git a/packages/agent-harness/src/output-lines.ts b/packages/agent-harness/src/output-lines.ts index 6554ee63..07ea7376 100644 --- a/packages/agent-harness/src/output-lines.ts +++ b/packages/agent-harness/src/output-lines.ts @@ -46,7 +46,9 @@ export function isSourceOrDiffLine(line: string): boolean { /^(?:(?:\.?\.?\/|\/)?(?:[^:\s]+\/)+[^:\s]+|[^:\s]+\.[a-z\d]+):\d+(?::\d+)?:/i.test(trimmed) || /^(?:const|let|var|function|class|import|export|return)\b/.test(trimmed) || /^(?:super|throw\s+new\s+Error|[\w$.]+\.(?:error|warn|log))\s*\(/.test(trimmed) || - /^(?:["'`]).*(?:["'`])[,;)]?$/.test(trimmed) || + // A single quoted string / JS string literal — not JSON `"key": "value"`, + // which Grok (and others) print as pretty-printed Internal error objects. + /^(?:["'`])(?:[^"'`\\]|\\.)*(?:["'`])[,;)]?$/.test(trimmed) || /\b(?:includes|startsWith|endsWith|\.match|\.test)\s*\(/.test(trimmed) || /=>/.test(trimmed) ); diff --git a/packages/agent-harness/src/registry.ts b/packages/agent-harness/src/registry.ts index 96dd8a53..2f56a2e3 100644 --- a/packages/agent-harness/src/registry.ts +++ b/packages/agent-harness/src/registry.ts @@ -22,6 +22,9 @@ import { const registry = new Map(); +/** Default harness when no harness is configured (e.g. `AGENT_HARNESS` unset). */ +export const DEFAULT_HARNESS_NAME = "claude-code"; + /** * Alias map: alternate harness ids → canonical registered name. * diff --git a/packages/agent-harness/src/resolver.ts b/packages/agent-harness/src/resolver.ts index 40b7600d..26da5412 100644 --- a/packages/agent-harness/src/resolver.ts +++ b/packages/agent-harness/src/resolver.ts @@ -3,7 +3,8 @@ * * Resolution order for harness name: * 1. `options.harnessName` - * 2. `AGENT_HARNESS` environment variable + * 2. `AGENT_HARNESS` environment variable (a comma-separated list uses its + * first entry — see `resolveHarnessChain` for full-chain resolution) * 3. Default to "claude-code" (backward compatible) * * Resolution order for executable path: @@ -17,7 +18,7 @@ import { execSync } from "child_process"; import { accessSync, constants, existsSync } from "fs"; import { resolve } from "path"; -import { getHarness, HARNESS_ALIASES, listHarnesses } from "./registry.js"; +import { DEFAULT_HARNESS_NAME, getHarness, HARNESS_ALIASES, listHarnesses } from "./registry.js"; import type { AgentHarness, ResolvedHarness } from "./types.js"; export interface HarnessResolutionOptions { @@ -41,6 +42,10 @@ export interface HarnessResolutionOptions { * Resolve which harness to use and the CLI executable path to invoke. * * Harness name: `options.harnessName` → `AGENT_HARNESS` → `"claude-code"`. + * A comma-separated `AGENT_HARNESS` (e.g. `claude-code,codex`) names a + * failover chain; this resolver always picks the first (priority) entry so + * one-shot flows keep behaving as a single-harness run — worker-mode failover + * across the rest of the chain lives in {@link resolveHarnessChain}. * Aliases (e.g. `agy` / deprecated `gemini` → `antigravity`) are applied before * registry lookup; deprecated aliases emit a one-line console warning. * @@ -65,10 +70,16 @@ export function resolveHarness(options?: HarnessResolutionOptions): ResolvedHarn const explicitHarnessName = options?.harnessName; let harnessName = explicitHarnessName; if (!harnessName) { - harnessName = env.AGENT_HARNESS; + // AGENT_HARNESS may name a comma-separated failover chain; env-driven + // (non-explicit) resolution always uses the first (priority) entry, kept + // raw so alias/deprecation warnings still fire for the requested name. + const firstEntry = env.AGENT_HARNESS?.split(",")[0]?.trim(); + if (firstEntry) { + harnessName = firstEntry; + } } if (!harnessName) { - harnessName = "claude-code"; + harnessName = DEFAULT_HARNESS_NAME; } const alias = HARNESS_ALIASES[harnessName]; diff --git a/packages/agent-harness/tests/detect-usage-limit.test.ts b/packages/agent-harness/tests/detect-usage-limit.test.ts index 6226d4e1..3c7606d1 100644 --- a/packages/agent-harness/tests/detect-usage-limit.test.ts +++ b/packages/agent-harness/tests/detect-usage-limit.test.ts @@ -81,20 +81,54 @@ describe("detectUsageLimit", () => { expect(result.resetsAt).toBe("6:03 PM"); }); + test("detects Codex's ERROR:-prefixed usage-limit line from the CLI", () => { + const out = + "ERROR: You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro), visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at 4:27 PM."; + const result = detectUsageLimit("", out); + expect(result.limited).toBe(true); + expect(result.resetsAt).toBe("4:27 PM"); + expect(result.matchedLine).toContain("You've hit your usage limit"); + }); + test("detects Codex named limits, workspace credits, and spend caps", () => { const messages = [ "You've hit your usage limit for GPT-5. Switch to another model now, or try again at 8:10 PM.", "Your workspace is out of credits. Add credits to continue.", "Your workspace is out of credits. Ask your workspace owner to add credits.", + "Your workspace is out of credits. Ask your workspace owner to refill in order to continue.", "You hit your spend cap set in your workspace. Increase your spend cap to continue.", "You hit your spend cap set in your workspace. Ask your workspace owner to increase the spend cap.", + "You hit your spend cap set by the owner of your workspace. Ask an owner to increase your spend cap to continue.", "Quota exceeded. Check your plan and billing details.", "To use Codex with your ChatGPT plan, upgrade to Plus at https://chatgpt.com/pricing.", + "To use Codex with your ChatGPT plan, upgrade to Plus: https://chatgpt.com/explore/plus.", + ]; + + for (const message of messages) { + expect(detectUsageLimit("", message).limited).toBe(true); + } + }); + + test("detects current Codex plan copy including try-again-later and admin upsell", () => { + const messages = [ + "You've hit your usage limit. Upgrade to Plus to continue using Codex (https://chatgpt.com/explore/plus), or try again later.", + "You've hit your usage limit. To get more access now, send a request to your admin or try again later.", + "You've hit your usage limit. Visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again later.", + "You've hit your usage limit. Try again later.", + "You've hit your usage limit. To continue using Codex, start a free trial of today, or try again at 6:03 PM.", + "ERROR: You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro), visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again later.", ]; for (const message of messages) { expect(detectUsageLimit("", message).limited).toBe(true); } + + const withReset = detectUsageLimit( + "", + "You've hit your usage limit. To get more access now, send a request to your admin or try again at 4:27 PM.", + ); + expect(withReset.limited).toBe(true); + expect(withReset.resetsAt).toBe("4:27 PM"); }); test("detects current Claude Code credit and allocation exhaustion messages", () => { @@ -140,6 +174,43 @@ describe("detectUsageLimit", () => { } }); + test("detects Grok Build's pretty-printed 402 usage-balance JSON", () => { + const out = [ + "Internal error: {", + ' "message": "API error (status 402 Payment Required): Grok Build usage balance exhausted",', + ' "http_status": 402', + "}", + ].join("\n"); + const result = detectUsageLimit("", out); + expect(result.limited).toBe(true); + expect(result.matchedLine).toContain("usage balance exhausted"); + }); + + test("detects Grok Build's Error-prefixed 402 reprint and compact JSON", () => { + const reprint = [ + "Error: Internal error: {", + ' "message": "API error (status 402 Payment Required): Grok Build usage balance exhausted",', + ' "http_status": 402', + "}", + ].join("\n"); + const compact = + 'Internal error: {"message":"API error (status 402 Payment Required): Grok Build usage balance exhausted","http_status":402}'; + + expect(detectUsageLimit("", reprint).limited).toBe(true); + expect(detectUsageLimit(compact, "").limited).toBe(true); + }); + + test("ignores Grok 402 phrases in source and prose", () => { + const transcript = [ + 'const message = "API error (status 402 Payment Required): Grok Build usage balance exhausted";', + '"Grok Build usage balance exhausted"', + "The docs mention Grok Build usage balance exhausted as a billing state.", + ].join("\n"); + + expect(detectUsageLimit("", transcript).limited).toBe(false); + expect(detectUsageLimit(transcript, "").limited).toBe(false); + }); + test("detects Goose credits exhaustion despite its successful exit", () => { const out = "Error: Credits exhausted: Insufficient credits to complete this request"; expect(detectUsageLimit("", out).limited).toBe(true); @@ -258,6 +329,34 @@ describe("detectUsageLimit", () => { expect(result.resetsAt).toBe("4hr 9min"); }); + test("detects OpenCode GoUsageLimitError and FreeUsageLimitError retry copy", () => { + const goLimit = + "5 hour usage limit reached. It will reset in 5 hours 23 minutes. To continue using this model now, enable usage from your available balance - https://opencode.ai/workspace/wrk_01K6XGM22R6FM8JVABE9XDQXGH/go"; + const goLimitResult = detectUsageLimit("", goLimit); + expect(goLimitResult.limited).toBe(true); + expect(goLimitResult.resetsAt).toBe("5 hours 23 minutes"); + + const unnamed = detectUsageLimit( + "", + "Usage limit reached. It will reset in 15 minutes. To continue using this model now, enable usage from your available balance", + ); + expect(unnamed.limited).toBe(true); + expect(unnamed.resetsAt).toBe("15 minutes"); + + expect( + detectUsageLimit( + "", + "Weekly usage limit reached. It will reset in 2 days. To continue using this model now, enable usage from your available balance", + ).limited, + ).toBe(true); + expect(detectUsageLimit("", "Free usage exceeded, subscribe to Go").limited).toBe(true); + expect(detectUsageLimit("", "Free usage exceeded").limited).toBe(true); + expect( + detectUsageLimit("", "Subscription quota exceeded. You can continue using free models.") + .limited, + ).toBe(true); + }); + test("extracts a usage limit from OpenCode's printed structured log", () => { const out = 'timestamp=2026-08-26T14:10:30.957Z level=ERROR run=be653f66 message="stream error" providerID=opencode-go modelID=glm-5.3 error.error="AI_APICallError: 5-hour usage limit reached. Resets in 4hr 9min."'; diff --git a/packages/agent-harness/tests/harness-chain.test.ts b/packages/agent-harness/tests/harness-chain.test.ts new file mode 100644 index 00000000..7fdd826b --- /dev/null +++ b/packages/agent-harness/tests/harness-chain.test.ts @@ -0,0 +1,214 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { parseHarnessList, resolveHarnessChain } from "../src/harness-chain.js"; +import { resolveHarness } from "../src/resolver.js"; +import { getHarness } from "../src/registry.js"; + +describe("parseHarnessList", () => { + test("defaults to claude-code when raw is undefined", () => { + expect(parseHarnessList(undefined)).toEqual(["claude-code"]); + }); + + test("defaults to claude-code when raw is empty or only separators", () => { + expect(parseHarnessList("")).toEqual(["claude-code"]); + expect(parseHarnessList(" , ,")).toEqual(["claude-code"]); + }); + + test("parses a single name", () => { + expect(parseHarnessList("codex")).toEqual(["codex"]); + }); + + test("splits on commas, trims, and drops empty entries", () => { + expect(parseHarnessList(" claude-code , codex ,, grok ")).toEqual([ + "claude-code", + "codex", + "grok", + ]); + }); + + test("applies aliases to canonical names", () => { + expect(parseHarnessList("agy")).toEqual(["antigravity"]); + expect(parseHarnessList("gemini,codex")).toEqual(["antigravity", "codex"]); + }); + + test("de-duplicates canonical names keeping the first occurrence", () => { + expect(parseHarnessList("codex, codex")).toEqual(["codex"]); + expect(parseHarnessList("gemini, antigravity")).toEqual(["antigravity"]); + expect(parseHarnessList("claude-code, agy, antigravity")).toEqual(["claude-code", "antigravity"]); + }); + + test("keeps unknown names so callers can warn about them", () => { + expect(parseHarnessList("nope")).toEqual(["nope"]); + }); +}); + +describe("resolveHarness with comma-separated AGENT_HARNESS", () => { + const originalEnv = { ...process.env }; + let warnings: string[]; + const originalWarn = console.warn; + + beforeEach(() => { + delete process.env.AGENT_HARNESS; + delete process.env.AGENT_CLI_PATH; + warnings = []; + console.warn = (msg?: unknown) => { + warnings.push(String(msg)); + }; + }); + + afterEach(() => { + console.warn = originalWarn; + for (const key of Object.keys(process.env)) { + if (!(key in originalEnv)) { + delete process.env[key]; + } + } + Object.assign(process.env, originalEnv); + }); + + test("uses the first (priority) entry of the list", () => { + process.env.AGENT_HARNESS = "codex,claude-code"; + const result = resolveHarness(); + expect(result.harness.name).toBe("codex"); + expect(result.path).toBe("codex"); + }); + + test("still warns when the first entry is a deprecated alias", () => { + process.env.AGENT_HARNESS = "gemini,codex"; + const result = resolveHarness(); + expect(result.harness.name).toBe("antigravity"); + expect(warnings.some((w) => w.includes("deprecated"))).toBe(true); + }); + + test("single value behaves exactly as before", () => { + process.env.AGENT_HARNESS = "codex"; + expect(resolveHarness().harness.name).toBe("codex"); + }); +}); + +describe("resolveHarnessChain", () => { + const originalEnv = { ...process.env }; + let warnings: string[]; + const originalWarn = console.warn; + + const alwaysInstalled = () => true; + + beforeEach(() => { + delete process.env.AGENT_HARNESS; + delete process.env.AGENT_CLI_PATH; + warnings = []; + console.warn = (msg?: unknown) => { + warnings.push(String(msg)); + }; + }); + + afterEach(() => { + console.warn = originalWarn; + for (const key of Object.keys(process.env)) { + if (!(key in originalEnv)) { + delete process.env[key]; + } + } + Object.assign(process.env, originalEnv); + }); + + test("returns a single entry for a single-harness value", () => { + const chain = resolveHarnessChain({ raw: "codex", isInstalled: alwaysInstalled }); + expect(chain.entries.map((e) => e.name)).toEqual(["codex"]); + expect(chain.multiHarness).toBe(false); + expect(chain.issues).toEqual([]); + }); + + test("defaults to claude-code when raw is undefined", () => { + const chain = resolveHarnessChain({ isInstalled: alwaysInstalled }); + expect(chain.entries.map((e) => e.name)).toEqual(["claude-code"]); + expect(chain.parsed).toEqual(["claude-code"]); + }); + + test("keeps priority order and flags multi-harness chains", () => { + const chain = resolveHarnessChain({ + raw: "codex, claude-code , grok", + isInstalled: alwaysInstalled, + }); + expect(chain.entries.map((e) => e.name)).toEqual(["codex", "claude-code", "grok"]); + expect(chain.multiHarness).toBe(true); + }); + + test("skips unknown entries with a warning", () => { + const chain = resolveHarnessChain({ + raw: "does-not-exist,codex", + isInstalled: alwaysInstalled, + }); + expect(chain.entries.map((e) => e.name)).toEqual(["codex"]); + expect(chain.issues).toHaveLength(1); + expect(chain.issues[0]!.reason).toBe("unknown"); + expect(chain.issues[0]!.requested).toBe("does-not-exist"); + expect(chain.issues[0]!.message).toContain("Available harnesses"); + expect(warnings).toEqual([]); + }); + + test("skips not-installed entries with a warning", () => { + const chain = resolveHarnessChain({ + raw: "codex,grok", + isInstalled: ({ name }) => name !== "codex", + }); + expect(chain.entries.map((e) => e.name)).toEqual(["grok"]); + expect(chain.issues).toHaveLength(1); + expect(chain.issues[0]!.reason).toBe("not-installed"); + expect(chain.issues[0]!.message).toContain("CODEX_CLI_PATH"); + }); + + test("throws when every entry is unknown", () => { + expect(() => + resolveHarnessChain({ raw: "nope, also-nope", isInstalled: alwaysInstalled }), + ).toThrow("Unknown agent harness"); + }); + + test("keeps the full list when everything is not installed", () => { + const chain = resolveHarnessChain({ + raw: "codex,grok", + isInstalled: () => false, + }); + expect(chain.entries.map((e) => e.name)).toEqual(["codex", "grok"]); + expect(chain.entries.every((e) => e.installed === false)).toBe(true); + }); + + test("checkInstalled=false skips installability probing", () => { + const chain = resolveHarnessChain({ raw: "codex,grok", checkInstalled: false }); + expect(chain.entries.map((e) => e.name)).toEqual(["codex", "grok"]); + expect(chain.issues).toEqual([]); + }); + + test("AGENT_CLI_PATH applies to the primary entry only", () => { + process.env.AGENT_CLI_PATH = "/custom/agent"; + const chain = resolveHarnessChain({ raw: "codex,grok", isInstalled: alwaysInstalled }); + expect(chain.entries[0]!.path).toBe("/custom/agent"); + expect(chain.entries[1]!.path).toBe("grok"); + }); + + test("harness-specific env overrides resolve per entry", () => { + process.env.GROK_CLI_PATH = "/custom/grok"; + const chain = resolveHarnessChain({ raw: "codex,grok", isInstalled: alwaysInstalled }); + expect(chain.entries[0]!.path).toBe("codex"); + expect(chain.entries[1]!.path).toBe("/custom/grok"); + }); + + test("resolves registry harness objects on each entry", () => { + const chain = resolveHarnessChain({ raw: "codex", isInstalled: alwaysInstalled }); + expect(chain.entries[0]!.harness).toBe(getHarness("codex")); + }); + + test("warns once for a deprecated alias inside the list", () => { + const chain = resolveHarnessChain({ raw: "gemini,codex", isInstalled: alwaysInstalled }); + expect(chain.entries.map((e) => e.name)).toEqual(["antigravity", "codex"]); + expect(warnings.filter((w) => w.includes("deprecated"))).toHaveLength(1); + }); + + test("warnDeprecated=false suppresses deprecation warnings", () => { + resolveHarnessChain({ + raw: "gemini", + warnDeprecated: false, + isInstalled: alwaysInstalled, + }); + expect(warnings).toEqual([]); + }); +}); diff --git a/packages/code/.devintern-code/.env.example b/packages/code/.devintern-code/.env.example index 6542394e..f07008fb 100644 --- a/packages/code/.devintern-code/.env.example +++ b/packages/code/.devintern-code/.env.example @@ -19,6 +19,12 @@ JIRA_API_TOKEN=your-api-token-here # Which AI agent to use: claude-code | opencode | codex | cursor | grok | deepseek # (also: gemini | kimi | qwen | goose | kilo-code | cline | pi) # Defaults to 'claude-code' if not specified +# +# Accepts a comma-separated, priority-ordered list for automatic failover in +# worker mode: when the first harness hits its usage limit, the worker +# switches to the next entry and fails back to the first once its window +# resets. Unknown or not-installed entries are warned about and skipped. +# AGENT_HARNESS=claude-code,codex AGENT_HARNESS=claude-code # Optional: Path to the agent CLI executable. diff --git a/packages/code/.env.example b/packages/code/.env.example index fcfe3a2f..d189048d 100644 --- a/packages/code/.env.example +++ b/packages/code/.env.example @@ -81,6 +81,12 @@ JIRA_API_TOKEN=your-api-token-here # Which AI agent to use: claude-code | opencode | codex | cursor | grok | deepseek # (also: gemini | kimi | qwen | goose | kilo-code | cline | pi) # Defaults to 'claude-code' if not specified +# +# Accepts a comma-separated, priority-ordered list for automatic failover in +# worker mode: when the first harness hits its usage limit, the worker +# switches to the next entry and fails back to the first once its window +# resets. Unknown or not-installed entries are warned about and skipped. +# AGENT_HARNESS=claude-code,codex AGENT_HARNESS=claude-code # Optional: Path to the agent CLI executable. diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 5aa5ebb0..d4518c2c 100755 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -97,6 +97,12 @@ import { clearRetryState, getRetryState, recordIncompleteAttempt } from "./lib/r import { shouldSkipRetry } from "./lib/retry-gate"; import { formatAgentInputNeededMarkdown } from "./lib/trackers/shared/markdown-comment-formatter"; import { reportTaskFailure } from "./lib/failure-feedback"; +import { + exitIfWorkerUsageLimit, + isWorkerChild, + USAGE_LIMIT_EXIT_CODE, + writeUsageLimitHint, +} from "./lib/usage-limit-protocol"; import { parseGitHubPrUrl, recordAgentPrFromUrl } from "./lib/worker-state"; import { Utils } from "./lib/utils"; import { isCommitAlreadyComplete, runAgentHarnessToFixGitHook } from "./lib/git-hook-fixer"; @@ -950,6 +956,9 @@ if (process.argv[2] === "init") { try { await addressReview(prUrl, { noPush, noReply, verbose }); } catch (error) { + if (exitIfWorkerUsageLimit(error)) { + return; + } const message = error instanceof Error ? error.message : String(error); // Close any run record addressReview opened before it failed (no-op // when none is active — addressReview also ends runs it completes). @@ -1041,6 +1050,9 @@ if (process.argv[2] === "init") { } process.exitCode = result.outcome === "failed" ? 1 : result.outcome === "deferred" ? 2 : 0; } catch (error) { + if (exitIfWorkerUsageLimit(error)) { + return; + } console.error(`❌ Error: ${(error as Error).message}`); // Thrown (unexpected) resolution errors are user actions that failed — // reported like address-review; `failed`/`deferred` outcomes above are @@ -1828,6 +1840,12 @@ async function processSingleTask(taskKey: string, taskIndex = 0, totalTasks = 1) /* ignore */ } } catch (clarityError) { + // Account-global usage limits must abort the run so the worker can + // fail over. Swallowing them here used to launch implementation on + // the same exhausted harness (Grok 402 during the clarity check). + if (clarityError instanceof UsageLimitError) { + throw clarityError; + } recordRunStage("feasibility", { status: "failed", summary: `assessment errored: ${(clarityError as Error).message}`, @@ -1977,6 +1995,30 @@ async function processSingleTask(taskKey: string, taskIndex = 0, totalTasks = 1) // loop aborts the remaining tasks; for a single task, exit 0 (no-op). if (error instanceof UsageLimitError) { await finishTaskRun("deferred", error.message); + if (isWorkerChild()) { + console.warn(`\n⏳ ${error.message}. Signaling worker to fail over.`); + // Hand the ticket back to To Do without a failure comment so the + // incomplete-attempt gate cannot strand it, and the parent can retry + // on the next harness (or pick it up again once a window elapses). + if (activeTaskContext?.movedToInProgress) { + try { + const todoStatus = getTodoStatusForProject( + activeTaskContext.projectKey, + loadProjectSettings(), + ); + if (todoStatus?.trim()) { + await activeTaskContext.tracker.transitionStatus(taskKey, todoStatus.trim()); + } + } catch { + /* best-effort */ + } + } + if (lockManager) { + lockManager.release(); + } + writeUsageLimitHint(error); + process.exit(USAGE_LIMIT_EXIT_CODE); + } console.warn(`\n⏳ ${error.message}. Stopping; will retry on the next scheduled run.`); // The ticket may already be "In Progress": leave feedback and move it // back so the deferred retry can actually pick it up. @@ -2313,6 +2355,14 @@ async function main(): Promise { // batch and exit 0 so the scheduler retries next window. if (error instanceof UsageLimitError) { await finishTaskRun("deferred", error.message); + if (isWorkerChild()) { + console.warn(`\n⏳ ${error.message}. Signaling worker to fail over.`); + if (lockManager) { + lockManager.release(); + } + writeUsageLimitHint(error); + process.exit(USAGE_LIMIT_EXIT_CODE); + } console.warn(`\n⏳ ${error.message}. Aborting estimation batch; will retry next run.`); if (lockManager) { lockManager.release(); @@ -2386,6 +2436,13 @@ async function main(): Promise { // hammering tasks that would all fail. Exit 0 so the scheduler retries // next window without marking the run failed. if (error instanceof UsageLimitError) { + if (isWorkerChild()) { + if (lockManager) { + lockManager.release(); + } + writeUsageLimitHint(error); + process.exit(USAGE_LIMIT_EXIT_CODE); + } const remaining = tasksToProcess.length - i - 1; console.warn( `\n⏳ ${error.message}. Aborting batch — ${remaining} task(s) left, ` + @@ -2586,8 +2643,9 @@ async function runClarityCheck( ); return; } - if (usageLimit?.limited) { - reject(new UsageLimitError(usageLimit.resetsAt)); + const usage = usageLimit ?? detectUsageLimit(stdoutOutput, stderrOutput); + if (usage.limited) { + reject(new UsageLimitError(usage.resetsAt)); return; } if (code === 0) { diff --git a/packages/code/src/lib/address-review.ts b/packages/code/src/lib/address-review.ts index 508722da..79cce3d0 100644 --- a/packages/code/src/lib/address-review.ts +++ b/packages/code/src/lib/address-review.ts @@ -11,6 +11,7 @@ import { spawnAgent, reapTree, resolveExecutablePathWithRetry, + UsageLimitError, } from "@devintern/agent-harness"; import { buildHeadlessAgentArgs, HEADLESS_AGENT_STDIO } from "./agent-spawn"; import { resolveAgentModel } from "./agent-model"; @@ -156,7 +157,7 @@ export async function runAgent( displayName: harness.displayName, }); - return new Promise((resolve) => { + return new Promise((resolve, reject) => { (async () => { // Use high default like regular development (500 turns) const maxTurns = parseInt(process.env.CLAUDE_MAX_TURNS || "500", 10); @@ -241,6 +242,11 @@ export async function runAgent( agent.on("close", (code: number | null) => { clearTimeout(timeout); sandboxCleanup().catch(() => {}); + if (usageLimited) { + const usage = detectUsageLimit(stdoutOutput, stderrOutput); + reject(new UsageLimitError(usage.resetsAt)); + return; + } const maxTurnsReached = detectMaxTurnsReached( stdoutOutput, stderrOutput, @@ -249,12 +255,16 @@ export async function runAgent( const output = stdoutOutput + stderrOutput; resolve({ - success: code === 0 && !maxTurnsReached && !timedOut && !usageLimited, + success: code === 0 && !maxTurnsReached && !timedOut, output: timedOut ? output + `\n\nTimed out after ${timeoutMinutes} minutes` : output, maxTurnsReached, }); }); })().catch((error) => { + if (error instanceof UsageLimitError) { + reject(error); + return; + } resolve({ success: false, output: `Failed to run ${harness.displayName}: ${error instanceof Error ? error.message : String(error)}`, @@ -899,7 +909,11 @@ export async function addressReview( endRun("succeeded"); return; } catch (error) { - endRun("failed", (error as Error).message); + if (error instanceof UsageLimitError) { + endRun("deferred", error.message); + } else { + endRun("failed", (error as Error).message); + } throw error; } finally { // Clean up any untracked files left by linters/tools/agent diff --git a/packages/code/src/lib/automation-acquirer.ts b/packages/code/src/lib/automation-acquirer.ts index 69f4b9f3..bfedbf6d 100644 --- a/packages/code/src/lib/automation-acquirer.ts +++ b/packages/code/src/lib/automation-acquirer.ts @@ -1,7 +1,8 @@ import { randomUUID } from "crypto"; -import { mkdirSync, writeFileSync } from "fs"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs"; import { spawn } from "child_process"; import type { ChildProcess } from "child_process"; +import { tmpdir } from "os"; import { join } from "path"; import { resolveConfigDir } from "@devintern/utils"; @@ -12,6 +13,12 @@ import { nextScheduleOccurrence } from "./automation-config"; import { AutomationStateStore } from "./automation-state"; import { workerTaskArgs } from "./task-polling-acquirer"; import { RUN_ORIGIN_ENV } from "./analytics"; +import { getWorkerFailover } from "./worker-failover"; +import { + readUsageLimitHint, + USAGE_LIMIT_EXIT_CODE, + USAGE_LIMIT_FILE_ENV, +} from "./usage-limit-protocol"; /** Environment markers the task pipeline reads to attribute scheduled runs. */ export const AUTOMATION_ORIGIN_ENV = RUN_ORIGIN_ENV; @@ -627,21 +634,22 @@ export function spawnAutomationProcess( }, ): SpawnedAutomationRun { const detached = process.platform !== "win32"; - const child: ChildProcess = spawn(executable, args, { - cwd: options.cwd, - env: options.env, - stdio: "inherit", - detached, - }); + let child: ChildProcess | null = null; let terminating = false; let exitResult: boolean | undefined; let terminationTimer: ReturnType | undefined; + let hintDir: string | undefined; let settle!: (ok: boolean) => void; const completion = new Promise((resolve) => { settle = resolve; }); + const cleanupHint = () => { + if (!hintDir) return; + rmSync(hintDir, { recursive: true, force: true }); + hintDir = undefined; + }; const kill = (signal: NodeJS.Signals) => { - if (child.pid && detached) { + if (child?.pid && detached) { try { process.kill(-child.pid, signal); return; @@ -650,13 +658,13 @@ export function spawnAutomationProcess( } } try { - child.kill(signal); + child?.kill(signal); } catch { // The child may already have exited. } }; const processGroupAlive = () => { - if (!child.pid || !detached) return false; + if (!child?.pid || !detached) return false; try { process.kill(-child.pid, 0); return true; @@ -670,26 +678,75 @@ export function spawnAutomationProcess( kill("SIGTERM"); terminationTimer = setTimeout(() => { kill("SIGKILL"); + cleanupHint(); settle(false); }, options.terminationGraceMs ?? TERMINATION_GRACE_MS); }; - child.once("close", (code) => { - exitResult = code === 0; - if (!terminating) settle(exitResult); - else if (!processGroupAlive()) { + const start = (env: Record): void => { + const spawned = spawn(executable, args, { + cwd: options.cwd, + env, + stdio: "inherit", + detached, + }); + child = spawned; + spawned.once("close", (code) => { + if (terminating) { + if (!processGroupAlive()) { + if (terminationTimer) clearTimeout(terminationTimer); + cleanupHint(); + settle(false); + } + return; + } + if (code === USAGE_LIMIT_EXIT_CODE) { + const failover = getWorkerFailover(); + if (failover && !failover.allLimited()) { + const outcome = failover.reportFromHint(readUsageLimitHint(env[USAGE_LIMIT_FILE_ENV])); + if (outcome.kind !== "exhausted") { + cleanupHint(); + const nextEnv = pinAutomationEnv(options.env); + hintDir = nextEnv.hintDir; + start(nextEnv.env); + return; + } + } + exitResult = false; + cleanupHint(); + settle(false); + return; + } + exitResult = code === 0; + cleanupHint(); + settle(exitResult); + }); + spawned.once("error", () => { + exitResult = false; if (terminationTimer) clearTimeout(terminationTimer); + cleanupHint(); settle(false); - } - }); - child.once("error", () => { - exitResult = false; - if (terminationTimer) clearTimeout(terminationTimer); - settle(false); - }); + }); + }; + + const initial = pinAutomationEnv(options.env); + hintDir = initial.hintDir; + start(initial.env); return { completion, terminate: beginTermination, }; } + +function pinAutomationEnv(base: Record): { + env: Record; + hintDir?: string; +} { + const failover = getWorkerFailover(); + if (!failover) { + return { env: base }; + } + const hintDir = mkdtempSync(join(tmpdir(), "devintern-usage-limit-")); + return { env: failover.childEnv(base, join(hintDir, "hint.json")), hintDir }; +} diff --git a/packages/code/src/lib/conflict-resolver.ts b/packages/code/src/lib/conflict-resolver.ts index 3e262246..0de83508 100644 --- a/packages/code/src/lib/conflict-resolver.ts +++ b/packages/code/src/lib/conflict-resolver.ts @@ -507,11 +507,17 @@ export async function resolveConflictsOnPr( outcome = "resolved"; } else { console.log(`⚔️ ${conflictedFiles.length} conflicted file(s); handing to the agent`); - const agentResult = await agentRunner( - buildConflictPrompt({ baseRef, branch, conflictedFiles }), - workDir, - verbose, - ); + let agentResult: { success: boolean; output: string }; + try { + agentResult = await agentRunner( + buildConflictPrompt({ baseRef, branch, conflictedFiles }), + workDir, + verbose, + ); + } catch (error) { + await Utils.executeGitCommand(["merge", "--abort"], { cwd: workDir }); + throw error; + } // Trust the tree, not the agent's word: nothing may be left unmerged. const unmerged = await Utils.executeGitCommand( @@ -618,6 +624,9 @@ export async function resolveConflictsOnPr( verbose, ); } catch (error) { + if (error instanceof Error && error.name === "UsageLimitError") { + throw error; + } fixResult = { success: false, output: (error as Error).message }; } // Trust the tree, not the agent's word: fold any leftover changes into diff --git a/packages/code/src/lib/git-hook-fixer.ts b/packages/code/src/lib/git-hook-fixer.ts index d699e4cf..4d05af6e 100644 --- a/packages/code/src/lib/git-hook-fixer.ts +++ b/packages/code/src/lib/git-hook-fixer.ts @@ -10,6 +10,7 @@ import { spawnAgent, reapTree, resolveExecutablePathWithRetry, + UsageLimitError, } from "@devintern/agent-harness"; import type { AgentHarness } from "@devintern/agent-harness"; import { buildHeadlessAgentArgs, HEADLESS_AGENT_STDIO } from "./agent-spawn"; @@ -200,7 +201,7 @@ export async function runAgentHarnessToFixGitHook( displayName: harness.displayName, }); - return new Promise((resolve) => { + return new Promise((resolve, reject) => { (async () => { console.log(`\n🔧 Attempting to fix git hook errors with ${harness.displayName}...`); @@ -341,8 +342,8 @@ ${hookType === "push" ? "- Make sure to amend the commit (git commit --amend --n return; } if (usageLimited) { - console.error(`❌ ${harness.displayName} hit a usage limit while fixing the git hook`); - resolve(false); + const usage = detectUsageLimit(stdoutOutput, stderrOutput); + reject(new UsageLimitError(usage.resetsAt)); return; } if (code === 0) { @@ -423,6 +424,10 @@ ${hookType === "push" ? "- Make sure to amend the commit (git commit --amend --n } }); })().catch((error) => { + if (error instanceof UsageLimitError) { + reject(error); + return; + } console.error( `❌ Failed to run ${harness.displayName} for git hook fix: ${error instanceof Error ? error.message : String(error)}`, ); diff --git a/packages/code/src/lib/harness-failover.ts b/packages/code/src/lib/harness-failover.ts new file mode 100644 index 00000000..597503b1 --- /dev/null +++ b/packages/code/src/lib/harness-failover.ts @@ -0,0 +1,277 @@ +/** + * Harness failover state machine for worker mode. + * + * Given a priority-ordered harness chain (from `AGENT_HARNESS=claude-code,codex`), + * tracks the active harness and per-harness usage-limit windows so the queue + * keeps processing on a fallback when the primary hits its limit, and returns + * to the primary once its window elapses (failback). + * + * This module is the in-memory source of truth. Persistence hooks let the + * webhook server mirror the state into the queue database (`webhook_meta` + * keys, per-harness) so a worker restart recovers the windows and the active + * harness instead of immediately retrying a still-limited agent. + * + * Selection is deterministic: the active harness is always the highest-priority + * (lowest-index) entry whose limit window has elapsed, so failback to the + * primary happens automatically as soon as its window ends. + */ + +import type { HarnessChainEntry } from "@devintern/agent-harness"; + +/** What happened after a harness reported a usage limit. */ +export type FailoverOutcome = + | { + kind: "switched"; + from: string; + to: string; + /** Epoch ms when the from-harness window ends. */ + untilMs: number; + } + | { + kind: "stayed"; + entry: string; + untilMs: number; + } + | { + kind: "exhausted"; + /** Every chain entry is limited; the caller must pause the queue. */ + untilMs: number; + }; + +/** Persistence + clock hooks injected by the host (webhook server / tests). */ +export interface HarnessFailoverOptions { + /** Priority-ordered usable chain entries (from `resolveHarnessChain`). */ + entries: HarnessChainEntry[]; + /** Injectable clock (epoch ms). Defaults to `Date.now`. */ + now?: () => number; + /** Persist a per-harness limit window (epoch ms). */ + persistLimit?: (harness: string, untilMs: number) => void; + /** Remove a harness's persisted limit window. */ + clearPersistedLimit?: (harness: string) => void; + /** Persist the current active harness name. */ + persistActive?: (harness: string) => void; + /** Log line override (tests). Defaults to `console.log`. */ + log?: (message: string) => void; +} + +/** + * Manage the active harness of a chain and its per-harness limit windows. + */ +export class HarnessFailover { + private readonly entries: HarnessChainEntry[]; + private readonly limited = new Map(); + private activeIndex = 0; + private readonly now: () => number; + private readonly persistLimit?: (harness: string, untilMs: number) => void; + private readonly clearPersistedLimit?: (harness: string) => void; + private readonly persistActive?: (harness: string) => void; + private readonly log: (message: string) => void; + + constructor(options: HarnessFailoverOptions) { + if (options.entries.length === 0) { + throw new Error("HarnessFailover requires at least one chain entry"); + } + this.entries = options.entries; + this.now = options.now ?? Date.now; + this.persistLimit = options.persistLimit; + this.clearPersistedLimit = options.clearPersistedLimit; + this.persistActive = options.persistActive; + this.log = options.log ?? ((message: string) => console.log(message)); + } + + /** The currently active chain entry. */ + get active(): HarnessChainEntry { + return this.entries[this.activeIndex]!; + } + + /** Canonical name of the active harness. */ + get activeName(): string { + return this.active.name; + } + + /** Whether the active harness is the first (priority) entry. */ + get onPrimary(): boolean { + return this.activeIndex === 0; + } + + /** Ordered canonical names, for logging ("a → b → c"). */ + describeChain(): string { + return this.entries.map((e) => e.name).join(" → "); + } + + /** + * Snapshot of the tracked per-harness limit windows (epoch ms). + * + * Entries whose window has already ended are left in place until + * {@link windowElapsed} clears them, so a host timer can observe the + * expiration and run the failback. + */ + windows(): Record { + return Object.fromEntries(this.limited); + } + + /** + * Seed state from persisted per-harness windows (queue DB) after a restart. + * + * Future windows for harnesses still in the chain are restored; expired + * windows and windows for harnesses no longer in the chain (list reordered + * or shortened) are dropped from persistence as stale state. Selection + * afterwards is priority-driven: the highest-priority entry without an + * open window becomes active, so a still-limited primary is never retried + * and a fallback from before the restart is kept when it is the best + * available entry. The persisted active name is only used to warn when it + * has left the chain. + * + * @param windows - Persisted harness → limit-until (epoch ms) map + * @param activeName - Persisted active harness name, when known + * @returns Warning lines for stale state the caller should log. + */ + restore(windows: Record, activeName?: string | null): string[] { + const warnings: string[] = []; + const nowMs = this.now(); + + for (const [harness, untilMs] of Object.entries(windows)) { + if (!this.entries.some((e) => e.name === harness)) { + warnings.push( + `Failover state references harness "${harness}", which is not in the current AGENT_HARNESS chain; ignoring its limit window.`, + ); + this.clearPersistedLimit?.(harness); + continue; + } + if (untilMs > nowMs) { + this.limited.set(harness, untilMs); + } else { + // Stale window (already elapsed while the worker was down) — clear it. + this.clearPersistedLimit?.(harness); + } + } + + if (activeName && !this.entries.some((e) => e.name === activeName)) { + warnings.push( + `Persisted active harness "${activeName}" is not in the current AGENT_HARNESS chain; falling back to the highest-priority available entry.`, + ); + } + + const target = this.selectAvailable(); + if (target) { + this.activeIndex = this.entries.indexOf(target); + } else { + // Everything limited: park on the primary; the queue starts paused and + // the resume path reselects when the earliest window elapses. + this.activeIndex = 0; + } + return warnings; + } + + /** Whether the named harness currently has an open limit window. */ + isLimited(harness: string): boolean { + const until = this.limited.get(harness); + return until !== undefined && until > this.now(); + } + + /** + * Record a usage limit for the active harness and fail over when possible. + * + * Extends an existing window (keeps the furthest reset), then switches to + * the highest-priority entry that is not limited. When every entry is + * limited the outcome is `exhausted` and the caller must pause its queue + * until {@link earliestResetMs}. + * + * @param resetUntilMs - Epoch ms when the active harness's window ends + * (already resolved from the reset hint or a fallback cooldown). + * @returns What the failover decided. + */ + reportUsageLimit(resetUntilMs: number): FailoverOutcome { + const from = this.active; + this.setWindow(from.name, resetUntilMs); + + const target = this.selectAvailable(); + if (!target) { + const untilMs = this.earliestResetMs() ?? resetUntilMs; + this.log( + `⛔ All harnesses in the chain are usage-limited (${this.describeChain()}); ` + + `resuming when the earliest window ends at ${new Date(untilMs).toISOString()}.`, + ); + return { kind: "exhausted", untilMs }; + } + + const toIndex = this.entries.indexOf(target); + this.activeIndex = toIndex; + this.persistActive?.(this.activeName); + + if (target.name !== from.name) { + const fallbackPosition = + toIndex === 0 ? "primary" : `fallback ${toIndex + 1}/${this.entries.length}`; + this.log( + `🔁 ${from.name} hit a usage limit (until ${new Date(resetUntilMs).toISOString()}) — ` + + `failing over to ${target.name} (${fallbackPosition} in the AGENT_HARNESS chain).`, + ); + return { kind: "switched", from: from.name, to: target.name, untilMs: resetUntilMs }; + } + + return { kind: "stayed", entry: target.name, untilMs: resetUntilMs }; + } + + /** + * Clear an elapsed limit window and fail back when that unlocks a + * higher-priority harness. + * + * Called by the host timer when a persisted window ends. If the primary + * (priority) harness becomes available again while a fallback is active, + * the active harness returns to it — that is the failback. + * + * @param harness - Harness whose window elapsed. + * @returns The harness now active, if a switch happened; otherwise null. + */ + windowElapsed(harness: string): string | null { + const hadWindow = this.limited.delete(harness); + if (hadWindow) { + this.clearPersistedLimit?.(harness); + } + + const target = this.selectAvailable(); + if (!target || target.name === this.activeName) { + return null; + } + + const previous = this.active; + this.activeIndex = this.entries.indexOf(target); + this.persistActive?.(this.activeName); + this.log( + target.name === this.entries[0]!.name + ? `⏪ ${harness} usage-limit window elapsed — failing back to primary harness ${target.name} (from ${previous.name}).` + : `🔁 ${harness} usage-limit window elapsed — resuming on ${target.name} (from ${previous.name}).`, + ); + return target.name; + } + + /** + * Epoch ms when the earliest open window ends (null when none open). + * + * Always in the future when non-null; expired-but-uncleaned windows are + * ignored. The host arms its resume/failback timer at this instant. + */ + earliestResetMs(): number | null { + const nowMs = this.now(); + const open = [...this.limited.values()].filter((until) => until > nowMs); + return open.length > 0 ? Math.min(...open) : null; + } + + /** Whether every chain entry currently has an open limit window. */ + allLimited(): boolean { + return this.entries.every((e) => this.isLimited(e.name)); + } + + /** Highest-priority entry without an open window, or null when all limited. */ + private selectAvailable(): HarnessChainEntry | null { + return this.entries.find((e) => !this.isLimited(e.name)) ?? null; + } + + /** Store a window, keeping the furthest reset when one is already open. */ + private setWindow(harness: string, untilMs: number): void { + const existing = this.limited.get(harness) ?? 0; + const until = Math.max(existing, untilMs); + this.limited.set(harness, until); + this.persistLimit?.(harness, until); + } +} diff --git a/packages/code/src/lib/relay-acquirer.ts b/packages/code/src/lib/relay-acquirer.ts index 559b5a74..be29c9cb 100644 --- a/packages/code/src/lib/relay-acquirer.ts +++ b/packages/code/src/lib/relay-acquirer.ts @@ -38,8 +38,9 @@ export interface RelayHandlers { * Review submitted on one of the agent's own PRs → address it. * @returns Whether the run completed; `false` means it failed or matched * no workspace repo (never silently swallowed by the caller). + * `"deferred"` means every harness is usage-limited. */ - addressPr(repo: string, prNumber: number): Promise; + addressPr(repo: string, prNumber: number): Promise; /** New PR conversation comment → mention/permission gates decide inside. */ handlePrComment(repo: string, prNumber: number, commentId: number): Promise; /** Tracker task changed → re-evaluate the matching team/default source. */ @@ -187,11 +188,17 @@ export class RelayAcquirer implements Acquirer { } console.log(`📌 [relay] review feedback on ${repo}#${pr}`); const ok = await handlers.addressPr(repo, pr); - console.log( - ok - ? `✅ [relay] ${repo}#${pr} feedback addressed` - : `⚠️ [relay] ${repo}#${pr} feedback run did not complete cleanly`, - ); + if (ok === "deferred") { + console.log( + `⏳ [relay] ${repo}#${pr} deferred; will retry when a harness is available`, + ); + } else { + console.log( + ok + ? `✅ [relay] ${repo}#${pr} feedback addressed` + : `⚠️ [relay] ${repo}#${pr} feedback run did not complete cleanly`, + ); + } return; } case "pr.comment_created": { diff --git a/packages/code/src/lib/review-polling-acquirer.ts b/packages/code/src/lib/review-polling-acquirer.ts index 0d4ed4d6..8b769be8 100644 --- a/packages/code/src/lib/review-polling-acquirer.ts +++ b/packages/code/src/lib/review-polling-acquirer.ts @@ -27,6 +27,8 @@ import { spawn } from "child_process"; import { captureError } from "@devintern/utils"; +import { parseHarnessList } from "@devintern/agent-harness"; + import { agentPrKey, agentPrStateCursorSource, @@ -38,7 +40,9 @@ import { nextScheduleOccurrence } from "./automation-config"; import type { CronOrIntervalSchedule } from "./automation-config"; import { parseEnvInteger } from "./env-integer"; import type { RunStore } from "./run-recorder"; +import type { TaskExecutionResult } from "./task-polling-acquirer"; import type { WebhookQueue } from "./webhook-queue"; +import { cliResultToTaskResult, runWithFailover } from "./worker-failover"; import type { WorkerState } from "./worker-state"; import type { ConflictResolutionMode } from "./workspace/config"; import type { Acquirer } from "../worker"; @@ -89,7 +93,7 @@ export interface ReviewPollingAcquirerOptions { queue: WebhookQueue; github: ReviewPollingGitHub; /** Handle feedback on one PR; returns success (injected for tests). */ - addressPr: (repo: string, prNumber: number) => Promise; + addressPr: (repo: string, prNumber: number) => Promise; /** * Resolve merge conflicts on one of the agent's own PRs (injected for * tests). Omit to disable automatic conflict resolution. @@ -262,7 +266,7 @@ export function runAddressReviewViaCli( cwd?: string; env?: Record; } = {}, -): Promise { +): Promise { return serializePrRun(repo, prNumber, () => runSubcommandViaCli("address-review", repo, prNumber, opts), ); @@ -309,6 +313,53 @@ function runResolveSubcommand( outputStdio?: "inherit" | "ignore"; }, ): Promise { + return runResolveWithFailover(repo, prNumber, extraArgs, opts); +} + +async function runResolveWithFailover( + repo: string, + prNumber: number, + extraArgs: string[], + opts: { + cwd?: string; + env?: Record; + timeoutMs?: number; + entrypoint?: string; + outputStdio?: "inherit" | "ignore"; + }, +): Promise { + let last: AutomaticResolveResult | null = null; + const status = await runWithFailover(async (env) => { + const spawned = await spawnResolveOnce(repo, prNumber, extraArgs, { ...opts, env }); + last = spawned.result; + return spawned.code; + }, opts.env ?? process.env); + if (status === "deferred") { + return { + outcome: "deferred", + message: "agent usage limit; waiting for a harness to become available", + }; + } + return ( + last ?? { + outcome: status === "ok" ? "skipped" : "failed", + message: status === "ok" ? "resolver completed" : "resolver exited with a non-zero code", + } + ); +} + +function spawnResolveOnce( + repo: string, + prNumber: number, + extraArgs: string[], + opts: { + cwd?: string; + env?: Record; + timeoutMs?: number; + entrypoint?: string; + outputStdio?: "inherit" | "ignore"; + }, +): Promise<{ code: number; result: AutomaticResolveResult }> { const prUrl = `https://github.com/${repo}/pull/${prNumber}`; return new Promise((resolve) => { let result: AutomaticResolveResult | null = null; @@ -351,7 +402,10 @@ function runResolveSubcommand( child.on("close", (code) => { if (timer) clearTimeout(timer); if (timedOut) { - resolve({ outcome: "failed", message: `resolver timed out after ${timeoutMs}ms` }); + resolve({ + code: 1, + result: { outcome: "failed", message: `resolver timed out after ${timeoutMs}ms` }, + }); return; } if (!resultOverflow) { @@ -371,8 +425,9 @@ function runResolveSubcommand( } } } - resolve( - result ?? { + resolve({ + code: code ?? 1, + result: result ?? { outcome: code === 0 ? "skipped" : code === 2 ? "deferred" : "failed", message: code === 0 @@ -381,16 +436,19 @@ function runResolveSubcommand( ? "resolver deferred" : `resolver exited with code ${code}`, }, - ); + }); }); child.on("error", (error) => { if (timer) clearTimeout(timer); - resolve({ outcome: "failed", message: `failed to spawn resolver: ${error.message}` }); + resolve({ + code: 1, + result: { outcome: "failed", message: `failed to spawn resolver: ${error.message}` }, + }); }); }); } -function runSubcommandViaCli( +async function runSubcommandViaCli( subcommand: string, repo: string, prNumber: number, @@ -398,21 +456,26 @@ function runSubcommandViaCli( cwd?: string; env?: Record; } = {}, -): Promise { +): Promise { const prUrl = `https://github.com/${repo}/pull/${prNumber}`; - return new Promise((resolve) => { - const child = spawn(process.execPath, [process.argv[1], subcommand, prUrl], { - stdio: ["inherit", "inherit", "inherit"], - cwd: opts.cwd, - env: opts.env ?? process.env, - }); - child.on("close", (code) => resolve(code === 0)); - child.on("error", (error) => { - captureError(error, { command: subcommand, repo, prNumber, stage: "spawn" }); - console.error(`❌ Failed to spawn ${subcommand} for ${prUrl}: ${error.message}`); - resolve(false); - }); - }); + const status = await runWithFailover( + (env) => + new Promise((resolve) => { + const child = spawn(process.execPath, [process.argv[1], subcommand, prUrl], { + stdio: ["inherit", "inherit", "inherit"], + cwd: opts.cwd, + env, + }); + child.on("close", (code) => resolve(code ?? 1)); + child.on("error", (error) => { + captureError(error, { command: subcommand, repo, prNumber, stage: "spawn" }); + console.error(`❌ Failed to spawn ${subcommand} for ${prUrl}: ${error.message}`); + resolve(1); + }); + }), + opts.env ?? process.env, + ); + return cliResultToTaskResult(status); } /** @@ -673,11 +736,17 @@ export class ReviewPollingAcquirer implements Acquirer { console.log(`\n📌 [${this.name}] new review feedback on ${repo}#${prNumber}`); const ok = await addressPr(repo, prNumber); - console.log( - ok - ? `✅ [${this.name}] ${repo}#${prNumber} feedback addressed` - : `⚠️ [${this.name}] ${repo}#${prNumber} feedback run did not complete cleanly`, - ); + if (ok === "deferred") { + console.log( + `⏳ [${this.name}] ${repo}#${prNumber} deferred; will retry when a harness is available`, + ); + } else { + console.log( + ok + ? `✅ [${this.name}] ${repo}#${prNumber} feedback addressed` + : `⚠️ [${this.name}] ${repo}#${prNumber} feedback run did not complete cleanly`, + ); + } } private async maybeSyncBase(repo: string, prNumber: number, pr: PolledPr): Promise { @@ -781,7 +850,7 @@ export class ReviewPollingAcquirer implements Acquirer { prNumber, prUrl: `https://github.com/${repo}/pull/${prNumber}`, branch: fresh.head.ref, - harness: this.options.harness ?? process.env.AGENT_HARNESS ?? "claude-code", + harness: this.options.harness ?? parseHarnessList(process.env.AGENT_HARNESS)[0], attempt, }) ?? null; } catch (error) { diff --git a/packages/code/src/lib/task-polling-acquirer.ts b/packages/code/src/lib/task-polling-acquirer.ts index 8c38c9ac..83117528 100644 --- a/packages/code/src/lib/task-polling-acquirer.ts +++ b/packages/code/src/lib/task-polling-acquirer.ts @@ -24,6 +24,7 @@ import { TASK_POLL_LAST_DRAIN_KEY } from "./worker-state"; import type { WebhookQueue } from "./webhook-queue"; import type { WorkerState } from "./worker-state"; import type { Acquirer } from "../worker"; +import { cliResultToTaskResult, runWithFailover } from "./worker-failover"; export interface ReadyTask { key: string; @@ -79,25 +80,31 @@ export function workerTaskArgs(): string[] { * @param opts - Working directory and environment for the subprocess; * the workspace worker routes each task to its repo's worktree * with per-repo env; direct callers inherit both - * @returns true when the CLI exited 0 + * @returns true when the CLI exited 0, `"deferred"` when every harness in + * the failover chain is usage-limited, false on any other failure */ -export function runTaskViaCli( +export async function runTaskViaCli( taskKey: string, extraArgs: string[] = workerTaskArgs(), opts: { cwd?: string; env?: Record } = {}, -): Promise { - return new Promise((resolve) => { - const child = spawn(process.execPath, [process.argv[1], taskKey, ...extraArgs], { - stdio: "inherit", - cwd: opts.cwd, - env: opts.env ?? process.env, - }); - child.on("close", (code) => resolve(code === 0)); - child.on("error", (error) => { - console.error(`❌ Failed to spawn task run for ${taskKey}: ${error.message}`); - resolve(false); - }); - }); +): Promise { + const result = await runWithFailover( + (env) => + new Promise((resolve) => { + const child = spawn(process.execPath, [process.argv[1], taskKey, ...extraArgs], { + stdio: "inherit", + cwd: opts.cwd, + env, + }); + child.on("close", (code) => resolve(code ?? 1)); + child.on("error", (error) => { + console.error(`❌ Failed to spawn task run for ${taskKey}: ${error.message}`); + resolve(1); + }); + }), + opts.env ?? process.env, + ); + return cliResultToTaskResult(result); } /** diff --git a/packages/code/src/lib/usage-limit-protocol.ts b/packages/code/src/lib/usage-limit-protocol.ts new file mode 100644 index 00000000..273b3d96 --- /dev/null +++ b/packages/code/src/lib/usage-limit-protocol.ts @@ -0,0 +1,109 @@ +/** + * Protocol between a long-running worker and the CLI subprocesses it spawns. + * + * Fleet/polling/review/automation runs execute `devintern` as a child. When + * that child hits a usage limit it must not look like a generic failure + * (exit 1 + failure comment) — the parent owns failover and needs a distinct + * signal plus the parsed reset window. + * + * Exit 75 is EX_TEMPFAIL (sysexits): "temporary failure, try again later". + */ + +import { readFileSync, writeFileSync } from "fs"; + +import { resetHintToMs, UsageLimitError } from "@devintern/agent-harness"; + +/** sysexits EX_TEMPFAIL — the worker retries on the next harness. */ +export const USAGE_LIMIT_EXIT_CODE = 75; + +/** Set to `"1"` on CLI subprocesses the worker will fail over for. */ +export const WORKER_CHILD_ENV = "DEVINTERN_WORKER_CHILD"; + +/** Absolute path the child writes a JSON usage-limit hint into. */ +export const USAGE_LIMIT_FILE_ENV = "DEVINTERN_USAGE_LIMIT_FILE"; + +/** Fallback cooldown when the reset hint cannot be parsed. */ +export const RATE_LIMIT_FALLBACK_MS = 60 * 60 * 1000; + +export interface UsageLimitHint { + /** Epoch ms when the limited harness's window ends. */ + untilMs: number; + /** Human-readable reset hint from the agent output, when present. */ + resetsAt?: string; +} + +/** True when this process was spawned by the worker for failover. */ +export function isWorkerChild(): boolean { + return process.env[WORKER_CHILD_ENV] === "1"; +} + +/** + * Persist the parsed reset window so the parent can fail over without + * re-scanning the child's inherited stdio. + * + * @param error - Usage-limit error raised by the agent spawn + */ +export function writeUsageLimitHint(error: UsageLimitError): void { + const path = process.env[USAGE_LIMIT_FILE_ENV]; + if (!path) { + return; + } + const untilMs = resetHintToMs(error.resetHint, Date.now()) ?? Date.now() + RATE_LIMIT_FALLBACK_MS; + writeFileSync( + path, + JSON.stringify({ + resetsAt: error.resetHint ?? null, + untilMs, + }), + ); +} + +/** + * Read a hint file written by {@link writeUsageLimitHint}. + * + * Missing, empty, or malformed files fall back to a 1-hour cooldown so a + * crashed child still trips failover instead of spinning. + * + * @param path - Hint file path (from {@link USAGE_LIMIT_FILE_ENV}) + */ +export function readUsageLimitHint(path: string | undefined): UsageLimitHint { + const fallback = Date.now() + RATE_LIMIT_FALLBACK_MS; + if (!path) { + return { untilMs: fallback }; + } + try { + const raw = readFileSync(path, "utf8").trim(); + if (!raw) { + return { untilMs: fallback }; + } + const parsed = JSON.parse(raw) as { untilMs?: unknown; resetsAt?: unknown }; + const resetsAt = typeof parsed.resetsAt === "string" ? parsed.resetsAt : undefined; + const parsedUntil = typeof parsed.untilMs === "number" ? parsed.untilMs : NaN; + const untilMs = + Number.isFinite(parsedUntil) && parsedUntil > Date.now() + ? parsedUntil + : (resetHintToMs(resetsAt, Date.now()) ?? fallback); + return { untilMs, resetsAt }; + } catch { + return { untilMs: fallback }; + } +} + +/** + * If this is a worker child and `error` is a usage limit, write the hint + * and exit 75. Returns false so callers can continue with one-shot handling. + * + * @param error - Caught error from an agent run + * @returns Always false when it does not exit + */ +export function exitIfWorkerUsageLimit(error: unknown): boolean { + if (!(error instanceof UsageLimitError)) { + return false; + } + if (!isWorkerChild()) { + return false; + } + console.warn(`\n⏳ ${error.message}. Signaling worker to fail over.`); + writeUsageLimitHint(error); + process.exit(USAGE_LIMIT_EXIT_CODE); +} diff --git a/packages/code/src/lib/webhook-queue.ts b/packages/code/src/lib/webhook-queue.ts index 55bfe9c8..1dd50b73 100644 --- a/packages/code/src/lib/webhook-queue.ts +++ b/packages/code/src/lib/webhook-queue.ts @@ -317,6 +317,57 @@ export class WebhookQueue { return Number.isFinite(ms) ? ms : null; } + /** Every persisted per-harness rate-limit window, keyed by harness name. */ + getAllRateLimits(): Record { + const rows = this.db + .query(`SELECT key, value FROM webhook_meta WHERE key LIKE 'rate_limit:%'`) + .all() as { key: string; value: string }[]; + const result: Record = {}; + for (const row of rows) { + const ms = Number(row.value); + if (Number.isFinite(ms)) { + result[row.key.slice("rate_limit:".length)] = ms; + } + } + return result; + } + + /** Meta key for the failover chain's active harness. */ + private activeHarnessKey(): string { + return "failover:active_harness"; + } + + /** + * Persist the failover chain's active harness so a worker restart resumes + * on it instead of snapping back to the primary mid-window. + * + * @param harness - Canonical harness name (e.g. `codex`) + */ + setActiveHarness(harness: string): void { + this.db.run( + `INSERT INTO webhook_meta (key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + [this.activeHarnessKey(), harness], + ); + } + + /** Forget the persisted active harness (e.g. when the chain changed). */ + clearActiveHarness(): void { + this.db.run(`DELETE FROM webhook_meta WHERE key = ?`, [this.activeHarnessKey()]); + } + + /** + * Read the persisted active harness of the failover chain. + * + * @returns Canonical harness name, or `null` when none was persisted. + */ + getActiveHarness(): string | null { + const row = this.db + .query(`SELECT value FROM webhook_meta WHERE key = ?`) + .get(this.activeHarnessKey()) as { value: string } | undefined; + return row?.value ?? null; + } + /** * Check whether a provider-issued event id was already handled. * diff --git a/packages/code/src/lib/worker-failover.ts b/packages/code/src/lib/worker-failover.ts new file mode 100644 index 00000000..d46cb1e2 --- /dev/null +++ b/packages/code/src/lib/worker-failover.ts @@ -0,0 +1,314 @@ +/** + * Shared harness failover for every long-running worker surface. + * + * Webhook serve, fleet polling, PR mentions, review addressing, conflict + * resolution, scheduled automations, and estimations all share one + * {@link HarnessFailover} instance so a usage limit on Codex fails over to + * Grok (etc.) instead of only working inside `devintern webhook serve`. + * + * CLI subprocesses are pinned to the active harness via `AGENT_HARNESS` and + * signal a limit with {@link USAGE_LIMIT_EXIT_CODE}; this module retries the + * same spawn on the next chain entry, or reports `deferred` when the chain + * is exhausted. + */ + +import { mkdtempSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +import { resetHintToMs, resolveHarnessChain } from "@devintern/agent-harness"; +import type { + HarnessChainEntry, + ResolvedHarness, + ResolvedHarnessChain, +} from "@devintern/agent-harness"; + +import { HarnessFailover } from "./harness-failover"; +import type { FailoverOutcome } from "./harness-failover"; +import type { WebhookQueue } from "./webhook-queue"; +import { + RATE_LIMIT_FALLBACK_MS, + readUsageLimitHint, + USAGE_LIMIT_EXIT_CODE, + USAGE_LIMIT_FILE_ENV, + WORKER_CHILD_ENV, +} from "./usage-limit-protocol"; +import type { UsageLimitHint } from "./usage-limit-protocol"; + +export type CliFailoverResult = "ok" | "failed" | "deferred"; + +export interface WorkerFailoverOptions { + /** Queue used to persist windows and the active harness across restarts. */ + queue?: WebhookQueue | null; + /** + * Probe installability at startup (default true). Tests that do not want + * PATH checks pass false, matching webhook-server's lazy fallback. + */ + checkInstalled?: boolean; + /** Override `AGENT_HARNESS` (tests). */ + raw?: string; + /** Called when every chain entry is limited (pause new agent work). */ + onPause?: (info: { untilMs: number; harness: string; resetHint?: string }) => void; + /** Called when a window elapses and at least one harness is available. */ + onResume?: () => void; + log?: (message: string) => void; +} + +/** + * In-process failover controller: chain resolution, persistence, failback + * timers, and child-env pinning. + */ +export class WorkerFailover { + readonly chain: ResolvedHarnessChain; + readonly manager: HarnessFailover; + private timer: ReturnType | null = null; + private paused = false; + private readonly onPause?: WorkerFailoverOptions["onPause"]; + private readonly onResume?: WorkerFailoverOptions["onResume"]; + private readonly log: (message: string) => void; + + constructor(options: WorkerFailoverOptions = {}) { + this.chain = resolveHarnessChain({ + checkInstalled: options.checkInstalled ?? true, + raw: options.raw, + }); + this.onPause = options.onPause; + this.onResume = options.onResume; + this.log = options.log ?? ((message) => console.log(message)); + + const queue = options.queue; + this.manager = new HarnessFailover({ + entries: this.chain.entries, + persistLimit: queue ? (harness, untilMs) => queue.setRateLimit(harness, untilMs) : undefined, + clearPersistedLimit: queue ? (harness) => queue.clearRateLimit(harness) : undefined, + persistActive: queue ? (harness) => queue.setActiveHarness(harness) : undefined, + log: this.log, + }); + + if (queue) { + for (const warning of this.manager.restore( + queue.getAllRateLimits(), + queue.getActiveHarness(), + )) { + console.warn(`⚠️ ${warning}`); + } + } + } + + /** Canonical name of the active harness. */ + get activeName(): string { + return this.manager.activeName; + } + + /** Active chain entry. */ + get active(): HarnessChainEntry { + return this.manager.active; + } + + /** Resolved harness + path for in-process spawns (webhook serve). */ + resolvedHarness(): ResolvedHarness { + return { harness: this.active.harness, path: this.active.path }; + } + + describeChain(): string { + return this.manager.describeChain(); + } + + windows(): Record { + return this.manager.windows(); + } + + allLimited(): boolean { + return this.manager.allLimited(); + } + + /** Startup banner line (`Agent harness: a → b (failover enabled)`). */ + describeStartup(): string { + return `Agent harness: ${this.manager.describeChain()}${ + this.chain.multiHarness ? " (failover enabled)" : "" + }`; + } + + /** + * Environment overlay for a CLI child: pin `AGENT_HARNESS` to the active + * entry so one-shot resolution uses that harness, and point at a hint file. + */ + childEnv( + base: Record, + hintPath: string, + ): Record { + return { + ...base, + AGENT_HARNESS: this.manager.activeName, + [WORKER_CHILD_ENV]: "1", + [USAGE_LIMIT_FILE_ENV]: hintPath, + }; + } + + /** + * Record a usage limit and fail over. Pauses via {@link onPause} when the + * chain is exhausted; always rearms the failback timer. + */ + reportFromHint(hint?: Partial & { resetsAt?: string }): FailoverOutcome { + const until = + hint?.untilMs && hint.untilMs > Date.now() + ? hint.untilMs + : (resetHintToMs(hint?.resetsAt, Date.now()) ?? Date.now() + RATE_LIMIT_FALLBACK_MS); + const harness = this.manager.activeName; + const outcome = this.manager.reportUsageLimit(until); + if (outcome.kind === "exhausted") { + this.paused = true; + this.onPause?.({ untilMs: outcome.untilMs, harness, resetHint: hint?.resetsAt }); + } + this.armTimers(); + return outcome; + } + + /** Start (or restart) the failback/resume timer at the earliest window. */ + armTimers(): void { + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + const nextMs = this.manager.earliestResetMs(); + if (nextMs === null) { + return; + } + this.timer = setTimeout( + () => { + this.timer = null; + const nowMs = Date.now(); + for (const [harness, until] of Object.entries(this.manager.windows())) { + if (until <= nowMs) { + this.manager.windowElapsed(harness); + } + } + if (this.paused && !this.manager.allLimited()) { + this.paused = false; + this.onResume?.(); + } + this.armTimers(); + }, + Math.max(0, nextMs - Date.now()), + ); + } + + stop(): void { + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + } + + /** + * After restoring persisted windows, pause immediately when every entry is + * still limited instead of recording a fresh limit on the active harness. + */ + pauseIfExhaustedOnRestore(): void { + if (this.manager.allLimited()) { + this.paused = true; + const untilMs = this.manager.earliestResetMs() ?? Date.now(); + this.onPause?.({ untilMs, harness: this.manager.activeName }); + } + this.armTimers(); + } + + /** Warn about dropped chain entries and log the effective chain. */ + announceStartup(): void { + for (const issue of this.chain.issues) { + console.warn(`⚠️ ${issue.message}`); + } + this.log(` ${this.describeStartup()}`); + } +} + +let instance: WorkerFailover | null = null; + +/** + * Initialize process-wide failover. Replaces any previous instance (tests). + * + * @param options - Persistence, pause/resume hooks, and installability + */ +export function startWorkerFailover(options: WorkerFailoverOptions = {}): WorkerFailover { + instance?.stop(); + instance = new WorkerFailover(options); + instance.announceStartup(); + instance.pauseIfExhaustedOnRestore(); + return instance; +} + +/** The process-wide controller, or null when failover has not been started. */ +export function getWorkerFailover(): WorkerFailover | null { + return instance; +} + +/** + * Lazy fallback for in-process callers (webhook-server tests) that spawn + * before {@link startWorkerFailover}. Does not persist or probe PATH. + */ +export function ensureWorkerFailover(): WorkerFailover { + if (instance) { + return instance; + } + instance = new WorkerFailover({ checkInstalled: false }); + return instance; +} + +/** Drop the process-wide controller (tests). */ +export function resetWorkerFailover(): void { + instance?.stop(); + instance = null; +} + +/** + * Run a CLI child, retrying on usage-limit exit 75 until the chain is + * exhausted. When failover has not been started, the spawn runs once with + * the caller's env (one-shot / tests). + * + * @param spawnOnce - Spawn the child with the given env; return its exit code + * @param baseEnv - Environment to pin the active harness onto + */ +export async function runWithFailover( + spawnOnce: (env: Record) => Promise, + baseEnv: Record = { ...process.env }, +): Promise { + const failover = getWorkerFailover(); + if (!failover) { + const code = await spawnOnce(baseEnv); + return code === 0 ? "ok" : "failed"; + } + + const attempts = Math.max(1, failover.chain.entries.length); + for (let attempt = 0; attempt < attempts; attempt++) { + if (failover.allLimited()) { + return "deferred"; + } + const hintDir = mkdtempSync(join(tmpdir(), "devintern-usage-limit-")); + const hintPath = join(hintDir, "hint.json"); + try { + const code = await spawnOnce(failover.childEnv(baseEnv, hintPath)); + if (code !== USAGE_LIMIT_EXIT_CODE) { + return code === 0 ? "ok" : "failed"; + } + const hint = readUsageLimitHint(hintPath); + const outcome = failover.reportFromHint(hint); + if (outcome.kind === "exhausted") { + return "deferred"; + } + } finally { + rmSync(hintDir, { recursive: true, force: true }); + } + } + return "deferred"; +} + +/** Map {@link runWithFailover} onto the poller's `boolean | "deferred"` result. */ +export function cliResultToTaskResult(result: CliFailoverResult): boolean | "deferred" { + if (result === "ok") { + return true; + } + if (result === "deferred") { + return "deferred"; + } + return false; +} diff --git a/packages/code/src/lib/workspace/fleet-events.ts b/packages/code/src/lib/workspace/fleet-events.ts index 096672eb..2265482d 100644 --- a/packages/code/src/lib/workspace/fleet-events.ts +++ b/packages/code/src/lib/workspace/fleet-events.ts @@ -12,6 +12,7 @@ import { runAddressReviewViaCli, runResolveConflictsViaCli } from "../review-polling-acquirer"; import type { AutomaticResolveResult } from "../review-polling-acquirer"; +import type { TaskExecutionResult } from "../task-polling-acquirer"; import type { RepoConfig, WorkspaceConfig } from "./config"; import { buildRepoEnv, gitHubSlugFromRemote } from "./env"; import { toRoutableTask } from "./router"; @@ -32,7 +33,7 @@ export interface FleetEventDeps { cwd: string; env: Record; }, - ) => Promise; + ) => Promise; /** Base-sync runner (injected for tests; defaults to the CLI subprocess). */ runResolve?: typeof runResolveConflictsViaCli; verbose?: boolean; @@ -40,16 +41,23 @@ export interface FleetEventDeps { coordinator?: RunCoordinator; } -type AddressPr = (slug: string, prNumber: number) => Promise; +type AddressPr = (slug: string, prNumber: number) => Promise; /** * Serialize feedback handling per PR and collapse events received while a run * is active into one follow-up reconciliation. Relay, review polling, and * mention sweeping can all observe the same GitHub action; the follow-up run * re-fetches feedback after the active run has persisted its addressed marks. + * + * A `false` outcome from any run wins (a run did not complete cleanly); + * otherwise the last outcome is reported, so a follow-up run that succeeded + * clears an earlier `"deferred"`. */ export function coalescePrFeedbackRuns(addressPr: AddressPr): AddressPr { - const active = new Map }>(); + const active = new Map< + string, + { rerunRequested: boolean; promise: Promise } + >(); return async (slug, prNumber) => { const key = `${slug.toLowerCase()}#${prNumber}`; @@ -59,13 +67,21 @@ export function coalescePrFeedbackRuns(addressPr: AddressPr): AddressPr { return existing.promise; } - const state = { rerunRequested: false, promise: Promise.resolve(false) }; + const state = { + rerunRequested: false, + promise: Promise.resolve(false as TaskExecutionResult), + }; state.promise = (async () => { try { - let ok = true; + let ok: TaskExecutionResult = true; do { state.rerunRequested = false; - ok = (await addressPr(slug, prNumber)) && ok; + const outcome = await addressPr(slug, prNumber); + if (outcome === false || ok === false) { + ok = false; + } else { + ok = outcome; + } } while (state.rerunRequested); return ok; } finally { diff --git a/packages/code/src/lib/workspace/workspace-worker.ts b/packages/code/src/lib/workspace/workspace-worker.ts index 6254ea93..e413a739 100644 --- a/packages/code/src/lib/workspace/workspace-worker.ts +++ b/packages/code/src/lib/workspace/workspace-worker.ts @@ -53,6 +53,7 @@ import { EstimationAcquirer } from "../estimation-acquirer"; import { RunCoordinator } from "../run-coordinator"; import type { AutomationRunContext } from "../automation-acquirer"; import { flushAnalytics, RUN_ORIGIN_ENV, trackWorkerStarted } from "../analytics"; +import { startWorkerFailover } from "../worker-failover"; import { RetryQueueAcquirer } from "./retry-acquirer"; /** Orphaned-run feedback cutoff: `WORKER_ORPHAN_MAX_AGE_HOURS`, default 7 days. */ @@ -188,7 +189,7 @@ export interface WorkspaceTaskAcquirerDeps { taskKey: string, extraArgs: string[], opts: { cwd: string; env: Record }, - ) => Promise; + ) => Promise; /** Repo run lock factory (injected for tests). */ repoLock?: (repoName: string) => LockManager; /** Process-level agent-run gate; only set when scheduled estimation exists. */ @@ -517,7 +518,7 @@ export function createFleetTaskExecutor( }); const ok = deps.coordinator ? await deps.coordinator.run(invoke) : await invoke(); - if (ok) { + if (ok === true) { await repoManager.removeTaskWorktree(repo.name, worktree); } else { console.warn(`⚠️ ${scope} keeping worktree for debugging: ${worktree}`); @@ -718,6 +719,18 @@ export async function runWorkspaceWorker(options: RunWorkspaceWorkerOptions): Pr } const state = openWorkspaceState(workspaceDir); + startWorkerFailover({ + queue: state.queue, + onPause: ({ untilMs, harness, resetHint }) => { + console.warn( + `⏳ ${harness} hit a usage limit${resetHint ? ` (resets ${resetHint})` : ""} and no fallback harness is available. ` + + `Deferring new agent work until ${new Date(untilMs).toISOString()}.`, + ); + }, + onResume: () => { + console.log("▶️ Usage-limit windows elapsed — resuming agent work on the available harness"); + }, + }); const repoManager = new RepoManager(workspaceDir); // Preserve the worker's existing concurrency when scheduled estimation is // absent or fully disabled. The account-global gate is needed only once an @@ -1125,7 +1138,7 @@ export async function buildFleetEventAcquirers(options: { : Boolean(process.env.GITHUB_TOKEN || hasCustomAppCredentials); const slugs = fleetGitHubSlugs(config); let github: import("../github-reviews").GitHubReviewsClient | undefined; - let addressPr: ((repo: string, prNumber: number) => Promise) | undefined; + let addressPr: ((repo: string, prNumber: number) => Promise) | undefined; let handleMention: | ((repo: string, comment: { user: { login: string } }, prNumber: number) => Promise) | undefined; diff --git a/packages/code/src/webhook-server.ts b/packages/code/src/webhook-server.ts index 670b9b4e..dc06e9ec 100644 --- a/packages/code/src/webhook-server.ts +++ b/packages/code/src/webhook-server.ts @@ -16,13 +16,12 @@ import { detectMaxTurnsReached, findMaxTurnsReachedLine, detectUsageLimit, - resetHintToMs, - resolveHarness, spawnAgent, reapTree, resolveExecutablePathWithRetry, UsageLimitError, } from "@devintern/agent-harness"; +import type { ResolvedHarness } from "@devintern/agent-harness"; import { buildHeadlessAgentArgs, HEADLESS_AGENT_STDIO } from "./lib/agent-spawn"; import { parseEnvInteger } from "./lib/env-integer"; import { resolveAgentModel } from "./lib/agent-model"; @@ -32,6 +31,8 @@ import { captureError, flushErrorTracking } from "@devintern/utils"; import { GitHubAppAuth } from "./lib/github-app-auth"; import { GitHubReviewsClient } from "./lib/github-reviews"; import { LEGACY_DB_PATH, WebhookQueue, resolveQueueDbPath } from "./lib/webhook-queue"; +import { ensureWorkerFailover, startWorkerFailover } from "./lib/worker-failover"; +import type { WorkerFailover } from "./lib/worker-failover"; import { WorkerState } from "./lib/worker-state"; import { formatReviewPrompt } from "./lib/review-formatter"; import { Utils } from "./lib/utils"; @@ -76,71 +77,61 @@ const reviewQueue = new PQueue({ concurrency: 1 }); // Persistent webhook queue (initialized in startWebhookServer) let webhookQueue: WebhookQueue | null = null; -// Fallback cooldown when a usage-limit reset hint can't be parsed. -const RATE_LIMIT_FALLBACK_MS = 60 * 60 * 1000; // 1 hour - -// In-memory mirror of the active rate-limit window for the current harness. -let rateLimitedUntil: number | null = null; -let rateLimitResumeTimer: ReturnType | null = null; +// Failover chain state lives in WorkerFailover (shared with the fleet worker). +// Assigned in `startWebhookServer` (lazily via `ensureFailover` for tests). +let failover: WorkerFailover | null = null; // Cleanup rate limiter periodically setInterval(() => rateLimiter.cleanup(), 60000); // Note: We use a single reusable worktree, so no periodic cleanup needed -/** Name of the agent harness this server drives (e.g. `claude-code`). */ +/** + * Lazily build the failover state from the `AGENT_HARNESS` chain. + * + * Startup always initializes with installability checks; this fallback covers + * direct module use before `startWebhookServer` runs (e.g. in tests). + */ +function ensureFailover(): WorkerFailover { + if (failover) { + return failover; + } + failover = ensureWorkerFailover(); + return failover; +} + +/** Name of the agent harness currently driving this server (e.g. `claude-code`). */ function currentHarnessName(): string { - return resolveHarness().harness.name; + return ensureFailover().activeName; } /** - * Pause the review queue until the current harness's usage limit resets. + * Resolve the active harness and its executable path for an agent spawn. * - * Idempotent: extends the window if a later reset arrives. The persisted state - * is keyed by harness so a restart with a different `AGENT_HARNESS` is not - * wrongly blocked. + * Per-harness env overrides (`_CLI_PATH`) were already applied when + * the chain was resolved, so every spawn uses the right CLI for whichever + * harness failover selected. `AGENT_MODEL` is read at spawn time and applies + * to the active harness (the string is harness-specific by nature). * - * @param resetHint - Human-readable reset hint from the agent output + * @returns The resolved harness and executable path to spawn. */ -function enterRateLimitPause(resetHint?: string): void { - const harness = currentHarnessName(); - const until = resetHintToMs(resetHint, Date.now()) ?? Date.now() + RATE_LIMIT_FALLBACK_MS; - - // Keep the latest (furthest) reset if one is already active. - rateLimitedUntil = Math.max(rateLimitedUntil ?? 0, until); - webhookQueue?.setRateLimit(harness, rateLimitedUntil); - - if (!reviewQueue.isPaused) { - reviewQueue.pause(); - } - - const waitMs = Math.max(0, rateLimitedUntil - Date.now()); - const resetAtIso = new Date(rateLimitedUntil).toISOString(); - console.warn( - `⏳ ${harness} hit a usage limit${resetHint ? ` (resets ${resetHint})` : ""}. ` + - `Pausing webhook queue until ${resetAtIso} (~${Math.round(waitMs / 60000)} min). ` + - `Queued and incoming events will wait and drain on resume.`, - ); - - scheduleRateLimitResume(); +function resolveActiveHarness(): ResolvedHarness { + return ensureFailover().resolvedHarness(); } -/** (Re)arm the timer that resumes the queue when the rate-limit window ends. */ -function scheduleRateLimitResume(): void { - if (rateLimitResumeTimer) { - clearTimeout(rateLimitResumeTimer); - } - if (rateLimitedUntil === null) { - return; - } - const waitMs = Math.max(0, rateLimitedUntil - Date.now()); - rateLimitResumeTimer = setTimeout(() => { - const harness = currentHarnessName(); - rateLimitedUntil = null; - rateLimitResumeTimer = null; - webhookQueue?.clearRateLimit(harness); - console.log(`▶️ Usage limit window elapsed for ${harness} — resuming webhook queue`); - reviewQueue.start(); - }, waitMs); +/** + * Handle a usage-limit report from the active harness. + * + * With a multi-harness `AGENT_HARNESS` chain, fail over to the highest-priority + * harness whose limit window has elapsed and keep processing — the queue only + * pauses when every harness in the chain is limited (which is also the exact + * behavior of a single-harness configuration). The window is persisted per + * harness and the failback timer armed, so the worker returns to the primary + * harness as soon as its window ends. + * + * @param resetHint - Human-readable reset hint from the agent output + */ +function handleUsageLimit(resetHint?: string): void { + ensureFailover().reportFromHint({ resetsAt: resetHint }); } /** @@ -492,9 +483,10 @@ async function processReviewWithPersistence( } } catch (error) { if (error instanceof UsageLimitError) { - // Deferred by an account-global usage limit — pause and re-queue for - // after reset instead of counting a failure. - enterRateLimitPause(error.resetHint); + // Deferred by an account-global usage limit — fail over to the next + // harness (or pause and re-queue for after reset) instead of counting + // a failure. + handleUsageLimit(error.resetHint); if (eventId && webhookQueue) { webhookQueue.requeuePending(eventId); } @@ -535,7 +527,7 @@ async function processIssueCommentWithPersistence( } } catch (error) { if (error instanceof UsageLimitError) { - enterRateLimitPause(error.resetHint); + handleUsageLimit(error.resetHint); if (eventId && webhookQueue) { webhookQueue.requeuePending(eventId); } @@ -823,7 +815,7 @@ async function processReviewAsync( const autoReviewOutputDir = `/tmp/devintern-auto-review-${prNumber}`; const baseBranch = event.pull_request.base.ref; - const { harness: reviewHarness, path: reviewPath } = resolveHarness(); + const { harness: reviewHarness, path: reviewPath } = resolveActiveHarness(); try { const autoReviewResult = await runAutoReviewLoop({ repository: `${owner}/${repo}`, @@ -879,7 +871,8 @@ async function processReviewAsync( const hitMaxTurns = agentResult.maxTurnsReached === true; // A usage limit is account-global: don't burn this event as a failure — - // signal the wrapper to pause the queue and re-queue it for after reset. + // signal the wrapper to fail over to the next harness (or pause the queue + // and re-queue the event for after reset). if (agentResult.usageLimited) { throw new UsageLimitError(agentResult.usageResetHint); } @@ -895,7 +888,7 @@ async function processReviewAsync( // Get hook retries configuration const hookRetries = parseInt(process.env.HOOK_RETRIES || "10", 10); - const { harness, path: executablePath } = resolveHarness(); + const { harness, path: executablePath } = resolveActiveHarness(); const maxTurns = parseInt(process.env.CLAUDE_MAX_TURNS || "500", 10); // Verify Agent didn't switch branches during execution (e.g., checking out main for comparison) @@ -1098,7 +1091,7 @@ async function processReviewAsync( console.log("\n🔄 Running auto-review loop (without pushing)..."); const autoReviewOutputDir = `/tmp/devintern-auto-review-${prNumber}`; const baseBranchForReview = event.pull_request.base.ref; - const { harness: reviewHarness2, path: reviewPath2 } = resolveHarness(); + const { harness: reviewHarness2, path: reviewPath2 } = resolveActiveHarness(); try { const autoReviewResult = await runAutoReviewLoop({ repository: `${owner}/${repo}`, @@ -1209,6 +1202,12 @@ async function processReviewAsync( console.log(`\n✅ Successfully addressed review for PR #${prNumber}`); } catch (error) { + if (error instanceof UsageLimitError) { + // Account-global usage limit: propagate to the persistence wrapper so + // it can fail over to the next harness (or pause + re-queue until the + // window resets) without burning the event as a failure. + throw error; + } console.error(`❌ Error processing review: ${(error as Error).message}`); if (config.debug) { console.error((error as Error).stack); @@ -1274,7 +1273,7 @@ async function runAgentHarnessForReview( usageLimited?: boolean; usageResetHint?: string; }> { - const { harness, path: executablePath } = resolveHarness(); + const { harness, path: executablePath } = resolveActiveHarness(); // Wait out any in-progress CLI auto-update swap before spawning, so a // transient `spawn ENOENT` doesn't abort the review. const resolvedPath = await resolveExecutablePathWithRetry(executablePath, { @@ -1430,18 +1429,24 @@ async function runAgentHarnessForReview( }); } -/** Return JSON health payload including webhook queue stats. */ +/** Return JSON health payload including webhook queue stats and failover state. */ function handleHealthCheck(): Response { const queueStats = webhookQueue?.getStats() || { pending: 0, processing: 0, failed: 0, }; + const manager = failover; return jsonResponse({ status: "ok", timestamp: new Date().toISOString(), version: "1.0.0", queue: queueStats, + harness: { + active: manager?.activeName ?? currentHarnessName(), + chain: manager?.describeChain() ?? currentHarnessName(), + rateLimitedUntil: manager?.windows() ?? {}, + }, }); } @@ -1539,21 +1544,31 @@ export async function startWebhookServer( ); } - // Re-apply a usage-limit pause if the current harness is still rate-limited - // from before a restart. Keyed by harness so switching AGENT_HARNESS clears it. - const harness = currentHarnessName(); - const persistedLimit = webhookQueue.getRateLimit(harness); - if (persistedLimit && persistedLimit > Date.now()) { - rateLimitedUntil = persistedLimit; + // Shared failover controller: same chain, windows, and failback timers the + // fleet worker uses. Persist through the queue DB so a restart resumes on + // the right harness. + failover = startWorkerFailover({ + queue: webhookQueue, + onPause: ({ untilMs, harness, resetHint }) => { + if (!reviewQueue.isPaused) { + reviewQueue.pause(); + } + const waitMs = Math.max(0, untilMs - Date.now()); + console.warn( + `⏳ ${harness} hit a usage limit${resetHint ? ` (resets ${resetHint})` : ""} and no fallback harness is available. ` + + `Pausing webhook queue until ${new Date(untilMs).toISOString()} (~${Math.round(waitMs / 60000)} min). ` + + `Queued and incoming events will wait and drain on resume.`, + ); + }, + onResume: () => { + if (reviewQueue.isPaused) { + console.log(`▶️ Usage-limit windows elapsed — resuming webhook queue`); + reviewQueue.start(); + } + }, + }); + if (failover.allLimited() && !reviewQueue.isPaused) { reviewQueue.pause(); - scheduleRateLimitResume(); - console.warn( - `⏳ ${harness} is still rate-limited until ${new Date(persistedLimit).toISOString()} ` + - `— webhook queue starts paused; recovered events will wait.`, - ); - } else if (persistedLimit) { - // Stale window (already elapsed) — clear it. - webhookQueue.clearRateLimit(harness); } // Recover pending/processing events from previous runs diff --git a/packages/code/tests/harness-failover.test.ts b/packages/code/tests/harness-failover.test.ts new file mode 100644 index 00000000..51e84642 --- /dev/null +++ b/packages/code/tests/harness-failover.test.ts @@ -0,0 +1,250 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { existsSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +import { getHarness } from "@devintern/agent-harness"; +import type { HarnessChainEntry } from "@devintern/agent-harness"; + +import { HarnessFailover } from "../src/lib/harness-failover"; +import { WebhookQueue } from "../src/lib/webhook-queue"; + +/** Build a chain entry from a registered harness name. */ +function entry(name: string): HarnessChainEntry { + const harness = getHarness(name); + if (!harness) { + throw new Error(`Harness "${name}" is not registered`); + } + return { name: harness.name, harness, path: harness.defaultPath, installed: true }; +} + +/** Build a failover over the given chain with captured logs and persistence. */ +function makeFailover(names: string[], now = () => 1000) { + const lines: string[] = []; + const limits: Record = {}; + const cleared: string[] = []; + const state = { active: null as string | null }; + const failover = new HarnessFailover({ + entries: names.map(entry), + now, + persistLimit: (harness, untilMs) => { + limits[harness] = untilMs; + }, + clearPersistedLimit: (harness) => { + cleared.push(harness); + delete limits[harness]; + }, + persistActive: (harness) => { + state.active = harness; + }, + log: (message) => lines.push(message), + }); + return { failover, lines, limits, cleared, state }; +} + +describe("HarnessFailover", () => { + test("single-entry chain is exhausted when the harness is limited", () => { + const h = makeFailover(["claude-code"]); + const outcome = h.failover.reportUsageLimit(2000); + expect(outcome.kind).toBe("exhausted"); + expect(outcome.kind === "exhausted" && outcome.untilMs).toBe(2000); + expect(h.limits["claude-code"]).toBe(2000); + }); + + test("fails over to the next entry when the primary is limited", () => { + const h = makeFailover(["claude-code", "codex"]); + const outcome = h.failover.reportUsageLimit(2000); + expect(outcome).toEqual({ + kind: "switched", + from: "claude-code", + to: "codex", + untilMs: 2000, + }); + expect(h.failover.activeName).toBe("codex"); + expect(h.failover.onPrimary).toBe(false); + expect(h.state.active).toBe("codex"); + expect(h.limits["claude-code"]).toBe(2000); + expect(h.lines.join("\n")).toContain("failing over to codex"); + }); + + test("advances again when the fallback also hits a limit mid-task", () => { + const h = makeFailover(["claude-code", "codex", "grok"]); + h.failover.reportUsageLimit(2000); + const second = h.failover.reportUsageLimit(3000); + expect(second).toEqual({ kind: "switched", from: "codex", to: "grok", untilMs: 3000 }); + expect(h.failover.activeName).toBe("grok"); + expect(h.limits["codex"]).toBe(3000); + }); + + test("exhausts when every entry is limited and reports the earliest reset", () => { + const h = makeFailover(["claude-code", "codex"]); + h.failover.reportUsageLimit(5000); + const outcome = h.failover.reportUsageLimit(3000); + expect(outcome.kind).toBe("exhausted"); + expect(outcome.kind === "exhausted" && outcome.untilMs).toBe(3000); + expect(h.failover.earliestResetMs()).toBe(3000); + expect(h.failover.allLimited()).toBe(true); + }); + + test("keeps the furthest reset when a window is extended", () => { + const h = makeFailover(["claude-code"]); + h.failover.reportUsageLimit(5000); + h.failover.reportUsageLimit(3000); + expect(h.failover.windows()["claude-code"]).toBe(5000); + expect(h.limits["claude-code"]).toBe(5000); + }); + + test("fails back to the primary when its window elapses", () => { + const h = makeFailover(["claude-code", "codex", "grok"]); + h.failover.reportUsageLimit(5000); // claude limited → codex + h.failover.reportUsageLimit(3000); // codex limited → grok + expect(h.failover.activeName).toBe("grok"); + + let now = 3500; + const switched = h.failover.windowElapsed("codex"); + expect(switched).toBe("codex"); + expect(h.failover.activeName).toBe("codex"); + expect(h.cleared).toContain("codex"); + expect(h.lines.join("\n")).toContain("resuming on codex"); + + now = 6000; + const failback = h.failover.windowElapsed("claude-code"); + expect(failback).toBe("claude-code"); + expect(h.failover.activeName).toBe("claude-code"); + expect(h.failover.onPrimary).toBe(true); + expect(h.lines.join("\n")).toContain("failing back to primary harness claude-code"); + void now; + }); + + test("windowElapsed is a no-op when no higher-priority harness unlocks", () => { + const h = makeFailover(["claude-code", "codex"]); + h.failover.reportUsageLimit(5000); // → codex, claude limited + expect(h.failover.windowElapsed("grok")).toBeNull(); + expect(h.failover.activeName).toBe("codex"); + }); + + test("persistence hooks receive every mutation", () => { + const h = makeFailover(["claude-code", "codex"]); + h.failover.reportUsageLimit(2000); + h.failover.windowElapsed("claude-code"); + expect(h.limits["claude-code"]).toBeUndefined(); + expect(h.cleared).toContain("claude-code"); + expect(h.state.active).toBe("claude-code"); + }); + + test("restore seeds windows and honors a persisted active harness", () => { + const h = makeFailover(["claude-code", "codex"]); + const warnings = h.failover.restore({ "claude-code": 5000 }, "codex"); + expect(warnings).toEqual([]); + expect(h.failover.activeName).toBe("codex"); + expect(h.failover.isLimited("claude-code")).toBe(true); + expect(h.failover.allLimited()).toBe(false); + }); + + test("restore warns and falls back when the persisted active left the chain", () => { + const h = makeFailover(["claude-code", "codex"]); + const warnings = h.failover.restore({}, "grok"); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("grok"); + expect(h.failover.activeName).toBe("claude-code"); + }); + + test("restore ignores windows for harnesses outside the chain", () => { + const h = makeFailover(["claude-code", "codex"]); + const warnings = h.failover.restore({ grok: 5000 }, null); + expect(warnings).toHaveLength(1); + expect(h.failover.isLimited("grok")).toBe(false); + expect(h.failover.activeName).toBe("claude-code"); + }); + + test("restore drops already-elapsed windows", () => { + const h = makeFailover(["claude-code", "codex"]); + const warnings = h.failover.restore({ "claude-code": 500 }, null); + expect(warnings).toEqual([]); + expect(h.failover.isLimited("claude-code")).toBe(false); + expect(h.cleared).toContain("claude-code"); + expect(h.failover.activeName).toBe("claude-code"); + }); + + test("restore parks on the primary when every entry is limited", () => { + const h = makeFailover(["claude-code", "codex"]); + h.failover.restore({ "claude-code": 5000, codex: 4000 }, null); + expect(h.failover.allLimited()).toBe(true); + expect(h.failover.earliestResetMs()).toBe(4000); + }); + + test("describeChain renders the priority order", () => { + const h = makeFailover(["claude-code", "codex"]); + expect(h.failover.describeChain()).toBe("claude-code → codex"); + }); +}); + +describe("WebhookQueue failover persistence", () => { + let dbPath: string; + + beforeEach(() => { + dbPath = join(tmpdir(), `hf-test-${Date.now()}-${Math.random().toString(36).slice(2)}.db`); + }); + + afterEach(() => { + for (const suffix of ["", "-wal", "-shm"]) { + const file = `${dbPath}${suffix}`; + if (existsSync(file)) { + rmSync(file, { force: true }); + } + } + }); + + test("active harness and windows survive a reopen", () => { + const queue = new WebhookQueue({ dbPath }); + queue.setRateLimit("claude-code", 1234); + queue.setRateLimit("codex", 5678); + queue.setActiveHarness("codex"); + + const reopened = new WebhookQueue({ dbPath }); + expect(reopened.getAllRateLimits()).toEqual({ "claude-code": 1234, codex: 5678 }); + expect(reopened.getActiveHarness()).toBe("codex"); + + reopened.clearRateLimit("claude-code"); + expect(reopened.getAllRateLimits()).toEqual({ codex: 5678 }); + + reopened.clearActiveHarness(); + expect(reopened.getActiveHarness()).toBeNull(); + reopened.close(); + }); + + test("getAllRateLimits is empty when nothing is persisted", () => { + const queue = new WebhookQueue({ dbPath }); + expect(queue.getAllRateLimits()).toEqual({}); + expect(queue.getActiveHarness()).toBeNull(); + queue.close(); + }); + + test("a failover round-trip through the queue restores correctly", () => { + const first = new WebhookQueue({ dbPath }); + const h = makeFailover(["claude-code", "codex"]); + // Simulate the webhook server wiring failover persistence to the queue. + const manager = new HarnessFailover({ + entries: ["claude-code", "codex"].map(entry), + now: () => 1000, + persistLimit: (harness, untilMs) => first.setRateLimit(harness, untilMs), + clearPersistedLimit: (harness) => first.clearRateLimit(harness), + persistActive: (harness) => first.setActiveHarness(harness), + log: () => {}, + }); + void h; + manager.reportUsageLimit(9000); + first.close(); + + const second = new WebhookQueue({ dbPath }); + const restored = new HarnessFailover({ + entries: ["claude-code", "codex"].map(entry), + now: () => 2000, + log: () => {}, + }); + restored.restore(second.getAllRateLimits(), second.getActiveHarness()); + expect(restored.activeName).toBe("codex"); + expect(restored.isLimited("claude-code")).toBe(true); + second.close(); + }); +}); diff --git a/packages/code/tests/worker-failover.test.ts b/packages/code/tests/worker-failover.test.ts new file mode 100644 index 00000000..468c656d --- /dev/null +++ b/packages/code/tests/worker-failover.test.ts @@ -0,0 +1,92 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +import { UsageLimitError } from "@devintern/agent-harness"; + +import { + resetWorkerFailover, + runWithFailover, + startWorkerFailover, +} from "../src/lib/worker-failover"; +import { + readUsageLimitHint, + USAGE_LIMIT_EXIT_CODE, + USAGE_LIMIT_FILE_ENV, + writeUsageLimitHint, +} from "../src/lib/usage-limit-protocol"; + +afterEach(() => { + resetWorkerFailover(); +}); + +describe("runWithFailover", () => { + test("without a worker controller, a usage-limit exit is a plain failure", async () => { + const result = await runWithFailover(async () => USAGE_LIMIT_EXIT_CODE); + expect(result).toBe("failed"); + }); + + test("retries the same spawn on the next harness after exit 75", async () => { + startWorkerFailover({ checkInstalled: false, raw: "codex,grok", log: () => {} }); + + const seen: string[] = []; + const result = await runWithFailover(async (env) => { + seen.push(env.AGENT_HARNESS ?? ""); + if (seen.length === 1) { + writeFileSync( + env[USAGE_LIMIT_FILE_ENV]!, + JSON.stringify({ untilMs: Date.now() + 60_000, resetsAt: "4:27 PM" }), + ); + return USAGE_LIMIT_EXIT_CODE; + } + return 0; + }); + + expect(result).toBe("ok"); + expect(seen).toEqual(["codex", "grok"]); + }); + + test("returns deferred when every harness in the chain is limited", async () => { + startWorkerFailover({ checkInstalled: false, raw: "codex,grok", log: () => {} }); + + const result = await runWithFailover(async (env) => { + writeFileSync(env[USAGE_LIMIT_FILE_ENV]!, JSON.stringify({ untilMs: Date.now() + 60_000 })); + return USAGE_LIMIT_EXIT_CODE; + }); + + expect(result).toBe("deferred"); + }); + + test("pins AGENT_HARNESS to the active entry on the first attempt", async () => { + startWorkerFailover({ checkInstalled: false, raw: "codex,grok,cursor", log: () => {} }); + + const result = await runWithFailover(async (env) => { + expect(env.AGENT_HARNESS).toBe("codex"); + expect(env.DEVINTERN_WORKER_CHILD).toBe("1"); + return 0; + }); + expect(result).toBe("ok"); + }); +}); + +describe("usage-limit hint file", () => { + test("round-trips untilMs and the reset hint", () => { + const dir = mkdtempSync(join(tmpdir(), "devintern-hint-")); + const path = join(dir, "hint.json"); + const previous = process.env[USAGE_LIMIT_FILE_ENV]; + process.env[USAGE_LIMIT_FILE_ENV] = path; + try { + writeUsageLimitHint(new UsageLimitError("4:27 PM")); + const hint = readUsageLimitHint(path); + expect(hint.resetsAt).toBe("4:27 PM"); + expect(hint.untilMs).toBeGreaterThan(Date.now()); + } finally { + if (previous === undefined) { + delete process.env[USAGE_LIMIT_FILE_ENV]; + } else { + process.env[USAGE_LIMIT_FILE_ENV] = previous; + } + } + }); +});