diff --git a/packages/gittensory-engine/src/governor/run-halt.ts b/packages/gittensory-engine/src/governor/run-halt.ts new file mode 100644 index 0000000000..11f505157a --- /dev/null +++ b/packages/gittensory-engine/src/governor/run-halt.ts @@ -0,0 +1,159 @@ +// Governor run-loop halt enforcement (#2347): composes the pure non-convergence detector and the +// budget/turn/termination cap calculator at every iteration boundary before the portfolio queue claims +// the next item. Either signal tripping halts the current run; the caller releases in-flight work and +// blocks further claims until a human clears the halt. + +import type { GovernorLedgerEvent, GovernorLedgerEventType } from "../governor-ledger.js"; +import { + classifyPortfolioConvergence, + DEFAULT_PORTFOLIO_CONVERGENCE_THRESHOLDS, + type PortfolioConvergenceInput, + type PortfolioConvergenceThresholds, +} from "../portfolio/non-convergence.js"; +import { + evaluateGovernorCaps, + type GovernorCapLimits, + type GovernorCapReport, + type GovernorCapUsage, +} from "./budget-cap.js"; + +/** Issue vocabulary alias for the pure budget/turn/termination cap calculator. */ +export const evaluateBudgetCaps = evaluateGovernorCaps; + +export type NonConvergenceSignal = { + tripped: boolean; + status: ReturnType["status"]; + reasons: string[]; +}; + +/** Issue vocabulary wrapper over {@link classifyPortfolioConvergence}. */ +export function detectNonConvergence( + input: PortfolioConvergenceInput, + thresholds: PortfolioConvergenceThresholds = DEFAULT_PORTFOLIO_CONVERGENCE_THRESHOLDS, +): NonConvergenceSignal { + const verdict = classifyPortfolioConvergence(input, thresholds); + return { + tripped: verdict.status === "non_convergent", + status: verdict.status, + reasons: verdict.reasons, + }; +} + +export type RunLoopHaltReason = + | "cleared" + | "prior_halt" + | "non_convergence" + | "budget_exceeded" + | "turn_cap_exceeded" + | "termination_cap_exceeded"; + +export type RunLoopHaltVerdict = { + shouldHalt: boolean; + canClaimNext: boolean; + reason: RunLoopHaltReason; + convergence: NonConvergenceSignal; + caps: GovernorCapReport; + ledgerEventType: GovernorLedgerEventType; + ledgerReason: string; +}; + +function haltVerdict( + reason: RunLoopHaltReason, + convergence: NonConvergenceSignal, + caps: GovernorCapReport, + ledgerEventType: GovernorLedgerEventType, + ledgerReason: string, +): RunLoopHaltVerdict { + return { + shouldHalt: true, + canClaimNext: false, + reason, + convergence, + caps, + ledgerEventType, + ledgerReason, + }; +} + +/** + * Consult non-convergence and budget caps at a run-loop iteration boundary. A prior halt sticks until + * the caller clears it; otherwise either signal tripping halts the run and blocks further queue claims. + */ +export function evaluateRunLoopHalt(input: { + runHalted: boolean; + usage: GovernorCapUsage; + limits: GovernorCapLimits; + convergence: PortfolioConvergenceInput; + convergenceThresholds?: PortfolioConvergenceThresholds; +}): RunLoopHaltVerdict { + const convergence = detectNonConvergence(input.convergence, input.convergenceThresholds); + const caps = evaluateGovernorCaps(input.usage, input.limits); + + if (input.runHalted) { + return haltVerdict("prior_halt", convergence, caps, "denied", "run_already_halted"); + } + + if (caps.verdict === "kill_switch") { + return haltVerdict( + "termination_cap_exceeded", + convergence, + caps, + "kill_switch", + "termination_cap_exceeded", + ); + } + if (caps.budget.exceeded) { + return haltVerdict("budget_exceeded", convergence, caps, "denied", "budget_cap_exceeded"); + } + if (caps.turns.exceeded) { + return haltVerdict("turn_cap_exceeded", convergence, caps, "denied", "turn_cap_exceeded"); + } + if (convergence.tripped) { + return haltVerdict( + "non_convergence", + convergence, + caps, + "denied", + "non_convergence_detected", + ); + } + + return { + shouldHalt: false, + canClaimNext: true, + reason: "cleared", + convergence, + caps, + ledgerEventType: "allowed", + ledgerReason: "under_limit", + }; +} + +/** Reset latch after an operator clears a halted run. */ +export function clearRunLoopHalt(): { runHalted: false } { + return { runHalted: false }; +} + +/** Governor-ledger row for a run-loop halt decision (#2347 deliverable). */ +export function buildRunLoopHaltGovernorLedgerEvent( + repoFullName: string | null | undefined, + inFlightIdentifier: string | null | undefined, + verdict: RunLoopHaltVerdict, +): GovernorLedgerEvent { + return { + eventType: verdict.ledgerEventType, + repoFullName, + actionClass: "run_loop", + decision: verdict.shouldHalt ? "halt" : "continue", + reason: verdict.ledgerReason, + payload: { + haltReason: verdict.reason, + convergenceStatus: verdict.convergence.status, + convergenceReasons: verdict.convergence.reasons, + budgetExceeded: verdict.caps.budget.exceeded, + turnsExceeded: verdict.caps.turns.exceeded, + terminationExceeded: verdict.caps.termination.exceeded, + inFlightIdentifier: inFlightIdentifier ?? null, + }, + }; +} diff --git a/packages/gittensory-engine/src/index.ts b/packages/gittensory-engine/src/index.ts index 5710f2fed7..1fe50d2f52 100644 --- a/packages/gittensory-engine/src/index.ts +++ b/packages/gittensory-engine/src/index.ts @@ -148,6 +148,7 @@ export * from "./governor/budget-cap.js"; export * from "./governor/self-plagiarism.js"; export * from "./governor/reputation-throttle.js"; export * from "./governor/write-rate-limit.js"; +export * from "./governor/run-halt.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/gittensory-engine/test/run-halt-enforcement.test.ts b/packages/gittensory-engine/test/run-halt-enforcement.test.ts new file mode 100644 index 0000000000..b737b6efe8 --- /dev/null +++ b/packages/gittensory-engine/test/run-halt-enforcement.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + buildRunLoopHaltGovernorLedgerEvent, + detectNonConvergence, + evaluateBudgetCaps, + evaluateRunLoopHalt, +} from "../dist/index.js"; + +test("barrel: the public entrypoint re-exports run-loop halt enforcement (#2347)", () => { + assert.equal(typeof evaluateRunLoopHalt, "function"); + assert.equal(typeof detectNonConvergence, "function"); + assert.equal(typeof evaluateBudgetCaps, "function"); + assert.equal(typeof buildRunLoopHaltGovernorLedgerEvent, "function"); +}); + +test("evaluateRunLoopHalt: either signal tripping halts the run", () => { + const healthy = evaluateRunLoopHalt({ + runHalted: false, + usage: { budgetSpent: 1, turnsTaken: 1, elapsedMs: 1_000 }, + limits: { budget: 100, turns: 5, elapsedMs: 60_000 }, + convergence: { attempts: 1, consecutiveFailures: 0, reenqueues: 0, reachedDone: false }, + }); + assert.equal(healthy.shouldHalt, false); + + const budgetHalt = evaluateRunLoopHalt({ + runHalted: false, + usage: { budgetSpent: 100, turnsTaken: 1, elapsedMs: 1_000 }, + limits: { budget: 100, turns: 5, elapsedMs: 60_000 }, + convergence: { attempts: 1, consecutiveFailures: 0, reenqueues: 0, reachedDone: false }, + }); + assert.equal(budgetHalt.shouldHalt, true); + + assert.equal( + detectNonConvergence({ attempts: 4, consecutiveFailures: 3, reenqueues: 0, reachedDone: false }).tripped, + true, + ); + assert.equal(evaluateBudgetCaps({ budgetSpent: 1, turnsTaken: 1, elapsedMs: 1_000 }, { + budget: 100, + turns: 5, + elapsedMs: 60_000, + }).verdict, "allowed"); +}); diff --git a/packages/gittensory-miner/lib/governor-run-halt.d.ts b/packages/gittensory-miner/lib/governor-run-halt.d.ts new file mode 100644 index 0000000000..6cbb9dacd7 --- /dev/null +++ b/packages/gittensory-miner/lib/governor-run-halt.d.ts @@ -0,0 +1,37 @@ +import type { + GovernorCapLimits, + GovernorCapUsage, + PortfolioConvergenceInput, + PortfolioConvergenceThresholds, + RunLoopHaltVerdict, +} from "@jsonbored/gittensory-engine"; +import type { AppendGovernorEventInput, GovernorLedgerEntry } from "./governor-ledger.js"; +import type { QueueEntry } from "./portfolio-queue.js"; + +export type RunLoopInFlightItem = { + repoFullName: string; + identifier: string; +}; + +export type EvaluateRunLoopBoundaryGateInput = { + runHalted?: boolean; + usage: GovernorCapUsage; + limits: GovernorCapLimits; + convergence: PortfolioConvergenceInput; + convergenceThresholds?: PortfolioConvergenceThresholds; + inFlightItem?: RunLoopInFlightItem | null; + markFailed?: (repoFullName: string, identifier: string) => QueueEntry | null; +}; + +export type EvaluateRunLoopBoundaryGateResult = { + verdict: RunLoopHaltVerdict; + recorded: GovernorLedgerEntry | null; + runHalted: boolean; + canClaimNext: boolean; + releasedItem: QueueEntry | null; +}; + +export function evaluateRunLoopBoundaryGate( + input: EvaluateRunLoopBoundaryGateInput, + options?: { append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry }, +): EvaluateRunLoopBoundaryGateResult; diff --git a/packages/gittensory-miner/lib/governor-run-halt.js b/packages/gittensory-miner/lib/governor-run-halt.js new file mode 100644 index 0000000000..8c510ccb16 --- /dev/null +++ b/packages/gittensory-miner/lib/governor-run-halt.js @@ -0,0 +1,58 @@ +// Governor run-loop halt gate (#2347). Consults non-convergence + budget caps at each iteration boundary, +// releases in-flight portfolio items on a fresh halt, and records the decision to the governor ledger. + +import { + buildRunLoopHaltGovernorLedgerEvent, + evaluateRunLoopHalt, +} from "@jsonbored/gittensory-engine"; +import { appendGovernorEvent } from "./governor-ledger.js"; + +/** + * Evaluate run-loop halt signals before claiming the next portfolio item. + * + * @param {object} input + * @param {boolean} [input.runHalted] whether the run is already halted + * @param {import("@jsonbored/gittensory-engine").GovernorCapUsage} input.usage cumulative run usage + * @param {import("@jsonbored/gittensory-engine").GovernorCapLimits} input.limits run ceilings + * @param {import("@jsonbored/gittensory-engine").PortfolioConvergenceInput} input.convergence in-flight item history + * @param {import("@jsonbored/gittensory-engine").PortfolioConvergenceThresholds} [input.convergenceThresholds] + * @param {{ repoFullName: string, identifier: string } | null | undefined} [input.inFlightItem] + * @param {(repoFullName: string, identifier: string) => import("./portfolio-queue.js").QueueEntry | null} [input.markFailed] + * @param {{ append?: typeof appendGovernorEvent }} [options] + */ +export function evaluateRunLoopBoundaryGate(input, options = {}) { + const append = options.append ?? appendGovernorEvent; + const wasHalted = Boolean(input.runHalted); + const verdict = evaluateRunLoopHalt({ + runHalted: wasHalted, + usage: input.usage, + limits: input.limits, + convergence: input.convergence, + convergenceThresholds: input.convergenceThresholds, + }); + + const newlyHalted = !wasHalted && verdict.shouldHalt; + let releasedItem = null; + if (newlyHalted && input.inFlightItem && typeof input.markFailed === "function") { + releasedItem = input.markFailed(input.inFlightItem.repoFullName, input.inFlightItem.identifier); + } + + const recorded = + newlyHalted || (!wasHalted && !verdict.shouldHalt) + ? append( + buildRunLoopHaltGovernorLedgerEvent( + input.inFlightItem?.repoFullName ?? null, + input.inFlightItem?.identifier ?? null, + verdict, + ), + ) + : null; + + return { + verdict, + recorded, + runHalted: verdict.shouldHalt, + canClaimNext: verdict.canClaimNext, + releasedItem, + }; +} diff --git a/packages/gittensory-miner/lib/portfolio-queue-manager.d.ts b/packages/gittensory-miner/lib/portfolio-queue-manager.d.ts index 68ecee9b93..1f578a5d27 100644 --- a/packages/gittensory-miner/lib/portfolio-queue-manager.d.ts +++ b/packages/gittensory-miner/lib/portfolio-queue-manager.d.ts @@ -31,6 +31,7 @@ export type PortfolioQueueManager = { enqueue(item: EnqueueItem): QueueEntry; listQueue(repoFullName?: string | null): QueueEntry[]; markDone(repoFullName: string, identifier: string): QueueEntry | null; + markFailed(repoFullName: string, identifier: string): QueueEntry | null; claimNextBatch(): QueueEntry[]; close(): void; }; diff --git a/packages/gittensory-miner/lib/portfolio-queue-manager.js b/packages/gittensory-miner/lib/portfolio-queue-manager.js index ed7b8ba41a..ded8372027 100644 --- a/packages/gittensory-miner/lib/portfolio-queue-manager.js +++ b/packages/gittensory-miner/lib/portfolio-queue-manager.js @@ -88,6 +88,9 @@ export function initPortfolioQueueManager(options = {}) { markDone(repoFullName, identifier) { return store.markDone(repoFullName, identifier); }, + markFailed(repoFullName, identifier) { + return store.markFailed(repoFullName, identifier); + }, claimNextBatch() { return store.batchClaim((entries) => selectEligibleBatch(entries, caps)); }, diff --git a/packages/gittensory-miner/lib/portfolio-queue.d.ts b/packages/gittensory-miner/lib/portfolio-queue.d.ts index c1560421e5..bc8e30f3ed 100644 --- a/packages/gittensory-miner/lib/portfolio-queue.d.ts +++ b/packages/gittensory-miner/lib/portfolio-queue.d.ts @@ -20,6 +20,7 @@ export type PortfolioQueueStore = { dequeueNext(): QueueEntry | null; listQueue(repoFullName?: string | null): QueueEntry[]; markDone(repoFullName: string, identifier: string): QueueEntry | null; + markFailed(repoFullName: string, identifier: string): QueueEntry | null; batchClaim( selectFn: (entries: QueueEntry[]) => Array<{ repoFullName: string; identifier: string }>, ): QueueEntry[]; @@ -40,4 +41,6 @@ export function listQueue(repoFullName?: string | null): QueueEntry[]; export function markDone(repoFullName: string, identifier: string): QueueEntry | null; +export function markFailed(repoFullName: string, identifier: string): QueueEntry | null; + export function closeDefaultPortfolioQueueStore(): void; diff --git a/packages/gittensory-miner/lib/portfolio-queue.js b/packages/gittensory-miner/lib/portfolio-queue.js index b9c32b205a..8d64e962ea 100644 --- a/packages/gittensory-miner/lib/portfolio-queue.js +++ b/packages/gittensory-miner/lib/portfolio-queue.js @@ -105,6 +105,11 @@ export function initPortfolioQueueStore(dbPath = resolvePortfolioQueueDbPath()) const markDoneStatement = db.prepare( "UPDATE miner_portfolio_queue SET status = 'done' WHERE repo_full_name = ? AND identifier = ? AND status <> 'done'", ); + const markFailedStatement = db.prepare(` + UPDATE miner_portfolio_queue SET status = 'queued' + WHERE repo_full_name = ? AND identifier = ? AND status = 'in_progress' + RETURNING * + `); const listAllStatement = db.prepare(`SELECT * FROM miner_portfolio_queue ${ORDER}`); const listRepoStatement = db.prepare( `SELECT * FROM miner_portfolio_queue WHERE repo_full_name = ? ${ORDER}`, @@ -146,6 +151,13 @@ export function initPortfolioQueueStore(dbPath = resolvePortfolioQueueDbPath()) const row = getStatement.get(normalizedRepo, normalizedIdentifier); return row ? rowToEntry(row) : null; }, + /** Release an in-flight item back to `queued` when a run halts (#2347). */ + markFailed(repoFullName, identifier) { + const normalizedRepo = normalizeRepoFullName(repoFullName); + const normalizedIdentifier = normalizeIdentifier(identifier); + const row = markFailedStatement.get(normalizedRepo, normalizedIdentifier); + return row ? rowToEntry(row) : null; + }, /** * Transactional caps-aware batch claim hook used by portfolio-queue-manager.js: re-read active rows under an * exclusive lock, let the caller pick targets, then atomically flip each still-queued row to `in_progress`. @@ -198,6 +210,10 @@ export function markDone(repoFullName, identifier) { return getDefaultPortfolioQueueStore().markDone(repoFullName, identifier); } +export function markFailed(repoFullName, identifier) { + return getDefaultPortfolioQueueStore().markFailed(repoFullName, identifier); +} + export function closeDefaultPortfolioQueueStore() { if (!defaultPortfolioQueueStore) return; defaultPortfolioQueueStore.close(); diff --git a/packages/gittensory-miner/package.json b/packages/gittensory-miner/package.json index 410e6f8ad1..6c38514917 100644 --- a/packages/gittensory-miner/package.json +++ b/packages/gittensory-miner/package.json @@ -32,7 +32,7 @@ "expected-engine.version" ], "scripts": { - "build": "node --check bin/gittensory-miner.js && node --check lib/version.js && node --check lib/local-store.js && node --check lib/cli.js && node --check lib/deny-check.js && node --check lib/run-state-cli.js && node --check lib/update-check.js && node --check lib/worktree-allocator.js && node --check lib/opportunity-fanout.js && node --check lib/ci-poller.js && node --check lib/run-state.js && node --check lib/deny-hooks.js && node --check lib/event-ledger.js && node --check lib/event-ledger-cli.js && node --check lib/claim-ledger.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-adjudication.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-discovery.js && node --check lib/opportunity-ranker.js && node --check lib/plan-store.js && node --check lib/plan-store-cli.js && node --check lib/rejection-templates.js && node --check lib/governor-ledger.js && node --check lib/governor-ledger-cli.js && node --check lib/governor-open-pr.js && node --check lib/governor-write-rate-limit.js && node --check lib/attempt-log.js && node --check lib/manage-status.js && node --check lib/manage-poll.js && node --check lib/status.js && node --check lib/laptop-init.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-task-generation.js && node --check lib/calibration-types.js && node --check lib/calibration.js" + "build": "node --check bin/gittensory-miner.js && node --check lib/version.js && node --check lib/local-store.js && node --check lib/cli.js && node --check lib/deny-check.js && node --check lib/run-state-cli.js && node --check lib/update-check.js && node --check lib/worktree-allocator.js && node --check lib/opportunity-fanout.js && node --check lib/ci-poller.js && node --check lib/run-state.js && node --check lib/deny-hooks.js && node --check lib/event-ledger.js && node --check lib/event-ledger-cli.js && node --check lib/claim-ledger.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-adjudication.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-discovery.js && node --check lib/opportunity-ranker.js && node --check lib/plan-store.js && node --check lib/plan-store-cli.js && node --check lib/rejection-templates.js && node --check lib/governor-ledger.js && node --check lib/governor-ledger-cli.js && node --check lib/governor-open-pr.js && node --check lib/governor-write-rate-limit.js && node --check lib/governor-run-halt.js && node --check lib/attempt-log.js && node --check lib/manage-status.js && node --check lib/manage-poll.js && node --check lib/status.js && node --check lib/laptop-init.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-task-generation.js && node --check lib/calibration-types.js && node --check lib/calibration.js" }, "dependencies": { "@jsonbored/gittensory-engine": "*" diff --git a/test/unit/governor-run-halt.test.ts b/test/unit/governor-run-halt.test.ts new file mode 100644 index 0000000000..7e6f6e13b4 --- /dev/null +++ b/test/unit/governor-run-halt.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; +import { + buildRunLoopHaltGovernorLedgerEvent, + clearRunLoopHalt, + detectNonConvergence, + evaluateBudgetCaps, + evaluateRunLoopHalt, +} from "../../packages/gittensory-engine/src/governor/run-halt"; +import type { GovernorCapLimits, GovernorCapUsage } from "../../packages/gittensory-engine/src/governor/budget-cap"; +import type { PortfolioConvergenceInput } from "../../packages/gittensory-engine/src/portfolio/non-convergence"; + +const LIMITS: GovernorCapLimits = { budget: 100, turns: 5, elapsedMs: 60_000 }; +const HEALTHY_USAGE: GovernorCapUsage = { budgetSpent: 10, turnsTaken: 1, elapsedMs: 1_000 }; +const HEALTHY_CONVERGENCE: PortfolioConvergenceInput = { + attempts: 2, + consecutiveFailures: 0, + reenqueues: 0, + reachedDone: false, +}; + +describe("evaluateRunLoopHalt (#2347)", () => { + it("allows a healthy run to continue claiming on the iteration boundary", () => { + const verdict = evaluateRunLoopHalt({ + runHalted: false, + usage: HEALTHY_USAGE, + limits: LIMITS, + convergence: HEALTHY_CONVERGENCE, + }); + expect(verdict.shouldHalt).toBe(false); + expect(verdict.canClaimNext).toBe(true); + expect(verdict.reason).toBe("cleared"); + expect(verdict.ledgerEventType).toBe("allowed"); + }); + + it("halts a flapping run when non-convergence is detected and blocks further claims", () => { + const verdict = evaluateRunLoopHalt({ + runHalted: false, + usage: HEALTHY_USAGE, + limits: LIMITS, + convergence: { + attempts: 4, + consecutiveFailures: 3, + reenqueues: 0, + reachedDone: false, + }, + }); + expect(verdict.shouldHalt).toBe(true); + expect(verdict.canClaimNext).toBe(false); + expect(verdict.reason).toBe("non_convergence"); + expect(verdict.convergence.tripped).toBe(true); + }); + + it("halts immediately when a budget cap is breached on the next iteration boundary", () => { + const verdict = evaluateRunLoopHalt({ + runHalted: false, + usage: { budgetSpent: 100, turnsTaken: 1, elapsedMs: 1_000 }, + limits: LIMITS, + convergence: HEALTHY_CONVERGENCE, + }); + expect(verdict.shouldHalt).toBe(true); + expect(verdict.reason).toBe("budget_exceeded"); + expect(verdict.ledgerEventType).toBe("denied"); + }); + + it("halts on turn-cap and termination-cap breaches with the matching ledger severity", () => { + const turnCap = evaluateRunLoopHalt({ + runHalted: false, + usage: { budgetSpent: 1, turnsTaken: 5, elapsedMs: 1_000 }, + limits: LIMITS, + convergence: HEALTHY_CONVERGENCE, + }); + expect(turnCap.reason).toBe("turn_cap_exceeded"); + + const termination = evaluateRunLoopHalt({ + runHalted: false, + usage: { budgetSpent: 1, turnsTaken: 1, elapsedMs: 60_000 }, + limits: LIMITS, + convergence: HEALTHY_CONVERGENCE, + }); + expect(termination.reason).toBe("termination_cap_exceeded"); + expect(termination.ledgerEventType).toBe("kill_switch"); + }); + + it("keeps a prior halt latched until an operator clears it", () => { + const latched = evaluateRunLoopHalt({ + runHalted: true, + usage: HEALTHY_USAGE, + limits: LIMITS, + convergence: HEALTHY_CONVERGENCE, + }); + expect(latched.reason).toBe("prior_halt"); + expect(latched.canClaimNext).toBe(false); + expect(clearRunLoopHalt()).toEqual({ runHalted: false }); + }); + + it("prefers termination over budget and non-convergence when multiple signals trip together", () => { + const verdict = evaluateRunLoopHalt({ + runHalted: false, + usage: { budgetSpent: 999, turnsTaken: 999, elapsedMs: 60_000 }, + limits: LIMITS, + convergence: { attempts: 5, consecutiveFailures: 9, reenqueues: 9, reachedDone: false }, + }); + expect(verdict.reason).toBe("termination_cap_exceeded"); + }); + + it("buildRunLoopHaltGovernorLedgerEvent records halt context for retries and operator review", () => { + const verdict = evaluateRunLoopHalt({ + runHalted: false, + usage: HEALTHY_USAGE, + limits: LIMITS, + convergence: { attempts: 4, consecutiveFailures: 3, reenqueues: 0, reachedDone: false }, + }); + const event = buildRunLoopHaltGovernorLedgerEvent("acme/repo-a", "issue:42", verdict); + expect(event).toMatchObject({ + eventType: "denied", + actionClass: "run_loop", + decision: "halt", + reason: "non_convergence_detected", + payload: { + haltReason: "non_convergence", + inFlightIdentifier: "issue:42", + }, + }); + expect(event.payload?.convergenceReasons).toEqual( + expect.arrayContaining([expect.stringMatching(/consecutive failures/i)]), + ); + }); + + it("exposes issue vocabulary aliases over the pure calculators", () => { + expect(evaluateBudgetCaps(HEALTHY_USAGE, LIMITS).verdict).toBe("allowed"); + expect( + detectNonConvergence({ attempts: 4, consecutiveFailures: 3, reenqueues: 0, reachedDone: false }).tripped, + ).toBe(true); + }); +}); diff --git a/test/unit/miner-governor-run-halt.test.ts b/test/unit/miner-governor-run-halt.test.ts new file mode 100644 index 0000000000..7cce96f389 --- /dev/null +++ b/test/unit/miner-governor-run-halt.test.ts @@ -0,0 +1,148 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@jsonbored/gittensory-engine", async () => { + return import("../../packages/gittensory-engine/src/index"); +}); + +import { evaluateRunLoopBoundaryGate } from "../../packages/gittensory-miner/lib/governor-run-halt.js"; +import { initGovernorLedger } from "../../packages/gittensory-miner/lib/governor-ledger.js"; +import { initPortfolioQueueManager } from "../../packages/gittensory-miner/lib/portfolio-queue-manager.js"; +import { initPortfolioQueueStore } from "../../packages/gittensory-miner/lib/portfolio-queue.js"; + +const roots: string[] = []; +const ledgers: Array<{ close(): void }> = []; +const stores: Array<{ close(): void }> = []; + +afterEach(() => { + for (const ledger of ledgers.splice(0)) ledger.close(); + for (const store of stores.splice(0)) store.close(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +const LIMITS = { budget: 100, turns: 5, elapsedMs: 60_000 }; +const HEALTHY_USAGE = { budgetSpent: 10, turnsTaken: 1, elapsedMs: 1_000 }; +const HEALTHY_CONVERGENCE = { attempts: 1, consecutiveFailures: 0, reenqueues: 0, reachedDone: false }; + +describe("evaluateRunLoopBoundaryGate (#2347)", () => { + it("releases an in-flight portfolio item and records a halt when a flapping run is detected", () => { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-governor-run-halt-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const store = initPortfolioQueueStore(":memory:"); + stores.push(store); + const manager = initPortfolioQueueManager({ store, caps: { globalWipCap: 2, perRepoWipCap: 2 } }); + manager.enqueue({ repoFullName: "acme/repo-a", identifier: "issue:42", priority: 1 }); + const inFlight = store.dequeueNext(); + expect(inFlight?.status).toBe("in_progress"); + + const halted = evaluateRunLoopBoundaryGate( + { + runHalted: false, + usage: HEALTHY_USAGE, + limits: LIMITS, + convergence: { attempts: 4, consecutiveFailures: 3, reenqueues: 0, reachedDone: false }, + inFlightItem: { repoFullName: "acme/repo-a", identifier: "issue:42" }, + markFailed: manager.markFailed.bind(manager), + }, + { append: (event) => ledger.appendGovernorEvent(event) }, + ); + + expect(halted.runHalted).toBe(true); + expect(halted.canClaimNext).toBe(false); + expect(halted.releasedItem).toMatchObject({ identifier: "issue:42", status: "queued" }); + expect(halted.recorded?.eventType).toBe("denied"); + expect(halted.recorded?.actionClass).toBe("run_loop"); + + const blockedClaim = evaluateRunLoopBoundaryGate( + { + runHalted: halted.runHalted, + usage: HEALTHY_USAGE, + limits: LIMITS, + convergence: HEALTHY_CONVERGENCE, + }, + { append: (event) => ledger.appendGovernorEvent(event) }, + ); + expect(blockedClaim.canClaimNext).toBe(false); + const claimed = blockedClaim.canClaimNext ? manager.claimNextBatch() : []; + expect(claimed).toEqual([]); + }); + + it("halts immediately on a budget-cap breach at the next iteration boundary", () => { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-governor-run-halt-budget-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + + const halted = evaluateRunLoopBoundaryGate( + { + runHalted: false, + usage: { budgetSpent: 100, turnsTaken: 1, elapsedMs: 1_000 }, + limits: LIMITS, + convergence: HEALTHY_CONVERGENCE, + }, + { append: (event) => ledger.appendGovernorEvent(event) }, + ); + + expect(halted.runHalted).toBe(true); + expect(halted.verdict.reason).toBe("budget_exceeded"); + expect(halted.recorded?.reason).toBe("budget_cap_exceeded"); + }); + + it("never halts or records a halt for a healthy run under both signals", () => { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-governor-run-halt-healthy-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + + const healthy = evaluateRunLoopBoundaryGate( + { + runHalted: false, + usage: HEALTHY_USAGE, + limits: LIMITS, + convergence: HEALTHY_CONVERGENCE, + }, + { append: (event) => ledger.appendGovernorEvent(event) }, + ); + + expect(healthy.runHalted).toBe(false); + expect(healthy.canClaimNext).toBe(true); + expect(healthy.recorded?.eventType).toBe("allowed"); + expect(healthy.releasedItem).toBeNull(); + }); + + it("does not re-append ledger rows while a prior halt remains latched", () => { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-governor-run-halt-latched-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const append = vi.fn((event) => ledger.appendGovernorEvent(event)); + + const first = evaluateRunLoopBoundaryGate( + { + runHalted: false, + usage: { budgetSpent: 100, turnsTaken: 1, elapsedMs: 1_000 }, + limits: LIMITS, + convergence: HEALTHY_CONVERGENCE, + }, + { append }, + ); + expect(first.recorded).not.toBeNull(); + + const second = evaluateRunLoopBoundaryGate( + { + runHalted: true, + usage: HEALTHY_USAGE, + limits: LIMITS, + convergence: HEALTHY_CONVERGENCE, + }, + { append }, + ); + expect(second.recorded).toBeNull(); + expect(second.canClaimNext).toBe(false); + expect(append).toHaveBeenCalledTimes(1); + }); +}); diff --git a/test/unit/miner-portfolio-queue.test.ts b/test/unit/miner-portfolio-queue.test.ts index 64c5d44be1..e4545e426d 100644 --- a/test/unit/miner-portfolio-queue.test.ts +++ b/test/unit/miner-portfolio-queue.test.ts @@ -5,7 +5,10 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { QUEUE_STATUSES, closeDefaultPortfolioQueueStore, + dequeueNext, + enqueue, initPortfolioQueueStore, + markFailed, resolvePortfolioQueueDbPath, } from "../../packages/gittensory-miner/lib/portfolio-queue.js"; @@ -108,6 +111,24 @@ describe("gittensory-miner portfolio/queue store (#2292)", () => { expect(store.markDone("o/a", "work")).toBeNull(); }); + it("markFailed releases an in-progress item back to queued for a halted run (#2347)", () => { + const store = tempStore(); + store.enqueue({ repoFullName: "o/a", identifier: "work", priority: 1 }); + expect(store.dequeueNext()?.status).toBe("in_progress"); + expect(store.markFailed("o/a", "work")?.status).toBe("queued"); + expect(store.markFailed("o/a", "work")).toBeNull(); + expect(store.dequeueNext()?.identifier).toBe("work"); + }); + + it("markFailed is a no-op for queued or done items", () => { + const store = tempStore(); + store.enqueue({ repoFullName: "o/a", identifier: "queued", priority: 1 }); + expect(store.markFailed("o/a", "queued")).toBeNull(); + store.markDone("o/a", "queued"); + expect(store.markFailed("o/a", "queued")).toBeNull(); + expect(store.markFailed("o/a", "missing")).toBeNull(); + }); + it("isolates listQueue by repo and lists everything when unfiltered", () => { const store = tempStore(); store.enqueue({ repoFullName: "o/a", identifier: "1", priority: 1 }); @@ -153,6 +174,16 @@ describe("gittensory-miner portfolio/queue store (#2292)", () => { expect(store.dequeueNext()?.identifier).toBe("B"); }); + it("module-level markFailed delegates to the default portfolio-queue store (#2347)", () => { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-portfolio-default-")); + roots.push(root); + vi.stubEnv("GITTENSORY_MINER_PORTFOLIO_QUEUE_DB", join(root, "portfolio-queue.sqlite3")); + enqueue({ repoFullName: "o/a", identifier: "work", priority: 1 }); + expect(dequeueNext()?.status).toBe("in_progress"); + expect(markFailed("o/a", "work")?.status).toBe("queued"); + expect(markFailed("o/a", "work")).toBeNull(); + }); + it("rejects malformed inputs across the shared validation contract (enqueue, listQueue, markDone)", () => { const store = tempStore(); expect(() => store.enqueue({ repoFullName: "no-slash", identifier: "1" })).toThrow("invalid_repo_full_name"); @@ -165,5 +196,7 @@ describe("gittensory-miner portfolio/queue store (#2292)", () => { expect(() => store.listQueue("no-slash")).toThrow("invalid_repo_full_name"); expect(() => store.markDone("no-slash", "1")).toThrow("invalid_repo_full_name"); expect(() => store.markDone("o/a", " ")).toThrow("invalid_identifier"); + expect(() => store.markFailed("no-slash", "1")).toThrow("invalid_repo_full_name"); + expect(() => store.markFailed("o/a", " ")).toThrow("invalid_identifier"); }); });