From 71862986a40f2e607ce376a2cdf5468c3f2c8d92 Mon Sep 17 00:00:00 2001 From: jsdevninja Date: Tue, 21 Jul 2026 04:42:53 -0500 Subject: [PATCH] feat(ams): page PagerDuty on miner kill-switch trips On ledger trip, fire Events API v2 using the same LOOPOVER_ENABLE_PAGERDUTY / PAGERDUTY_ROUTING_KEY contract as notify-pagerduty. Resumes stay silent; paging is fire-and-forget. Pure alert builder lives at the miner IO seam so engine rebuilds do not trip MCP emit drift. Co-authored-by: Cursor --- .../content/docs/ams-kill-switch-incident.mdx | 7 + .../lib/governor-kill-switch.d.ts | 35 ++- .../lib/governor-kill-switch.js | 110 ++++++- .../lib/governor-kill-switch.ts | 145 ++++++++- src/services/notify-pagerduty.ts | 5 + .../unit/kill-switch-incident-runbook.test.ts | 32 ++ test/unit/miner-governor-kill-switch.test.ts | 289 +++++++++++++++++- 7 files changed, 611 insertions(+), 12 deletions(-) diff --git a/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx b/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx index 90a9bd9289..ac5f455ab5 100644 --- a/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx +++ b/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx @@ -52,6 +52,13 @@ Flag a misbehaving loop from any of: 3. Fleet observability — error-rate spikes, abnormal claim/submission patterns, elevated `consecutive_failures` / rejection-reason clustering, or an explicit customer/support escalation (see the rented-loop escalation path from #4806). +4. **Automated PagerDuty page** (#7666) — when `LOOPOVER_ENABLE_PAGERDUTY` is on and + `PAGERDUTY_ROUTING_KEY` is configured in the **miner** process, a kill-switch **trip** (engage) + fires an Events API v2 page using the same flag / routing-key / enqueue contract as ORB's + `notify-pagerduty` module. A resume does not page. The miner has no D1-backed severity floor or + cooldown (PagerDuty's own `dedup_key` still coalesces duplicate incidents). Treat an + `ams_kill_switch:*` incident as "operator already needed on the page" and start the 15-minute + response clock immediately. **Triage before acting:** decide one-repo vs fleet-wide. Prefer the narrowest scope that stops the harm so one bad tenant does not force an unnecessary global halt. diff --git a/packages/loopover-miner/lib/governor-kill-switch.d.ts b/packages/loopover-miner/lib/governor-kill-switch.d.ts index 191fa6e423..8d0820e149 100644 --- a/packages/loopover-miner/lib/governor-kill-switch.d.ts +++ b/packages/loopover-miner/lib/governor-kill-switch.d.ts @@ -19,11 +19,42 @@ export type RecordMinerKillSwitchTransitionInput = { previousScope: MinerKillSwitchScope; scope: MinerKillSwitchScope; }; +/** Pure page payload for a kill-switch TRIP (#7666). Null unless the transition is an engage into an active scope. */ +export type MinerKillSwitchPagerDutyAlert = { + repoFullName: string; + summary: string; + severity: "critical"; + dedupKey: string; + customDetails: { + previousScope: MinerKillSwitchScope; + scope: MinerKillSwitchScope; + reason: string; + }; +}; +/** + * Build the PagerDuty alert for a kill-switch trip. Returns null on resume / same-scope so clearing a halt + * never wakes anyone. `repoFullName` falls back to `ams/fleet` for a global halt with no single-repo context. + */ +export declare function buildMinerKillSwitchPagerDutyAlert(input: { + repoFullName?: string | null | undefined; + previousScope: MinerKillSwitchScope; + scope: MinerKillSwitchScope; +}): MinerKillSwitchPagerDutyAlert | null; +export type NotifyMinerKillSwitchTrip = (alert: MinerKillSwitchPagerDutyAlert, env: Record) => void | Promise; +/** + * Miner-side mirror of `triggerPagerDutyIncident` (#7666): same flag, same global routing key, same Events + * API v2 enqueue. No D1 audit/cooldown (miner has no Worker Env) -- PagerDuty's own `dedup_key` still + * coalesces duplicate incidents. Best-effort: never throws. + */ +export declare function notifyMinerKillSwitchPagerDuty(alert: MinerKillSwitchPagerDutyAlert, env?: Record): Promise; /** * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the - * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory - * or persisted); this module holds no state of its own. + * scope has not actually changed since the previous check -- callers own tracking the previous scope (in-memory + * or persisted); this module holds no state of its own. On a trip, also fires the PagerDuty page (#7666) + * unless `notify` is overridden (tests) or the integration flag/key is unset. */ export declare function recordMinerKillSwitchTransition(input: RecordMinerKillSwitchTransitionInput, options?: { append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry; + notify?: NotifyMinerKillSwitchTrip; + env?: Record; }): GovernorLedgerEntry | null; diff --git a/packages/loopover-miner/lib/governor-kill-switch.js b/packages/loopover-miner/lib/governor-kill-switch.js index 3c479144f3..96dc7b5073 100644 --- a/packages/loopover-miner/lib/governor-kill-switch.js +++ b/packages/loopover-miner/lib/governor-kill-switch.js @@ -2,6 +2,13 @@ // env, or for one repo, via its .loopover-miner.yml MinerGoalSpec) and records STATE TRANSITIONS to the // append-only governor ledger. Every-check allow/deny recording for a real write action is the fail-closed // Governor chokepoint's job (#2340), which consults this module first in its "safest wins" precedence. +// +// #7666: a TRIP also pages via the same PagerDuty Events API v2 contract ORB uses in +// `src/services/notify-pagerduty.ts` (LOOPOVER_ENABLE_PAGERDUTY + PAGERDUTY_ROUTING_KEY + enqueue URL + +// dedup_key). AMS trips only exist in this miner process (no hosted trip call site / no Worker Env), so +// the page lives here rather than calling `triggerPagerDutyIncident` directly. Resume stays silent — +// clearing a halt must not wake anyone. Best-effort and never throws: a paging failure must never block +// the ledger write or the mid-attempt abandon that depends on it. import { buildMinerKillSwitchTransitionGovernorLedgerEvent, isGlobalMinerKillSwitch, isMinerKillSwitchActive, resolveMinerKillSwitch, } from "@loopover/engine"; import { appendGovernorEvent } from "./governor-ledger.js"; /** @@ -14,16 +21,111 @@ export function checkMinerKillSwitch(input = {}) { const scope = resolveMinerKillSwitch({ global, repoPaused: input.repoPaused }); return { scope, active: isMinerKillSwitchActive(scope) }; } +/** + * Build the PagerDuty alert for a kill-switch trip. Returns null on resume / same-scope so clearing a halt + * never wakes anyone. `repoFullName` falls back to `ams/fleet` for a global halt with no single-repo context. + */ +export function buildMinerKillSwitchPagerDutyAlert(input) { + if (input.previousScope === input.scope) + return null; + if (!isMinerKillSwitchActive(input.scope)) + return null; + const repoFullName = (input.repoFullName ?? "").trim() || "ams/fleet"; + const reason = `${input.scope}_kill_switch_engaged`; + return { + repoFullName, + summary: input.scope === "global" + ? `AMS miner kill-switch engaged (global / fleet-wide)` + : `AMS miner kill-switch engaged (repo) for ${repoFullName}`, + severity: "critical", + dedupKey: `ams_kill_switch:${input.scope}:${repoFullName.toLowerCase()}`, + customDetails: { previousScope: input.previousScope, scope: input.scope, reason }, + }; +} +const PAGERDUTY_EVENTS_URL = "https://events.pagerduty.com/v2/enqueue"; +const ROUTING_KEY_RE = /^[a-f0-9]{32}$/i; +const TRUTHY_ENV = /^(1|true|yes|on)$/i; +function envString(env, name) { + const value = env[name]; + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; +} +function pagerDutyFailMessage(error) { + return (error instanceof Error ? error.message : String(error)).slice(0, 200); +} +function warnKillSwitchPagerDutyFailed(repo, error) { + console.warn(JSON.stringify({ event: "kill_switch_pagerduty_failed", repo, message: pagerDutyFailMessage(error) })); +} +/** + * Miner-side mirror of `triggerPagerDutyIncident` (#7666): same flag, same global routing key, same Events + * API v2 enqueue. No D1 audit/cooldown (miner has no Worker Env) -- PagerDuty's own `dedup_key` still + * coalesces duplicate incidents. Best-effort: never throws. + */ +export async function notifyMinerKillSwitchPagerDuty(alert, env = process.env) { + if (!TRUTHY_ENV.test((env.LOOPOVER_ENABLE_PAGERDUTY ?? "").trim())) + return; + const routingKey = envString(env, "PAGERDUTY_ROUTING_KEY"); + if (!routingKey || !ROUTING_KEY_RE.test(routingKey)) + return; + try { + const response = await fetch(PAGERDUTY_EVENTS_URL, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + routing_key: routingKey, + event_action: "trigger", + dedup_key: alert.dedupKey, + payload: { + summary: alert.summary.slice(0, 1024), + source: "loopover-miner", + severity: alert.severity, + timestamp: new Date().toISOString(), + component: alert.repoFullName, + custom_details: alert.customDetails, + }, + }), + signal: AbortSignal.timeout(5000), + }); + if (!response.ok) { + console.warn(JSON.stringify({ + event: "kill_switch_pagerduty_failed", + repo: alert.repoFullName, + status: response.status, + })); + } + } + catch (error) { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + } +} /** * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the - * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory - * or persisted); this module holds no state of its own. + * scope has not actually changed since the previous check -- callers own tracking the previous scope (in-memory + * or persisted); this module holds no state of its own. On a trip, also fires the PagerDuty page (#7666) + * unless `notify` is overridden (tests) or the integration flag/key is unset. */ export function recordMinerKillSwitchTransition(input, options = {}) { const event = buildMinerKillSwitchTransitionGovernorLedgerEvent(input); if (!event) return null; const append = options.append ?? appendGovernorEvent; - return append(event); + const recorded = append(event); + const alert = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: input.repoFullName, + previousScope: input.previousScope, + scope: input.scope, + }); + if (alert) { + const notify = options.notify ?? notifyMinerKillSwitchPagerDuty; + const env = options.env ?? process.env; + try { + void Promise.resolve(notify(alert, env)).catch((error) => { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + }); + } + catch (error) { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + } + } + return recorded; } -//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ292ZXJub3Ita2lsbC1zd2l0Y2guanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJnb3Zlcm5vci1raWxsLXN3aXRjaC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSw4R0FBOEc7QUFDOUcsd0dBQXdHO0FBQ3hHLDJHQUEyRztBQUMzRyx1R0FBdUc7QUFFdkcsT0FBTyxFQUNMLGlEQUFpRCxFQUNqRCx1QkFBdUIsRUFDdkIsdUJBQXVCLEVBQ3ZCLHNCQUFzQixHQUN2QixNQUFNLGtCQUFrQixDQUFDO0FBRTFCLE9BQU8sRUFBRSxtQkFBbUIsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBYTNEOzs7R0FHRztBQUNILE1BQU0sVUFBVSxvQkFBb0IsQ0FBQyxRQUFtQyxFQUFFO0lBQ3hFLE1BQU0sR0FBRyxHQUFHLEtBQUssQ0FBQyxHQUFHLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQztJQUNyQyxNQUFNLE1BQU0sR0FBRyx1QkFBdUIsQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUM1QyxNQUFNLEtBQUssR0FBRyxzQkFBc0IsQ0FBQyxFQUFFLE1BQU0sRUFBRSxVQUFVLEVBQUUsS0FBSyxDQUFDLFVBQVUsRUFBRSxDQUFDLENBQUM7SUFDL0UsT0FBTyxFQUFFLEtBQUssRUFBRSxNQUFNLEVBQUUsdUJBQXVCLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztBQUMzRCxDQUFDO0FBU0Q7Ozs7R0FJRztBQUNILE1BQU0sVUFBVSwrQkFBK0IsQ0FDN0MsS0FBMkMsRUFDM0MsVUFBaUYsRUFBRTtJQUVuRixNQUFNLEtBQUssR0FBRyxpREFBaUQsQ0FBQyxLQUFLLENBQUMsQ0FBQztJQUN2RSxJQUFJLENBQUMsS0FBSztRQUFFLE9BQU8sSUFBSSxDQUFDO0lBQ3hCLE1BQU0sTUFBTSxHQUFHLE9BQU8sQ0FBQyxNQUFNLElBQUksbUJBQW1CLENBQUM7SUFDckQsT0FBTyxNQUFNLENBQUMsS0FBaUMsQ0FBQyxDQUFDO0FBQ25ELENBQUMifQ== \ No newline at end of file +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ292ZXJub3Ita2lsbC1zd2l0Y2guanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJnb3Zlcm5vci1raWxsLXN3aXRjaC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSw4R0FBOEc7QUFDOUcsd0dBQXdHO0FBQ3hHLDJHQUEyRztBQUMzRyx1R0FBdUc7QUFDdkcsRUFBRTtBQUNGLHFGQUFxRjtBQUNyRix3R0FBd0c7QUFDeEcsd0dBQXdHO0FBQ3hHLHFHQUFxRztBQUNyRyx3R0FBd0c7QUFDeEcsa0VBQWtFO0FBRWxFLE9BQU8sRUFDTCxpREFBaUQsRUFDakQsdUJBQXVCLEVBQ3ZCLHVCQUF1QixFQUN2QixzQkFBc0IsR0FDdkIsTUFBTSxrQkFBa0IsQ0FBQztBQUUxQixPQUFPLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQWEzRDs7O0dBR0c7QUFDSCxNQUFNLFVBQVUsb0JBQW9CLENBQUMsUUFBbUMsRUFBRTtJQUN4RSxNQUFNLEdBQUcsR0FBRyxLQUFLLENBQUMsR0FBRyxJQUFJLE9BQU8sQ0FBQyxHQUFHLENBQUM7SUFDckMsTUFBTSxNQUFNLEdBQUcsdUJBQXVCLENBQUMsR0FBRyxDQUFDLENBQUM7SUFDNUMsTUFBTSxLQUFLLEdBQUcsc0JBQXNCLENBQUMsRUFBRSxNQUFNLEVBQUUsVUFBVSxFQUFFLEtBQUssQ0FBQyxVQUFVLEVBQUUsQ0FBQyxDQUFDO0lBQy9FLE9BQU8sRUFBRSxLQUFLLEVBQUUsTUFBTSxFQUFFLHVCQUF1QixDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7QUFDM0QsQ0FBQztBQXNCRDs7O0dBR0c7QUFDSCxNQUFNLFVBQVUsa0NBQWtDLENBQUMsS0FJbEQ7SUFDQyxJQUFJLEtBQUssQ0FBQyxhQUFhLEtBQUssS0FBSyxDQUFDLEtBQUs7UUFBRSxPQUFPLElBQUksQ0FBQztJQUNyRCxJQUFJLENBQUMsdUJBQXVCLENBQUMsS0FBSyxDQUFDLEtBQUssQ0FBQztRQUFFLE9BQU8sSUFBSSxDQUFDO0lBQ3ZELE1BQU0sWUFBWSxHQUFHLENBQUMsS0FBSyxDQUFDLFlBQVksSUFBSSxFQUFFLENBQUMsQ0FBQyxJQUFJLEVBQUUsSUFBSSxXQUFXLENBQUM7SUFDdEUsTUFBTSxNQUFNLEdBQUcsR0FBRyxLQUFLLENBQUMsS0FBSyxzQkFBc0IsQ0FBQztJQUNwRCxPQUFPO1FBQ0wsWUFBWTtRQUNaLE9BQU8sRUFDTCxLQUFLLENBQUMsS0FBSyxLQUFLLFFBQVE7WUFDdEIsQ0FBQyxDQUFDLHFEQUFxRDtZQUN2RCxDQUFDLENBQUMsNENBQTRDLFlBQVksRUFBRTtRQUNoRSxRQUFRLEVBQUUsVUFBVTtRQUNwQixRQUFRLEVBQUUsbUJBQW1CLEtBQUssQ0FBQyxLQUFLLElBQUksWUFBWSxDQUFDLFdBQVcsRUFBRSxFQUFFO1FBQ3hFLGFBQWEsRUFBRSxFQUFFLGFBQWEsRUFBRSxLQUFLLENBQUMsYUFBYSxFQUFFLEtBQUssRUFBRSxLQUFLLENBQUMsS0FBSyxFQUFFLE1BQU0sRUFBRTtLQUNsRixDQUFDO0FBQ0osQ0FBQztBQU9ELE1BQU0sb0JBQW9CLEdBQUcseUNBQXlDLENBQUM7QUFDdkUsTUFBTSxjQUFjLEdBQUcsaUJBQWlCLENBQUM7QUFDekMsTUFBTSxVQUFVLEdBQUcsb0JBQW9CLENBQUM7QUFFeEMsU0FBUyxTQUFTLENBQUMsR0FBdUMsRUFBRSxJQUFZO0lBQ3RFLE1BQU0sS0FBSyxHQUFHLEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUN4QixPQUFPLE9BQU8sS0FBSyxLQUFLLFFBQVEsSUFBSSxLQUFLLENBQUMsSUFBSSxFQUFFLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxDQUFDLENBQUMsQ0FBQyxTQUFTLENBQUM7QUFDekYsQ0FBQztBQUVELFNBQVMsb0JBQW9CLENBQUMsS0FBYztJQUMxQyxPQUFPLENBQUMsS0FBSyxZQUFZLEtBQUssQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLENBQUMsRUFBRSxHQUFHLENBQUMsQ0FBQztBQUNoRixDQUFDO0FBRUQsU0FBUyw2QkFBNkIsQ0FBQyxJQUFZLEVBQUUsS0FBYztJQUNqRSxPQUFPLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsRUFBRSxLQUFLLEVBQUUsOEJBQThCLEVBQUUsSUFBSSxFQUFFLE9BQU8sRUFBRSxvQkFBb0IsQ0FBQyxLQUFLLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQztBQUN0SCxDQUFDO0FBRUQ7Ozs7R0FJRztBQUNILE1BQU0sQ0FBQyxLQUFLLFVBQVUsOEJBQThCLENBQ2xELEtBQW9DLEVBQ3BDLE1BQTBDLE9BQU8sQ0FBQyxHQUFHO0lBRXJELElBQUksQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDLENBQUMsR0FBRyxDQUFDLHlCQUF5QixJQUFJLEVBQUUsQ0FBQyxDQUFDLElBQUksRUFBRSxDQUFDO1FBQUUsT0FBTztJQUMzRSxNQUFNLFVBQVUsR0FBRyxTQUFTLENBQUMsR0FBRyxFQUFFLHVCQUF1QixDQUFDLENBQUM7SUFDM0QsSUFBSSxDQUFDLFVBQVUsSUFBSSxDQUFDLGNBQWMsQ0FBQyxJQUFJLENBQUMsVUFBVSxDQUFDO1FBQUUsT0FBTztJQUU1RCxJQUFJLENBQUM7UUFDSCxNQUFNLFFBQVEsR0FBRyxNQUFNLEtBQUssQ0FBQyxvQkFBb0IsRUFBRTtZQUNqRCxNQUFNLEVBQUUsTUFBTTtZQUNkLE9BQU8sRUFBRSxFQUFFLGNBQWMsRUFBRSxrQkFBa0IsRUFBRTtZQUMvQyxJQUFJLEVBQUUsSUFBSSxDQUFDLFNBQVMsQ0FBQztnQkFDbkIsV0FBVyxFQUFFLFVBQVU7Z0JBQ3ZCLFlBQVksRUFBRSxTQUFTO2dCQUN2QixTQUFTLEVBQUUsS0FBSyxDQUFDLFFBQVE7Z0JBQ3pCLE9BQU8sRUFBRTtvQkFDUCxPQUFPLEVBQUUsS0FBSyxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQyxFQUFFLElBQUksQ0FBQztvQkFDckMsTUFBTSxFQUFFLGdCQUFnQjtvQkFDeEIsUUFBUSxFQUFFLEtBQUssQ0FBQyxRQUFRO29CQUN4QixTQUFTLEVBQUUsSUFBSSxJQUFJLEVBQUUsQ0FBQyxXQUFXLEVBQUU7b0JBQ25DLFNBQVMsRUFBRSxLQUFLLENBQUMsWUFBWTtvQkFDN0IsY0FBYyxFQUFFLEtBQUssQ0FBQyxhQUFhO2lCQUNwQzthQUNGLENBQUM7WUFDRixNQUFNLEVBQUUsV0FBVyxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUM7U0FDbEMsQ0FBQyxDQUFDO1FBQ0gsSUFBSSxDQUFDLFFBQVEsQ0FBQyxFQUFFLEVBQUUsQ0FBQztZQUNqQixPQUFPLENBQUMsSUFBSSxDQUNWLElBQUksQ0FBQyxTQUFTLENBQUM7Z0JBQ2IsS0FBSyxFQUFFLDhCQUE4QjtnQkFDckMsSUFBSSxFQUFFLEtBQUssQ0FBQyxZQUFZO2dCQUN4QixNQUFNLEVBQUUsUUFBUSxDQUFDLE1BQU07YUFDeEIsQ0FBQyxDQUNILENBQUM7UUFDSixDQUFDO0lBQ0gsQ0FBQztJQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7UUFDZiw2QkFBNkIsQ0FBQyxLQUFLLENBQUMsWUFBWSxFQUFFLEtBQUssQ0FBQyxDQUFDO0lBQzNELENBQUM7QUFDSCxDQUFDO0FBRUQ7Ozs7O0dBS0c7QUFDSCxNQUFNLFVBQVUsK0JBQStCLENBQzdDLEtBQTJDLEVBQzNDLFVBSUksRUFBRTtJQUVOLE1BQU0sS0FBSyxHQUFHLGlEQUFpRCxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ3ZFLElBQUksQ0FBQyxLQUFLO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDeEIsTUFBTSxNQUFNLEdBQUcsT0FBTyxDQUFDLE1BQU0sSUFBSSxtQkFBbUIsQ0FBQztJQUNyRCxNQUFNLFFBQVEsR0FBRyxNQUFNLENBQUMsS0FBaUMsQ0FBQyxDQUFDO0lBRTNELE1BQU0sS0FBSyxHQUFHLGtDQUFrQyxDQUFDO1FBQy9DLFlBQVksRUFBRSxLQUFLLENBQUMsWUFBWTtRQUNoQyxhQUFhLEVBQUUsS0FBSyxDQUFDLGFBQWE7UUFDbEMsS0FBSyxFQUFFLEtBQUssQ0FBQyxLQUFLO0tBQ25CLENBQUMsQ0FBQztJQUNILElBQUksS0FBSyxFQUFFLENBQUM7UUFDVixNQUFNLE1BQU0sR0FBRyxPQUFPLENBQUMsTUFBTSxJQUFJLDhCQUE4QixDQUFDO1FBQ2hFLE1BQU0sR0FBRyxHQUFHLE9BQU8sQ0FBQyxHQUFHLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQztRQUN2QyxJQUFJLENBQUM7WUFDSCxLQUFLLE9BQU8sQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssRUFBRSxHQUFHLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxDQUFDLEtBQWMsRUFBRSxFQUFFO2dCQUNoRSw2QkFBNkIsQ0FBQyxLQUFLLENBQUMsWUFBWSxFQUFFLEtBQUssQ0FBQyxDQUFDO1lBQzNELENBQUMsQ0FBQyxDQUFDO1FBQ0wsQ0FBQztRQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7WUFDZiw2QkFBNkIsQ0FBQyxLQUFLLENBQUMsWUFBWSxFQUFFLEtBQUssQ0FBQyxDQUFDO1FBQzNELENBQUM7SUFDSCxDQUFDO0lBRUQsT0FBTyxRQUFRLENBQUM7QUFDbEIsQ0FBQyJ9 \ No newline at end of file diff --git a/packages/loopover-miner/lib/governor-kill-switch.ts b/packages/loopover-miner/lib/governor-kill-switch.ts index 71a4de6773..f91958edc2 100644 --- a/packages/loopover-miner/lib/governor-kill-switch.ts +++ b/packages/loopover-miner/lib/governor-kill-switch.ts @@ -2,6 +2,13 @@ // env, or for one repo, via its .loopover-miner.yml MinerGoalSpec) and records STATE TRANSITIONS to the // append-only governor ledger. Every-check allow/deny recording for a real write action is the fail-closed // Governor chokepoint's job (#2340), which consults this module first in its "safest wins" precedence. +// +// #7666: a TRIP also pages via the same PagerDuty Events API v2 contract ORB uses in +// `src/services/notify-pagerduty.ts` (LOOPOVER_ENABLE_PAGERDUTY + PAGERDUTY_ROUTING_KEY + enqueue URL + +// dedup_key). AMS trips only exist in this miner process (no hosted trip call site / no Worker Env), so +// the page lives here rather than calling `triggerPagerDutyIncident` directly. Resume stays silent — +// clearing a halt must not wake anyone. Best-effort and never throws: a paging failure must never block +// the ledger write or the mid-attempt abandon that depends on it. import { buildMinerKillSwitchTransitionGovernorLedgerEvent, @@ -41,17 +48,147 @@ export type RecordMinerKillSwitchTransitionInput = { scope: MinerKillSwitchScope; }; +/** Pure page payload for a kill-switch TRIP (#7666). Null unless the transition is an engage into an active scope. */ +export type MinerKillSwitchPagerDutyAlert = { + repoFullName: string; + summary: string; + severity: "critical"; + dedupKey: string; + customDetails: { + previousScope: MinerKillSwitchScope; + scope: MinerKillSwitchScope; + reason: string; + }; +}; + +/** + * Build the PagerDuty alert for a kill-switch trip. Returns null on resume / same-scope so clearing a halt + * never wakes anyone. `repoFullName` falls back to `ams/fleet` for a global halt with no single-repo context. + */ +export function buildMinerKillSwitchPagerDutyAlert(input: { + repoFullName?: string | null | undefined; + previousScope: MinerKillSwitchScope; + scope: MinerKillSwitchScope; +}): MinerKillSwitchPagerDutyAlert | null { + if (input.previousScope === input.scope) return null; + if (!isMinerKillSwitchActive(input.scope)) return null; + const repoFullName = (input.repoFullName ?? "").trim() || "ams/fleet"; + const reason = `${input.scope}_kill_switch_engaged`; + return { + repoFullName, + summary: + input.scope === "global" + ? `AMS miner kill-switch engaged (global / fleet-wide)` + : `AMS miner kill-switch engaged (repo) for ${repoFullName}`, + severity: "critical", + dedupKey: `ams_kill_switch:${input.scope}:${repoFullName.toLowerCase()}`, + customDetails: { previousScope: input.previousScope, scope: input.scope, reason }, + }; +} + +export type NotifyMinerKillSwitchTrip = ( + alert: MinerKillSwitchPagerDutyAlert, + env: Record, +) => void | Promise; + +const PAGERDUTY_EVENTS_URL = "https://events.pagerduty.com/v2/enqueue"; +const ROUTING_KEY_RE = /^[a-f0-9]{32}$/i; +const TRUTHY_ENV = /^(1|true|yes|on)$/i; + +function envString(env: Record, name: string): string | undefined { + const value = env[name]; + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; +} + +function pagerDutyFailMessage(error: unknown): string { + return (error instanceof Error ? error.message : String(error)).slice(0, 200); +} + +function warnKillSwitchPagerDutyFailed(repo: string, error: unknown): void { + console.warn(JSON.stringify({ event: "kill_switch_pagerduty_failed", repo, message: pagerDutyFailMessage(error) })); +} + +/** + * Miner-side mirror of `triggerPagerDutyIncident` (#7666): same flag, same global routing key, same Events + * API v2 enqueue. No D1 audit/cooldown (miner has no Worker Env) -- PagerDuty's own `dedup_key` still + * coalesces duplicate incidents. Best-effort: never throws. + */ +export async function notifyMinerKillSwitchPagerDuty( + alert: MinerKillSwitchPagerDutyAlert, + env: Record = process.env, +): Promise { + if (!TRUTHY_ENV.test((env.LOOPOVER_ENABLE_PAGERDUTY ?? "").trim())) return; + const routingKey = envString(env, "PAGERDUTY_ROUTING_KEY"); + if (!routingKey || !ROUTING_KEY_RE.test(routingKey)) return; + + try { + const response = await fetch(PAGERDUTY_EVENTS_URL, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + routing_key: routingKey, + event_action: "trigger", + dedup_key: alert.dedupKey, + payload: { + summary: alert.summary.slice(0, 1024), + source: "loopover-miner", + severity: alert.severity, + timestamp: new Date().toISOString(), + component: alert.repoFullName, + custom_details: alert.customDetails, + }, + }), + signal: AbortSignal.timeout(5000), + }); + if (!response.ok) { + console.warn( + JSON.stringify({ + event: "kill_switch_pagerduty_failed", + repo: alert.repoFullName, + status: response.status, + }), + ); + } + } catch (error) { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + } +} + /** * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the - * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory - * or persisted); this module holds no state of its own. + * scope has not actually changed since the previous check -- callers own tracking the previous scope (in-memory + * or persisted); this module holds no state of its own. On a trip, also fires the PagerDuty page (#7666) + * unless `notify` is overridden (tests) or the integration flag/key is unset. */ export function recordMinerKillSwitchTransition( input: RecordMinerKillSwitchTransitionInput, - options: { append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry } = {}, + options: { + append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry; + notify?: NotifyMinerKillSwitchTrip; + env?: Record; + } = {}, ): GovernorLedgerEntry | null { const event = buildMinerKillSwitchTransitionGovernorLedgerEvent(input); if (!event) return null; const append = options.append ?? appendGovernorEvent; - return append(event as AppendGovernorEventInput); + const recorded = append(event as AppendGovernorEventInput); + + const alert = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: input.repoFullName, + previousScope: input.previousScope, + scope: input.scope, + }); + if (alert) { + const notify = options.notify ?? notifyMinerKillSwitchPagerDuty; + const env = options.env ?? process.env; + try { + void Promise.resolve(notify(alert, env)).catch((error: unknown) => { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + }); + } catch (error) { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + } + } + + return recorded; } diff --git a/src/services/notify-pagerduty.ts b/src/services/notify-pagerduty.ts index 073aeaf4a0..bfcb554a91 100644 --- a/src/services/notify-pagerduty.ts +++ b/src/services/notify-pagerduty.ts @@ -218,3 +218,8 @@ export async function triggerPagerDutyIncident( await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "error", message.slice(0, 280)); } } + +// AMS miner kill-switch trips (#7666) page from packages/loopover-miner/lib/governor-kill-switch.ts +// (recordMinerKillSwitchTransition). That path mirrors this module's Events API v2 contract (flag + +// routing key + enqueue URL + dedup_key). There is no hosted AMS trip call site today, so do not add an +// unused wrapper here. diff --git a/test/unit/kill-switch-incident-runbook.test.ts b/test/unit/kill-switch-incident-runbook.test.ts index 582fc5e0fd..7fcd59bfab 100644 --- a/test/unit/kill-switch-incident-runbook.test.ts +++ b/test/unit/kill-switch-incident-runbook.test.ts @@ -36,6 +36,8 @@ describe("kill-switch incident runbook (#4809)", () => { expect(runbook).toContain("15 minutes"); expect(runbook).toContain("2 minutes"); expect(runbook).toContain("#7180"); + expect(runbook).toContain("LOOPOVER_ENABLE_PAGERDUTY"); + expect(runbook).toContain("ams_kill_switch"); expect(resolveMinerKillSwitch({ global: true, repoPaused: true })).toBe("global"); expect(resolveMinerKillSwitch({ global: false, repoPaused: true })).toBe("repo"); @@ -78,3 +80,33 @@ describe("kill-switch incident runbook (#4809)", () => { expect(runbook).toMatch(/do not reach for those commands/i); }); }); + +describe("buildMinerKillSwitchPagerDutyAlert (#7666)", () => { + it("covers trip / resume / same-scope / fleet / blank-repo branches", async () => { + const { buildMinerKillSwitchPagerDutyAlert } = await import("../../packages/loopover-miner/lib/governor-kill-switch.js"); + + expect( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "none", scope: "repo" }), + ).toMatchObject({ + repoFullName: "acme/widgets", + severity: "critical", + dedupKey: "ams_kill_switch:repo:acme/widgets", + }); + + expect(buildMinerKillSwitchPagerDutyAlert({ previousScope: "none", scope: "global" })).toMatchObject({ + repoFullName: "ams/fleet", + dedupKey: "ams_kill_switch:global:ams/fleet", + }); + + expect(buildMinerKillSwitchPagerDutyAlert({ repoFullName: null, previousScope: "none", scope: "global" })).toMatchObject({ + repoFullName: "ams/fleet", + }); + + expect(buildMinerKillSwitchPagerDutyAlert({ repoFullName: " ", previousScope: "none", scope: "global" })).toMatchObject({ + repoFullName: "ams/fleet", + }); + + expect(buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "repo", scope: "none" })).toBeNull(); + expect(buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "repo", scope: "repo" })).toBeNull(); + }); +}); diff --git a/test/unit/miner-governor-kill-switch.test.ts b/test/unit/miner-governor-kill-switch.test.ts index 412defcbf8..8d87a8aa12 100644 --- a/test/unit/miner-governor-kill-switch.test.ts +++ b/test/unit/miner-governor-kill-switch.test.ts @@ -7,7 +7,7 @@ vi.mock("@loopover/engine", async () => { return import("../../packages/loopover-engine/src/index"); }); -import { checkMinerKillSwitch, recordMinerKillSwitchTransition } from "../../packages/loopover-miner/lib/governor-kill-switch.js"; +import { checkMinerKillSwitch, notifyMinerKillSwitchPagerDuty, recordMinerKillSwitchTransition } from "../../packages/loopover-miner/lib/governor-kill-switch.js"; import { initGovernorLedger } from "../../packages/loopover-miner/lib/governor-ledger.js"; const roots: string[] = []; @@ -119,7 +119,7 @@ describe("recordMinerKillSwitchTransition (#2341)", () => { actionClass: "open_pr", previousScope: "none", scope: "repo", - }); + }, { notify: () => undefined }); expect(tripped?.decision).toBe("tripped"); closeDefaultGovernorLedger(); @@ -133,4 +133,289 @@ describe("recordMinerKillSwitchTransition (#2341)", () => { else process.env.LOOPOVER_MINER_GOVERNOR_LEDGER_DB = previousDbPath; } }); + + it("pages on trip via the injectable notify hook and stays silent on resume (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-page-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const notify = vi.fn(); + + recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { append: (event) => ledger.appendGovernorEvent(event), notify }, + ); + expect(notify).toHaveBeenCalledTimes(1); + expect(notify.mock.calls[0]?.[0]).toMatchObject({ + repoFullName: "acme/widgets", + severity: "critical", + dedupKey: "ams_kill_switch:repo:acme/widgets", + }); + + notify.mockClear(); + recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "repo", scope: "none" }, + { append: (event) => ledger.appendGovernorEvent(event), notify }, + ); + expect(notify).not.toHaveBeenCalled(); + }); + + it("swallows a rejected notify promise without failing the ledger write (#7666)", async () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-notify-reject-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: async () => { + throw new Error("async pagerduty down"); + }, + }, + ); + expect(tripped?.decision).toBe("tripped"); + await vi.waitFor(() => { + expect(warn).toHaveBeenCalledWith(expect.stringContaining("kill_switch_pagerduty_failed")); + }); + warn.mockRestore(); + }); + + it("a sync void notify is accepted without treating it as a rejected promise (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-sync-void-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + let called = false; + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: () => { + called = true; + }, + }, + ); + expect(tripped?.decision).toBe("tripped"); + expect(called).toBe(true); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it("swallows a sync Error throw from notify (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-notify-error-throw-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "global" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: () => { + throw new Error("pagerduty error fail"); + }, + }, + ); + expect(tripped?.decision).toBe("tripped"); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("pagerduty error fail")); + warn.mockRestore(); + }); + + it("swallows a sync non-Error throw from notify (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-notify-string-throw-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "global" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: () => { + throw "pagerduty string fail"; + }, + }, + ); + expect(tripped?.decision).toBe("tripped"); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("pagerduty string fail")); + warn.mockRestore(); + }); + + it("swallows a non-Error rejected notify promise (#7666)", async () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-notify-string-reject-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: async () => { + throw "async string fail"; + }, + }, + ); + await vi.waitFor(() => { + expect(warn).toHaveBeenCalledWith(expect.stringContaining("async string fail")); + }); + warn.mockRestore(); + }); + + it("uses the default notify + process.env when overrides are omitted (#7666)", async () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-default-notify-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const calls: unknown[] = []; + vi.stubGlobal("fetch", async (...args: unknown[]) => { + calls.push(args); + return new Response(null, { status: 202 }); + }); + const previousFlag = process.env.LOOPOVER_ENABLE_PAGERDUTY; + const previousKey = process.env.PAGERDUTY_ROUTING_KEY; + process.env.LOOPOVER_ENABLE_PAGERDUTY = "1"; + process.env.PAGERDUTY_ROUTING_KEY = "a".repeat(32); + try { + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { append: (event) => ledger.appendGovernorEvent(event) }, + ); + expect(tripped?.decision).toBe("tripped"); + await vi.waitFor(() => { + expect(calls.length).toBeGreaterThan(0); + }); + } finally { + if (previousFlag === undefined) delete process.env.LOOPOVER_ENABLE_PAGERDUTY; + else process.env.LOOPOVER_ENABLE_PAGERDUTY = previousFlag; + if (previousKey === undefined) delete process.env.PAGERDUTY_ROUTING_KEY; + else process.env.PAGERDUTY_ROUTING_KEY = previousKey; + vi.unstubAllGlobals(); + } + }); +}); + +describe("notifyMinerKillSwitchPagerDuty (#7666)", () => { + const VALID_KEY = "a".repeat(32); + const ALERT = { + repoFullName: "acme/widgets", + summary: "AMS miner kill-switch engaged (repo) for acme/widgets", + severity: "critical" as const, + dedupKey: "ams_kill_switch:repo:acme/widgets", + customDetails: { previousScope: "none" as const, scope: "repo" as const, reason: "repo_kill_switch_engaged" }, + }; + + it("no-ops when the PagerDuty flag is off or unset", async () => { + const calls: unknown[] = []; + vi.stubGlobal("fetch", async (...args: unknown[]) => { + calls.push(args); + return new Response(null, { status: 202 }); + }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "0", PAGERDUTY_ROUTING_KEY: VALID_KEY }); + await notifyMinerKillSwitchPagerDuty(ALERT, { PAGERDUTY_ROUTING_KEY: VALID_KEY }); + expect(calls).toHaveLength(0); + vi.unstubAllGlobals(); + }); + + it("posts Events API v2 when enabled with a valid routing key", async () => { + const calls: Array<{ url: string; body: Record }> = []; + vi.stubGlobal("fetch", async (url: RequestInfo | URL, init?: RequestInit) => { + calls.push({ url: String(url), body: init?.body ? (JSON.parse(String(init.body)) as Record) : {} }); + return new Response(null, { status: 202 }); + }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: VALID_KEY }); + expect(calls).toHaveLength(1); + expect(calls[0]?.url).toBe("https://events.pagerduty.com/v2/enqueue"); + expect(calls[0]?.body).toMatchObject({ + routing_key: VALID_KEY, + event_action: "trigger", + dedup_key: "ams_kill_switch:repo:acme/widgets", + payload: { source: "loopover-miner", severity: "critical", component: "acme/widgets" }, + }); + vi.unstubAllGlobals(); + }); + + it("no-ops when the routing key is missing, blank, invalid, or non-string", async () => { + const calls: unknown[] = []; + vi.stubGlobal("fetch", async (...args: unknown[]) => { + calls.push(args); + return new Response(null, { status: 202 }); + }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "1" }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: " " }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: "not-a-key" }); + await notifyMinerKillSwitchPagerDuty(ALERT, { + LOOPOVER_ENABLE_PAGERDUTY: "1", + PAGERDUTY_ROUTING_KEY: 42 as unknown as string, + }); + expect(calls).toHaveLength(0); + vi.unstubAllGlobals(); + }); + + it("warns but does not throw when PagerDuty returns a non-ok status", async () => { + vi.stubGlobal("fetch", async () => new Response(null, { status: 500 })); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + await expect( + notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "yes", PAGERDUTY_ROUTING_KEY: VALID_KEY }), + ).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("kill_switch_pagerduty_failed")); + warn.mockRestore(); + vi.unstubAllGlobals(); + }); + + it("never throws when fetch rejects with a non-Error value", async () => { + vi.stubGlobal("fetch", async () => { + throw "network string fail"; + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + await expect( + notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "true", PAGERDUTY_ROUTING_KEY: VALID_KEY }), + ).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("network string fail")); + warn.mockRestore(); + vi.unstubAllGlobals(); + }); + + it("never throws when fetch rejects with an Error", async () => { + vi.stubGlobal("fetch", async () => { + throw new Error("network down"); + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + await expect( + notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "true", PAGERDUTY_ROUTING_KEY: VALID_KEY }), + ).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("network down")); + warn.mockRestore(); + vi.unstubAllGlobals(); + }); + + it("defaults env to process.env when the second arg is omitted", async () => { + const calls: unknown[] = []; + vi.stubGlobal("fetch", async (...args: unknown[]) => { + calls.push(args); + return new Response(null, { status: 202 }); + }); + const previousFlag = process.env.LOOPOVER_ENABLE_PAGERDUTY; + const previousKey = process.env.PAGERDUTY_ROUTING_KEY; + process.env.LOOPOVER_ENABLE_PAGERDUTY = "on"; + process.env.PAGERDUTY_ROUTING_KEY = VALID_KEY; + try { + await notifyMinerKillSwitchPagerDuty(ALERT); + expect(calls).toHaveLength(1); + } finally { + if (previousFlag === undefined) delete process.env.LOOPOVER_ENABLE_PAGERDUTY; + else process.env.LOOPOVER_ENABLE_PAGERDUTY = previousFlag; + if (previousKey === undefined) delete process.env.PAGERDUTY_ROUTING_KEY; + else process.env.PAGERDUTY_ROUTING_KEY = previousKey; + vi.unstubAllGlobals(); + } + }); });