From 658e6dab620923c5bc3b3044c7b3b28d51b00ae5 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Fri, 26 Jun 2026 03:52:18 -0700 Subject: [PATCH] fix(orb): stale-token grace when the broker mint blips in broker mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A brokered self-host holds no GitHub App key, so a single Orb token-mint failure was fatal — the review failed (→ retry/DLQ) and an Orb blip during the re-mint window stalled the fleet. createInstallationToken now catches a broker mint failure and, if the cached installation token is STILL within its real expiry, serves it (a valid token beats a stalled review; an actually-expired token is never reused). A genuine outage with no usable cached token emits an alertable structured log (orb_broker_unavailable) and rethrows so the queue's retry/DLQ handles it. --- src/github/app.ts | 21 +++++++++++++++--- test/unit/github-app.test.ts | 43 ++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/github/app.ts b/src/github/app.ts index 322db2e703..edba19ba55 100644 --- a/src/github/app.ts +++ b/src/github/app.ts @@ -4,6 +4,7 @@ import { makeInstallationOctokit } from "./client"; import { maintainerControlPanelUrl } from "./footer"; import type { AgentActionMode } from "../settings/agent-execution"; import { signRs256Jwt } from "../utils/crypto"; +import { errorMessage } from "../utils/json"; import { evaluateGateCheck, formatCheckRunOutput, formatGateCheckOutput, type CheckRunAnnotationContext, type CheckRunOutput, type GateCheckConclusion, type GateCheckEvaluation, type GateCheckPolicy } from "../rules/advisory"; type CheckRunResponse = { @@ -57,9 +58,23 @@ export async function createInstallationToken(env: Env, installationId: number): // secret, so this branch is inert there → byte-identical. The token caches the same way (the install id is the // self-host's single bound install). See src/orb/broker-client. if (isOrbBrokerMode(env)) { - const brokered = await fetchBrokeredInstallationToken(env); - installationTokenCache.set(installationId, { token: brokered.token, expiresAtMs: brokered.expiresAtMs }); - return brokered.token; + try { + const brokered = await fetchBrokeredInstallationToken(env); + installationTokenCache.set(installationId, { token: brokered.token, expiresAtMs: brokered.expiresAtMs }); + return brokered.token; + } catch (error) { + // Stale-token grace (#2): a brokered self-host holds no App key, so without this a single Orb mint failure + // fails the review (→ retry/DLQ) and an Orb blip during the re-mint window stalls the fleet. If the cached + // token is STILL within its real expiry, serve it — a valid token beats a stalled review (NO dangerous reuse: + // an actually-expired token is never served). Otherwise emit an alertable structured log and rethrow so the + // queue's retry/DLQ handles a genuine outage. + if (cached && cached.expiresAtMs > Date.now()) { + console.warn(JSON.stringify({ level: "warn", event: "orb_broker_degraded_serving_cached_token", installationId, expiresInMs: cached.expiresAtMs - Date.now(), error: errorMessage(error) })); + return cached.token; + } + console.error(JSON.stringify({ level: "error", event: "orb_broker_unavailable", installationId, error: errorMessage(error) })); + throw error; + } } const jwt = await createAppJwt(env); const response = await timeoutFetch(`https://api.github.com/app/installations/${installationId}/access_tokens`, { diff --git a/test/unit/github-app.test.ts b/test/unit/github-app.test.ts index c1103ccd8b..0e90ada541 100644 --- a/test/unit/github-app.test.ts +++ b/test/unit/github-app.test.ts @@ -147,6 +147,49 @@ describe("GitHub check runs", () => { expect(brokerCalls).toBe(1); }); + it("#2: serves a still-valid cached token when the Orb mint fails (stale-token grace, no fleet stall)", async () => { + let calls = 0; + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/v1/orb/token")) { + calls += 1; + // First mint returns a token expiring within the 2-min safety margin → the next call re-mints; that re-mint fails. + if (calls === 1) return Response.json({ token: "tok-1", installationId: 1001, expiresAt: new Date(Date.now() + 90_000).toISOString() }); + return new Response("orb down", { status: 503 }); + } + return new Response("nf", { status: 404 }); + }); + const env = createTestEnv({ ORB_ENROLLMENT_SECRET: "orbsec_test" }); + expect(await createInstallationToken(env, 1001)).toBe("tok-1"); // caches a near-expiry token + expect(await createInstallationToken(env, 1001)).toBe("tok-1"); // re-mint fails → grace serves the still-valid cached token + expect(calls).toBe(2); // the second call DID attempt a re-mint, then fell back to the cache + }); + + it("#2: rethrows when the broker is down and there is no still-valid cached token", async () => { + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/v1/orb/token")) return new Response("orb down", { status: 503 }); + return new Response("nf", { status: 404 }); + }); + await expect(createInstallationToken(createTestEnv({ ORB_ENROLLMENT_SECRET: "orbsec_test" }), 1002)).rejects.toThrow(); + }); + + it("#2: rethrows when the only cached token has actually expired (no dangerous reuse)", async () => { + let calls = 0; + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/v1/orb/token")) { + calls += 1; + if (calls === 1) return Response.json({ token: "tok-old", installationId: 1003, expiresAt: new Date(Date.now() - 1_000).toISOString() }); + return new Response("orb down", { status: 503 }); + } + return new Response("nf", { status: 404 }); + }); + const env = createTestEnv({ ORB_ENROLLMENT_SECRET: "orbsec_test" }); + expect(await createInstallationToken(env, 1003)).toBe("tok-old"); // caches an already-expired token + await expect(createInstallationToken(env, 1003)).rejects.toThrow(); // re-mint fails + cached expired → rethrow + }); + it("fetches repository collaborator permissions with installation credentials", async () => { const privateKey = await generatePrivateKeyPem(); const calls: string[] = [];