Skip to content

fix(selfhost): correlate Codex provider timeouts with job/PR context #3315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JSONbored merged 1 commit into from
Jul 5, 2026
Merged

fix(selfhost): correlate Codex provider timeouts with job/PR context #3315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 48 additions & 48 deletions apps/gittensory-ui/src/lib/selfhost-env-reference.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ export type SelfHostEnvReferenceRow = {
export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
{
name: "AI_COMBINE",
firstReference: "src/selfhost/ai.ts:1028",
firstReference: "src/selfhost/ai.ts:1080",
},
{
name: "AI_DUAL_REVIEW",
firstReference: "src/selfhost/ai.ts:1003",
firstReference: "src/selfhost/ai.ts:1055",
},
{
name: "AI_EMBED_API_KEY",
Expand All @@ -23,27 +23,27 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
},
{
name: "AI_EMBED_MODEL",
firstReference: "src/selfhost/ai.ts:900",
firstReference: "src/selfhost/ai.ts:952",
},
{
name: "AI_ON_MERGE",
firstReference: "src/selfhost/ai.ts:1030",
firstReference: "src/selfhost/ai.ts:1082",
},
{
name: "AI_PROVIDER",
firstReference: "src/selfhost/ai-config.ts:43",
},
{
name: "ANTHROPIC_AI_BASE_URL",
firstReference: "src/selfhost/ai.ts:904",
firstReference: "src/selfhost/ai.ts:956",
},
{
name: "ANTHROPIC_AI_MODEL",
firstReference: "src/selfhost/ai.ts:87",
firstReference: "src/selfhost/ai.ts:96",
},
{
name: "ANTHROPIC_API_KEY",
firstReference: "src/selfhost/ai.ts:903",
firstReference: "src/selfhost/ai.ts:955",
},
{
name: "BACKUP_ACKNOWLEDGED",
Expand All @@ -55,31 +55,31 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
},
{
name: "CLAUDE_AI_EFFORT",
firstReference: "src/selfhost/ai.ts:138",
firstReference: "src/selfhost/ai.ts:147",
},
{
name: "CLAUDE_AI_MODEL",
firstReference: "src/selfhost/ai.ts:79",
firstReference: "src/selfhost/ai.ts:88",
},
{
name: "CLAUDE_AI_TIMEOUT_MS",
firstReference: "src/selfhost/ai.ts:138",
firstReference: "src/selfhost/ai.ts:147",
},
{
name: "CODEX_AI_EFFORT",
firstReference: "src/selfhost/ai.ts:142",
firstReference: "src/selfhost/ai.ts:151",
},
{
name: "CODEX_AI_MODEL",
firstReference: "src/selfhost/ai.ts:83",
firstReference: "src/selfhost/ai.ts:92",
},
{
name: "CODEX_AI_TIMEOUT_MS",
firstReference: "src/selfhost/ai.ts:142",
firstReference: "src/selfhost/ai.ts:151",
},
{
name: "CODEX_HOME",
firstReference: "src/selfhost/ai.ts:309",
firstReference: "src/selfhost/ai.ts:318",
},
{
name: "CRON_INTERVAL_MS",
Expand Down Expand Up @@ -151,7 +151,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
},
{
name: "HOME",
firstReference: "src/selfhost/ai.ts:309",
firstReference: "src/selfhost/ai.ts:318",
},
{
name: "MAINTENANCE_ADMISSION_DEFER_MS",
Expand Down Expand Up @@ -199,39 +199,39 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
},
{
name: "OLLAMA_AI_API_KEY",
firstReference: "src/selfhost/ai.ts:897",
firstReference: "src/selfhost/ai.ts:949",
},
{
name: "OLLAMA_AI_BASE_URL",
firstReference: "src/selfhost/ai.ts:893",
firstReference: "src/selfhost/ai.ts:945",
},
{
name: "OLLAMA_AI_MODEL",
firstReference: "src/selfhost/ai.ts:91",
firstReference: "src/selfhost/ai.ts:100",
},
{
name: "OPENAI_AI_BASE_URL",
firstReference: "src/selfhost/ai.ts:895",
firstReference: "src/selfhost/ai.ts:947",
},
{
name: "OPENAI_AI_MODEL",
firstReference: "src/selfhost/ai.ts:92",
firstReference: "src/selfhost/ai.ts:101",
},
{
name: "OPENAI_API_KEY",
firstReference: "src/selfhost/ai.ts:897",
firstReference: "src/selfhost/ai.ts:949",
},
{
name: "OPENAI_COMPATIBLE_AI_API_KEY",
firstReference: "src/selfhost/ai.ts:897",
firstReference: "src/selfhost/ai.ts:949",
},
{
name: "OPENAI_COMPATIBLE_AI_BASE_URL",
firstReference: "src/selfhost/ai.ts:896",
firstReference: "src/selfhost/ai.ts:948",
},
{
name: "OPENAI_COMPATIBLE_AI_MODEL",
firstReference: "src/selfhost/ai.ts:93",
firstReference: "src/selfhost/ai.ts:102",
},
{
name: "ORB_AIR_GAP",
Expand Down Expand Up @@ -390,25 +390,25 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
"| Name | First reference |",
"| --- | --- |",
"| `AI_COMBINE` | `src/selfhost/ai.ts:1028` |",
"| `AI_DUAL_REVIEW` | `src/selfhost/ai.ts:1003` |",
"| `AI_COMBINE` | `src/selfhost/ai.ts:1080` |",
"| `AI_DUAL_REVIEW` | `src/selfhost/ai.ts:1055` |",
"| `AI_EMBED_API_KEY` | `src/server.ts:441` |",
"| `AI_EMBED_BASE_URL` | `src/server.ts:438` |",
"| `AI_EMBED_MODEL` | `src/selfhost/ai.ts:900` |",
"| `AI_ON_MERGE` | `src/selfhost/ai.ts:1030` |",
"| `AI_EMBED_MODEL` | `src/selfhost/ai.ts:952` |",
"| `AI_ON_MERGE` | `src/selfhost/ai.ts:1082` |",
"| `AI_PROVIDER` | `src/selfhost/ai-config.ts:43` |",
"| `ANTHROPIC_AI_BASE_URL` | `src/selfhost/ai.ts:904` |",
"| `ANTHROPIC_AI_MODEL` | `src/selfhost/ai.ts:87` |",
"| `ANTHROPIC_API_KEY` | `src/selfhost/ai.ts:903` |",
"| `ANTHROPIC_AI_BASE_URL` | `src/selfhost/ai.ts:956` |",
"| `ANTHROPIC_AI_MODEL` | `src/selfhost/ai.ts:96` |",
"| `ANTHROPIC_API_KEY` | `src/selfhost/ai.ts:955` |",
"| `BACKUP_ACKNOWLEDGED` | `src/server.ts:380` |",
"| `BROWSER_WS_ENDPOINT` | `src/selfhost/stubs/puppeteer.ts:11` |",
"| `CLAUDE_AI_EFFORT` | `src/selfhost/ai.ts:138` |",
"| `CLAUDE_AI_MODEL` | `src/selfhost/ai.ts:79` |",
"| `CLAUDE_AI_TIMEOUT_MS` | `src/selfhost/ai.ts:138` |",
"| `CODEX_AI_EFFORT` | `src/selfhost/ai.ts:142` |",
"| `CODEX_AI_MODEL` | `src/selfhost/ai.ts:83` |",
"| `CODEX_AI_TIMEOUT_MS` | `src/selfhost/ai.ts:142` |",
"| `CODEX_HOME` | `src/selfhost/ai.ts:309` |",
"| `CLAUDE_AI_EFFORT` | `src/selfhost/ai.ts:147` |",
"| `CLAUDE_AI_MODEL` | `src/selfhost/ai.ts:88` |",
"| `CLAUDE_AI_TIMEOUT_MS` | `src/selfhost/ai.ts:147` |",
"| `CODEX_AI_EFFORT` | `src/selfhost/ai.ts:151` |",
"| `CODEX_AI_MODEL` | `src/selfhost/ai.ts:92` |",
"| `CODEX_AI_TIMEOUT_MS` | `src/selfhost/ai.ts:151` |",
"| `CODEX_HOME` | `src/selfhost/ai.ts:318` |",
"| `CRON_INTERVAL_MS` | `src/server.ts:919` |",
"| `DATABASE_PATH` | `src/server.ts:250` |",
"| `DATABASE_URL` | `src/selfhost/preflight.ts:201` |",
Expand All @@ -426,7 +426,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
"| `GITHUB_INSTALLATION_CONCURRENCY_LIMIT` | `src/selfhost/installation-concurrency-admission.ts:43` |",
"| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:289` |",
"| `GITTENSORY_VERSION` | `src/selfhost/otel.ts:62` |",
"| `HOME` | `src/selfhost/ai.ts:309` |",
"| `HOME` | `src/selfhost/ai.ts:318` |",
"| `MAINTENANCE_ADMISSION_DEFER_MS` | `src/selfhost/maintenance-admission.ts:171` |",
"| `MAINTENANCE_ADMISSION_DRAIN_AGE_MS` | `src/selfhost/maintenance-admission.ts:145` |",
"| `MAINTENANCE_ADMISSION_ENABLED` | `src/selfhost/maintenance-admission.ts:126` |",
Expand All @@ -438,15 +438,15 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
"| `MIGRATIONS_DIR` | `src/server.ts:393` |",
"| `OBSERVABILITY_SMOKE_POLL_MS` | `scripts/smoke-observability-traces.mjs:8` |",
"| `OBSERVABILITY_SMOKE_TIMEOUT_MS` | `scripts/smoke-observability-traces.mjs:6` |",
"| `OLLAMA_AI_API_KEY` | `src/selfhost/ai.ts:897` |",
"| `OLLAMA_AI_BASE_URL` | `src/selfhost/ai.ts:893` |",
"| `OLLAMA_AI_MODEL` | `src/selfhost/ai.ts:91` |",
"| `OPENAI_AI_BASE_URL` | `src/selfhost/ai.ts:895` |",
"| `OPENAI_AI_MODEL` | `src/selfhost/ai.ts:92` |",
"| `OPENAI_API_KEY` | `src/selfhost/ai.ts:897` |",
"| `OPENAI_COMPATIBLE_AI_API_KEY` | `src/selfhost/ai.ts:897` |",
"| `OPENAI_COMPATIBLE_AI_BASE_URL` | `src/selfhost/ai.ts:896` |",
"| `OPENAI_COMPATIBLE_AI_MODEL` | `src/selfhost/ai.ts:93` |",
"| `OLLAMA_AI_API_KEY` | `src/selfhost/ai.ts:949` |",
"| `OLLAMA_AI_BASE_URL` | `src/selfhost/ai.ts:945` |",
"| `OLLAMA_AI_MODEL` | `src/selfhost/ai.ts:100` |",
"| `OPENAI_AI_BASE_URL` | `src/selfhost/ai.ts:947` |",
"| `OPENAI_AI_MODEL` | `src/selfhost/ai.ts:101` |",
"| `OPENAI_API_KEY` | `src/selfhost/ai.ts:949` |",
"| `OPENAI_COMPATIBLE_AI_API_KEY` | `src/selfhost/ai.ts:949` |",
"| `OPENAI_COMPATIBLE_AI_BASE_URL` | `src/selfhost/ai.ts:948` |",
"| `OPENAI_COMPATIBLE_AI_MODEL` | `src/selfhost/ai.ts:102` |",
"| `ORB_AIR_GAP` | `src/selfhost/orb-collector.ts:161` |",
"| `ORB_ANONYMIZE` | `src/selfhost/orb-collector.ts:174` |",
"| `ORB_APP_ID` | `src/selfhost/orb-collector.ts:59` |",
Expand Down
6 changes: 6 additions & 0 deletions src/queue/processors.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6151,6 +6151,10 @@ export async function runAiReviewForAdvisory(
// (the per-repo toggle). ANDed here with the operator flag + cutover allowlist to decide whether to ASK the
// model for line-anchored inline findings. Absent/false ⇒ the reviewer prompt is byte-identical (no findings).
reviewInlineComments?: boolean | undefined;
// The inbound webhook delivery id that triggered this review (#codex-timeout-fields) — forwarded to a
// self-host provider's failure log purely for operator correlation; never read by any review logic. Absent
// (e.g. a sweep/repair fan-out with no single originating delivery, or a unit test) ⇒ the log line omits it.
deliveryId?: string | undefined;
},
): Promise<
| {
Expand Down Expand Up @@ -6378,6 +6382,7 @@ export async function runAiReviewForAdvisory(
diff: enrichmentDiff,
actor: args.author,
mode: args.settings.aiReviewMode === "block" ? "block" : "advisory",
jobId: args.deliveryId,
providerKey,
grounding,
ragContext: ragContextResult?.text,
Expand Down Expand Up @@ -7943,6 +7948,7 @@ async function maybePublishPrPublicSurface(
reviewInstructions,
reviewExcludePaths,
reviewInlineComments,
deliveryId: webhook.deliveryId,
});
// `persistable === false` (only the lock-contention placeholder — see runAiReviewForAdvisory's return
// type doc comment) is excluded from EVERY write, not just the durable one: it describes a transient
Expand Down
58 changes: 55 additions & 3 deletions src/selfhost/ai.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,15 @@ interface AiRunOptions {
text?: string[]; // embedding input — the core's embedTexts passes { text: string[] }
max_tokens?: number;
temperature?: number;
// Correlation context for a provider-failure log (#codex-timeout-fields): purely observational, never read by a
// provider's own request logic. The caller (runWorkersOpinion) passes whatever of these it already has in scope
// for THIS review — job id and attempt are per-attempt, repoFullName/pullNumber identify the PR being reviewed —
// so an operator can correlate a `selfhost_ai_provider_failed` line back to the job/PR without cross-referencing
// timestamps. All optional: absent ⇒ the log line is byte-identical to before.
jobId?: string;
repoFullName?: string;
pullNumber?: number;
attempt?: number;
}
/** A chat completion (`response`) or an embedding result (`data`). Both optional: the core reads whichever it
* asked for (extractAiText → `response`, embedTexts → `data`), each defensive about the other being absent.
Expand Down Expand Up @@ -580,6 +589,13 @@ function logSelfHostAiProviderFailed(input: {
timeoutMs?: number | undefined;
error: unknown;
knownSecrets?: readonly string[] | undefined;
// Correlation context (#codex-timeout-fields), forwarded from the caller's AiRunOptions when supplied — never
// fabricated here. Undefined fields are dropped by JSON.stringify, so an omitted value keeps the log line
// byte-identical to before this field existed.
jobId?: string | undefined;
repoFullName?: string | undefined;
pullNumber?: number | undefined;
attempt?: number | undefined;
}): void {
console.error(
JSON.stringify({
Expand All @@ -589,6 +605,10 @@ function logSelfHostAiProviderFailed(input: {
model: input.model || "default",
effort: input.effort,
timeoutMs: input.timeoutMs,
jobId: input.jobId,
repoFullName: input.repoFullName,
pullNumber: input.pullNumber,
attempt: input.attempt,
error: errorMessage(input.error, input.knownSecrets),
}),
);
Expand Down Expand Up @@ -636,7 +656,18 @@ export function createClaudeCodeAi(parentEnv: Record<string, string | undefined>
if (!text) throw new Error("claude_code_empty_output");
return { response: text, usage: cliUsageFromStdout("claude-code", claudeModel, effort, stdoutForMetrics) };
} catch (error) {
logSelfHostAiProviderFailed({ provider: "claude-code", model: claudeModel, effort, timeoutMs, error, knownSecrets: token ? [token] : [] });
logSelfHostAiProviderFailed({
provider: "claude-code",
model: claudeModel,
effort,
timeoutMs,
error,
knownSecrets: token ? [token] : [],
jobId: options.jobId,
repoFullName: options.repoFullName,
pullNumber: options.pullNumber,
attempt: options.attempt,
});
throw error;
} finally {
if (attempted) recordCliUsageMetrics("claude-code", claudeModel, effort, stdoutForMetrics);
Expand Down Expand Up @@ -708,7 +739,17 @@ export function createCodexAi(
if (!text) throw new Error("codex_empty_output");
return { response: text, usage: cliUsageFromStdout("codex", codexModel, effort, stdoutForMetrics) };
} catch (error) {
logSelfHostAiProviderFailed({ provider: "codex", model: codexModel, effort, timeoutMs, error });
logSelfHostAiProviderFailed({
provider: "codex",
model: codexModel,
effort,
timeoutMs,
error,
jobId: options.jobId,
repoFullName: options.repoFullName,
pullNumber: options.pullNumber,
attempt: options.attempt,
});
throw error;
} finally {
if (attempted) recordCliUsageMetrics("codex", codexModel, effort, stdoutForMetrics);
Expand Down Expand Up @@ -805,7 +846,18 @@ export function createChainAi(providers: Array<{ name: string; ai: SelfHostAi }>
} catch (error) {
lastError = error;
failures.push({ provider: p.name, error: errorMessage(error) });
console.error(JSON.stringify({ level: "warn", event: "selfhost_ai_provider_failed_in_chain", provider: p.name, error: errorMessage(error) }));
console.error(
JSON.stringify({
level: "warn",
event: "selfhost_ai_provider_failed_in_chain",
provider: p.name,
jobId: options.jobId,
repoFullName: options.repoFullName,
pullNumber: options.pullNumber,
attempt: options.attempt,
error: errorMessage(error),
}),
);
}
}
recordAiProvidersExhausted();
Expand Down
Loading
Loading