Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions apps/gittensory-ui/src/lib/selfhost-env-reference.ts
Original file line number Diff line number Diff line change
Expand Up @@ -353,10 +353,6 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
name: "PAGERDUTY_COOLDOWN_MINUTES",
firstReference: "src/services/notify-pagerduty.ts",
},
{
name: "PAGERDUTY_MIN_SEVERITY",
firstReference: "src/services/notify-pagerduty.ts",
},
{
name: "PAGERDUTY_ROUTING_KEY",
firstReference: "src/services/notify-pagerduty.ts",
Expand Down Expand Up @@ -573,7 +569,6 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
"| `OTEL_TRACES_SAMPLER` | `src/selfhost/otel.ts` |",
"| `OTEL_TRACES_SAMPLER_ARG` | `src/selfhost/otel.ts` |",
"| `PAGERDUTY_COOLDOWN_MINUTES` | `src/services/notify-pagerduty.ts` |",
"| `PAGERDUTY_MIN_SEVERITY` | `src/services/notify-pagerduty.ts` |",
"| `PAGERDUTY_ROUTING_KEY` | `src/services/notify-pagerduty.ts` |",
"| `PGPOOL_MAX` | `src/selfhost/queue-common.ts` |",
"| `PGVECTOR_ENABLED` | `src/server.ts` |",
Expand Down
11 changes: 11 additions & 0 deletions src/env.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,17 @@ declare global {
* Defaults to 60 when unset. This is on top of PagerDuty's own `dedup_key` coalescing (which prevents
* duplicate *incidents*, not duplicate *pages* for a still-open one). */
PAGERDUTY_COOLDOWN_MINUTES?: string;
/** Sentry noise control (#5119): the minimum severity (`info` < `warning` < `error` < `critical`) that
* reaches Sentry from captureError/captureReviewFailure/forwardStructuredLogToSentry (src/selfhost/sentry.ts),
* for any repo not present in SENTRY_REPO_MIN_SEVERITY (a JSON `{repoFullName: severity}` map, same
* deliberately-untyped pattern as PAGERDUTY_REPO_MIN_SEVERITY). Defaults to `error` when unset — matches
* every capture path's pre-#5119 behavior exactly, so an operator who never touches these vars sees no
* change. Lower a specific repo's threshold via the map (e.g. to `info`) for full visibility while
* actively debugging it, without raising Sentry noise everywhere else. Shares one severity-threshold
* resolver with PAGERDUTY_MIN_SEVERITY (src/services/severity-threshold.ts) — not a parallel concept. */
SENTRY_MIN_SEVERITY?: string;
/** Per-repo override map for SENTRY_MIN_SEVERITY — see its doc comment for the shape and precedence. */
SENTRY_REPO_MIN_SEVERITY?: string;
GITTENSORY_CONTRIBUTOR_ISSUE_TOKEN?: string;
PRODUCT_USAGE_HASH_SALT?: string;
/** Server-to-server API bearer token — bypasses per-repo write checks (src/auth/security.ts). */
Expand Down
66 changes: 57 additions & 9 deletions src/selfhost/sentry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
} from "./otel";
import { hashedInstallationIdWith } from "./review-tracing";
import { queueDeadLetterReviveIntervalMs } from "./queue-common";
import { meetsSeverityThreshold, resolveSeverityThreshold, type LoopoverSeverity } from "../services/severity-threshold";

type SentryNs = typeof import("@sentry/node");
type SentryClient = NonNullable<ReturnType<SentryNs["init"]>>;
Expand Down Expand Up @@ -432,6 +433,41 @@ export async function buildSentryOpenTelemetryBridge(): Promise<OpenTelemetryBri
};
}

/** The repo a capture's context belongs to, for per-repo severity-threshold lookup (#5119) -- mirrors
* applyOperationalTags's own `repo`-over-`repository` normalization. `""` (never `undefined`) so
* {@link resolveSentryMinSeverity} always has a lookup key: a non-repo-scoped capture's (empty) per-repo map
* lookup simply misses and falls through to the global threshold, which is the correct behavior. */
function contextRepoFullName(context: Record<string, unknown> | undefined): string {
if (!context) return "";
const repo = typeof context.repo === "string" ? context.repo : typeof context.repository === "string" ? context.repository : undefined;
return repo ?? "";
}

/** Resolve the minimum severity Sentry capture for `repoFullName`: SENTRY_REPO_MIN_SEVERITY (a JSON
* `{repoFullName: severity}` map) wins, else the global SENTRY_MIN_SEVERITY, else `"error"` -- the quietest
* safe default, matching today's de facto behavior (every capture path below was already error/fatal-only
* before this resolver existed). Reads the real Node `process.env` directly: captureError/
* captureReviewFailure/forwardStructuredLogToSentry take no `env` parameter (initSentry's own env argument is
* not retained), so this is the only env self-host functions in this file can reach at capture time. */
function resolveSentryMinSeverity(repoFullName: string): LoopoverSeverity {
const processEnv = (globalThis as unknown as { process?: { env?: Record<string, string | undefined> } }).process?.env ?? {};
return resolveSeverityThreshold(processEnv as unknown as Env, repoFullName, "SENTRY_MIN_SEVERITY", "SENTRY_REPO_MIN_SEVERITY");
}

/** Map a structured log's own `level` field (Sentry-native `debug`/`info`/`warning`/`warn`/`error`/`fatal`) onto
* the shared 4-tier {@link LoopoverSeverity} taxonomy for threshold comparison. `debug` folds into `info` (the
* taxonomy has no separate debug tier, matching PagerDutySeverity's shape for consistency -- #5119). A level
* that ISN'T one of these recognized severity words (e.g. `"audit"` -- a log CATEGORY, not a severity grade)
* is treated as the quietest tier (`info`), never promoted to `error` -- matching this function's pre-#5119
* behavior of silently skipping anything that wasn't literally `error`/`fatal`. */
function normalizeLoopoverSeverity(level: string): LoopoverSeverity {
const lower = level.toLowerCase();
if (lower === "critical" || lower === "fatal") return "critical";
if (lower === "error") return "error";
if (lower === "warning" || lower === "warn") return "warning";
return "info";
}

/** Name a captured Error before capture so its Sentry issue title reads "eventName: message" instead of the
* generic "Error: message" (or a caught exception's own class name, e.g. "HttpError: ..."). Mirrors
* forwardStructuredLogToSentry's `errorEvent.name = event` below, but never mutates the caught value: some
Expand All @@ -449,7 +485,9 @@ function namedCaptureError(error: unknown, eventName?: string): Error {
return namedError;
}

/** Capture an error with optional structured context. No-op when Sentry is off. `eventName`, when given, becomes
/** Capture an error with optional structured context. No-op when Sentry is off OR the repo's resolved severity
* threshold (#5119) is above `error` (the fixed grade every call here represents) -- suppressed from Sentry,
* still visible in Workers Logs/stdout via the console call that led here. `eventName`, when given, becomes
* the Sentry issue title's prefix (see {@link namedCaptureError}) AND the grouping fingerprint (#5010) --
* Sentry's default stack-trace-based grouping fragments the SAME logical failure into separate issues whenever
* it is captured from more than one call site (e.g. two different functions each constructing the identical
Expand All @@ -461,6 +499,7 @@ export function captureError(
eventName?: string,
): void {
if (!active || !Sentry) return;
if (!meetsSeverityThreshold("error", resolveSentryMinSeverity(contextRepoFullName(context)))) return;
Sentry.withScope((scope) => {
setOtelTraceScope(scope);
if (context) { const safeContext = hashedInstallationContext(context); scope.setContext("gittensory", safeContext); applyOperationalTags(scope, safeContext); }
Expand All @@ -470,7 +509,8 @@ export function captureError(
}

/** Capture a failed review at ERROR level, tagged by repo/PR/SHA for triage. A review that cannot be produced is a
* real failure the maintainer must SEE — not a warning that hides in the noise. No-op when off. `eventName`, when
* real failure the maintainer must SEE — not a warning that hides in the noise. No-op when off OR the repo's
* resolved severity threshold (#5119) is above `error` (this always captures at error grade). `eventName`, when
* given, becomes the Sentry issue title's prefix AND the grouping fingerprint -- see {@link captureError}'s
* identical discipline and #5010. */
export function captureReviewFailure(
Expand All @@ -479,6 +519,7 @@ export function captureReviewFailure(
eventName?: string,
): void {
if (!active || !Sentry) return;
if (!meetsSeverityThreshold("error", resolveSentryMinSeverity(contextRepoFullName(context)))) return;
Sentry.withScope((scope) => {
scope.setLevel("error");
setOtelTraceScope(scope);
Expand Down Expand Up @@ -562,10 +603,14 @@ function summarizeLogFields(obj: Record<string, unknown>): string {
.join(", ");
}

/** Forward a structured console line to Sentry when it is an ERROR-level log. The engine logs operational
* failures (orb_broker_unavailable, gate-check errors, relay drops, …) as JSON strings, often via console.error.
* No-op when Sentry is off, the line isn't a JSON object string, or its level isn't error/fatal — routine logs
* (audit/info/no-level: job_complete, regate_sweep_throttled, …) are intentionally skipped. */
/** Forward a structured console line to Sentry when its level meets the repo's resolved severity threshold
* (#5119, default `error` — matches this function's pre-#5119 hardcoded error/fatal-only behavior byte for
* byte). The engine logs operational failures (orb_broker_unavailable, gate-check errors, relay drops, …) as
* JSON strings, often via console.error. No-op when Sentry is off, the line isn't a JSON object string, or it
* carries no level at all (and isn't from the error sink) — a log with no severity signal is a data-completeness
* gap, not a below-threshold decision, so it is always skipped regardless of any repo's configured threshold.
* An operator can lower a specific repo's threshold (SENTRY_REPO_MIN_SEVERITY) to `warning` or `info` to see
* routine logs from that repo while actively debugging it, without raising Sentry noise everywhere else. */
export function forwardStructuredLogToSentry(line: unknown, fromErrorSink = false): void {
if (!active || !Sentry) return;
if (typeof line !== "string" || line.charCodeAt(0) !== 123 /* "{" */) return;
Expand All @@ -579,11 +624,14 @@ export function forwardStructuredLogToSentry(line: unknown, fromErrorSink = fals
const safeObj = hashedInstallationContext(obj);
// A console.error sink is error-level by DEFAULT even when the JSON omits an explicit level (many engine error
// logs do) — that's how those errors reach Sentry instead of printing to stderr and vanishing. An EXPLICIT level
// always wins, so a deliberate level:"warn" emitted via console.error is still skipped.
// always wins over the error-sink default.
const explicitLevel = typeof obj.level === "string" ? obj.level : undefined;
const level = explicitLevel ?? (fromErrorSink ? "error" : undefined);
if (level !== "error" && level !== "fatal") return;
const severity = level === "fatal" ? "fatal" : "error";
if (!level) return; // no severity signal at all — never forwarded, independent of any threshold
const loopoverSeverity = normalizeLoopoverSeverity(level);
if (!meetsSeverityThreshold(loopoverSeverity, resolveSentryMinSeverity(contextRepoFullName(safeObj)))) return;
// Sentry's own native level string (setLevel below) — critical maps back to "fatal", its Sentry-native spelling.
const severity = loopoverSeverity === "critical" ? "fatal" : loopoverSeverity === "warning" ? "warning" : loopoverSeverity === "info" ? "info" : "error";
const event = typeof obj.event === "string" ? obj.event : undefined;
// Lead the Sentry title with the real failure detail (message → error), not just the event slug, so an operator
// sees WHAT broke straight from the issue list instead of having to open the context blob.
Expand Down
24 changes: 10 additions & 14 deletions src/services/notify-pagerduty.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { countRecentAuditEventsForActorAndTarget, recordAuditEvent } from "../db/repositories";
import { errorMessage } from "../utils/json";
import { meetsSeverityThreshold, resolveSeverityThreshold, type LoopoverSeverity } from "./severity-threshold";

// PagerDuty Events API v2 (https://developer.pagerduty.com/docs/events-api-v2/overview/). Experimental,
// default-OFF (LOOPOVER_ENABLE_PAGERDUTY) — a self-host operator opts in per #4937's paging epic.
Expand Down Expand Up @@ -88,23 +89,18 @@ export function resolvePagerDutyRoutingKey(env: Env, repoFullName: string): Page
: { status: "disabled", reason: fallback ? "invalid_global_key" : "missing_global_key" };
}

export type PagerDutySeverity = "critical" | "error" | "warning" | "info";

const SEVERITY_RANK: Record<PagerDutySeverity, number> = { info: 0, warning: 1, error: 2, critical: 3 };

function isPagerDutySeverity(value: unknown): value is PagerDutySeverity {
return value === "critical" || value === "error" || value === "warning" || value === "info";
}
/** @deprecated alias of {@link LoopoverSeverity} -- kept so existing imports (ops-wire.ts's
* classifyAnomalySeverity/worstAnomaly) don't need a rename. Shares the codebase's one severity-threshold
* concept (#5119) instead of a PagerDuty-only copy. */
export type PagerDutySeverity = LoopoverSeverity;

/** Resolve the minimum severity that pages for `repoFullName`: per-repo map entry, else the global override,
* else {@link DEFAULT_MIN_SEVERITY} — the quietest safe default, so an operator who never touches these vars
* still only gets paged for active-incident-grade conditions, never routine calibration nudges. */
* still only gets paged for active-incident-grade conditions, never routine calibration nudges. Delegates to
* the shared {@link resolveSeverityThreshold} resolver (#5119) so PagerDuty and Sentry share one
* severity-threshold concept, not two parallel ones. */
export function resolvePagerDutyMinSeverity(env: Env, repoFullName: string): PagerDutySeverity {
const map = repoJsonMap(env, "PAGERDUTY_REPO_MIN_SEVERITY");
const mapped = map[repoFullName.toLowerCase()];
if (isPagerDutySeverity(mapped)) return mapped;
const global = envString(env, "PAGERDUTY_MIN_SEVERITY");
return isPagerDutySeverity(global) ? global : DEFAULT_MIN_SEVERITY;
return resolveSeverityThreshold(env, repoFullName, "PAGERDUTY_MIN_SEVERITY", "PAGERDUTY_REPO_MIN_SEVERITY", DEFAULT_MIN_SEVERITY);
}

/** Coerce a JSON-map value or raw env string to a positive minute count; anything else (absent, zero,
Expand Down Expand Up @@ -171,7 +167,7 @@ export async function triggerPagerDutyIncident(
}

const minSeverity = resolvePagerDutyMinSeverity(env, params.repoFullName);
if (SEVERITY_RANK[params.severity] < SEVERITY_RANK[minSeverity]) {
if (!meetsSeverityThreshold(params.severity, minSeverity)) {
await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "denied", "below_min_severity", {
severity: params.severity,
minSeverity,
Expand Down
63 changes: 63 additions & 0 deletions src/services/severity-threshold.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// Shared severity-threshold resolver (#5119): global-default + per-repo-override precedence for gating how
// much observability noise an operator's ops channels receive. notify-pagerduty.ts's own
// resolvePagerDutyMinSeverity was the original, single-purpose version of this (paging only); sentry.ts's
// capture paths now share the SAME resolver so there is one severity-threshold concept in the codebase, not
// two parallel ones.

export type LoopoverSeverity = "critical" | "error" | "warning" | "info";

export const SEVERITY_RANK: Record<LoopoverSeverity, number> = { info: 0, warning: 1, error: 2, critical: 3 };

export function isLoopoverSeverity(value: unknown): value is LoopoverSeverity {
return value === "critical" || value === "error" || value === "warning" || value === "info";
}

/** True when `severity` meets or exceeds `threshold` (higher {@link SEVERITY_RANK}) -- the shared "should this
* actually fire" comparison every severity-gated channel (PagerDuty pages, Sentry captures) makes. */
export function meetsSeverityThreshold(severity: LoopoverSeverity, threshold: LoopoverSeverity): boolean {
return SEVERITY_RANK[severity] >= SEVERITY_RANK[threshold];
}

function envString(env: Env, name: string): string | undefined {
const fromEnv = (env as unknown as Record<string, unknown>)[name];
if (typeof fromEnv === "string" && fromEnv.trim().length > 0) return fromEnv.trim();
/* v8 ignore next 2 -- process.env is the self-host Node fallback; Worker/D1 tests pass values on Env. */
const processEnv = (globalThis as unknown as { process?: { env?: Record<string, string | undefined> } }).process?.env;
const fromProcess = processEnv?.[name];
return typeof fromProcess === "string" && fromProcess.trim().length > 0 ? fromProcess.trim() : undefined;
}

/** Parse a `{repoFullName: value}` JSON map off `envName`, lower-casing repo keys. Malformed/absent -> `{}`. */
function repoJsonMap(env: Env, envName: string): Record<string, unknown> {
const raw = envString(env, envName);
if (!raw) return {};
try {
const parsed = JSON.parse(raw) as unknown;
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
const out: Record<string, unknown> = {};
for (const [repo, value] of Object.entries(parsed)) out[repo.toLowerCase()] = value;
return out;
} catch {
return {};
}
}

/** Resolve the minimum severity threshold for `repoFullName`: a valid `repoMapVarName` JSON-map entry wins,
* else a valid `globalVarName` override, else `fallback` (default `"error"` -- the quietest safe default, so
* an operator who never touches these vars keeps today's de facto behavior). Mirrors
* {@link resolveDiscordWebhook}/{@link resolvePagerDutyRoutingKey}'s exact per-repo-override-wins-over-global
* precedence. `repoFullName` may be `""` for a non-repo-scoped event -- the (empty) map lookup simply misses
* and falls through to the global threshold, which is the correct behavior for global-only events. */
export function resolveSeverityThreshold(
env: Env,
repoFullName: string,
globalVarName: string,
repoMapVarName: string,
fallback: LoopoverSeverity = "error",
): LoopoverSeverity {
const map = repoJsonMap(env, repoMapVarName);
const mapped = map[repoFullName.toLowerCase()];
if (isLoopoverSeverity(mapped)) return mapped;
const global = envString(env, globalVarName);
return isLoopoverSeverity(global) ? global : fallback;
}
Loading
Loading