diff --git a/src/selfhost/d1-size-probe.ts b/src/selfhost/d1-size-probe.ts index 3fb70b549d..a2379ad183 100644 --- a/src/selfhost/d1-size-probe.ts +++ b/src/selfhost/d1-size-probe.ts @@ -27,6 +27,7 @@ import { LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES, RETENTION_POLICY } from "../db/reten import { errorMessage } from "../utils/json"; import { incr } from "./metrics"; import type { VectorSample } from "./metrics"; +import { capturePostHogReviewFailure } from "./posthog"; export interface D1SizeProbeEnv { CLOUDFLARE_D1_MONITOR_ACCOUNT_ID?: string | undefined; @@ -175,6 +176,61 @@ interface D1ProbeSample { let lastSample: D1ProbeSample | null = null; +// -- Size-threshold ALERTING (#9435). The gauges below this block only ever reach /metrics, which means an +// operator with no scrape/dashboard stack gets NOTHING when the database approaches its cap -- and the cap is +// an outage, not a degradation: at 10 GB every write fails with `D1_ERROR: Exceeded maximum DB size`, +// including the webhook relay's own INSERT, so inbound delivery stops fleet-wide (observed 2026-07-06 and +// again 2026-07-26). These thresholds turn the same sample into an explicit structured console.error plus a +// PostHog capture, both of which land in surfaces the operator already watches, with no metrics stack needed. +// +// Latched per level: an alert fires when a sample CROSSES a threshold, not on every probe tick, and re-arms +// only after the size drops back below that threshold (hysteresis via strict-less-than on reset). -1/absent +// samples (probe failure keeps the previous reading) never change the latch. +/** D1's documented per-database maximum; file_size at this value = every write fails. */ +const D1_SIZE_CAP_BYTES = 10 * 1024 ** 3; +/** Early heads-up: ~3 GB of headroom left. Plenty of time to widen retention or plan a migration. */ +const D1_SIZE_WARN_RATIO = 0.7; +/** Act NOW: at the fleet's measured write rate the remaining headroom is weeks, not months. */ +const D1_SIZE_CRITICAL_RATIO = 0.85; + +type D1SizeAlertLevel = "none" | "warn" | "critical"; +let lastAlertLevel: D1SizeAlertLevel = "none"; + +function d1SizeAlertLevelFor(fileSizeBytes: number): D1SizeAlertLevel { + if (fileSizeBytes >= D1_SIZE_CAP_BYTES * D1_SIZE_CRITICAL_RATIO) return "critical"; + if (fileSizeBytes >= D1_SIZE_CAP_BYTES * D1_SIZE_WARN_RATIO) return "warn"; + return "none"; +} + +const D1_SIZE_ALERT_RANK: Record = { none: 0, warn: 1, critical: 2 }; + +/** Evaluate the freshly-sampled file size against the alert thresholds; exported for direct unit testing. */ +export function checkD1SizeThreshold(fileSizeBytes: number): void { + const level = d1SizeAlertLevelFor(fileSizeBytes); + if (D1_SIZE_ALERT_RANK[level] > D1_SIZE_ALERT_RANK[lastAlertLevel]) { + const percentOfCap = Math.round((fileSizeBytes / D1_SIZE_CAP_BYTES) * 100); + const message = `Cloudflare D1 database size ${percentOfCap}% of the 10 GB cap (${fileSizeBytes} bytes) — at 100% every write fails and inbound webhook delivery stops fleet-wide`; + incr("loopover_d1_size_threshold_alerts_total", { level }); + console.error( + JSON.stringify({ + level: "error", + event: "d1_size_threshold", + alertLevel: level, + fileSizeBytes, + percentOfCap, + capBytes: D1_SIZE_CAP_BYTES, + }), + ); + capturePostHogReviewFailure(new Error(message), { kind: "infra", alert_level: level, file_size_bytes: fileSizeBytes, percent_of_cap: percentOfCap }, "d1_size_threshold"); + } else if (D1_SIZE_ALERT_RANK[level] < D1_SIZE_ALERT_RANK[lastAlertLevel]) { + // Recovery (retention widened, data migrated): note it once, at plain log grade, and re-arm the latch. + console.log( + JSON.stringify({ level: "info", event: "d1_size_threshold_recovered", from: lastAlertLevel, to: level, fileSizeBytes }), + ); + } + lastAlertLevel = level; +} + function redactD1ProbeSecret(message: string, apiToken: string): string { return message.split(apiToken).join("[redacted]"); } @@ -227,6 +283,9 @@ export async function runD1SizeProbe(env: D1SizeProbeEnv, fetchImpl: typeof fetc fileSizeBytes: freshInfo?.fileSizeBytes ?? lastSample?.fileSizeBytes ?? -1, tableRowCounts: [...tableRowCountsByTable.values()], }; + // Threshold check only on a FRESH size reading: a failed fetch (carried-forward or -1 sample) must neither + // fire a duplicate alert nor "recover" the latch on stale data. + if (freshInfo) checkD1SizeThreshold(freshInfo.fileSizeBytes); } /** -1 sentinel (matching loopover_host_load_avg1_per_core's convention): distinguishes "probe disabled or @@ -254,7 +313,8 @@ export function d1SignalSnapshotsRowsPerKeySample(): number { return dedup.rowCount / dedup.distinctKeyCount; } -/** Test-only: reset the module-level sample between tests. */ +/** Test-only: reset the module-level sample and alert latch between tests. */ export function resetD1SizeProbeForTest(): void { lastSample = null; + lastAlertLevel = "none"; } diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts index d135305d51..ce148888fd 100644 --- a/src/selfhost/metrics.ts +++ b/src/selfhost/metrics.ts @@ -148,6 +148,7 @@ export const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ ["loopover_d1_table_row_count", { help: "Row count for a monitored D1 table, from the same probe as loopover_d1_database_size_bytes, labeled by table.", type: "gauge" }], ["loopover_signal_snapshots_rows_per_key", { help: "signal_snapshots row count divided by its distinct (signal_type, target_key) count, scoped to the latest-only-dedup signal types dedupeSignalSnapshots converges to ~1 row per key; -1 when the probe is disabled or has never completed a successful sample.", type: "gauge" }], ["loopover_d1_probe_errors_total", { help: "D1 size/row-count Management API probe failures, by part (database_info/table_row_count).", type: "counter" }], + ["loopover_d1_size_threshold_alerts_total", { help: "D1 size-threshold alerts fired on crossing 70%/85% of the 10 GB cap, by level (warn/critical). Latched per level; recovery re-arms (#9435).", type: "counter" }], ["loopover_agent_action_permission_denied_total", { help: "Agent actions denied for missing a required GitHub App write permission, by action class.", type: "counter" }], ["loopover_agent_action_permission_denied_suppressed_total", { help: "Repeat permission denials suppressed within the cooldown window (still counted here, but not re-audited), by action class.", type: "counter" }], ["loopover_ai_review_frozen_reuse_total", { help: "AI review passes that reused a frozen (maintainer-gated) prior verdict instead of re-running.", type: "counter" }], diff --git a/test/unit/selfhost-d1-size-probe.test.ts b/test/unit/selfhost-d1-size-probe.test.ts index d47dc182bf..0af625cf50 100644 --- a/test/unit/selfhost-d1-size-probe.test.ts +++ b/test/unit/selfhost-d1-size-probe.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES } from "../../src/db/retention"; import { + checkD1SizeThreshold, d1DatabaseSizeBytesSample, d1SignalSnapshotsRowsPerKeySample, d1TableRowCountSamples, @@ -13,6 +14,7 @@ import { type D1SizeProbeConfig, type D1SizeProbeEnv, } from "../../src/selfhost/d1-size-probe"; +import * as posthogModule from "../../src/selfhost/posthog"; import { renderMetrics, resetMetrics, gauge, gaugeVector, counterValue } from "../../src/selfhost/metrics"; afterEach(() => { @@ -368,3 +370,83 @@ describe("D1 metrics end-to-end via renderMetrics()", () => { expect(out).not.toContain('loopover_d1_table_row_count{table='); }); }); + +describe("checkD1SizeThreshold (#9435)", () => { + const GB = 1024 ** 3; + + it("stays silent below the warn threshold", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + checkD1SizeThreshold(6 * GB); // 60% — below the 70% warn line + expect(errorSpy).not.toHaveBeenCalled(); + expect(counterValue("loopover_d1_size_threshold_alerts_total", { level: "warn" })).toBe(0); + }); + + it("fires ONCE on crossing warn, with the structured log, counter, and PostHog capture", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogReviewFailure"); + checkD1SizeThreshold(7.5 * GB); // 75% — warn + checkD1SizeThreshold(7.6 * GB); // still warn — latched, no second alert + expect(errorSpy).toHaveBeenCalledTimes(1); + const logged = JSON.parse(errorSpy.mock.calls[0]![0] as string) as Record; + expect(logged).toMatchObject({ event: "d1_size_threshold", alertLevel: "warn", percentOfCap: 75 }); + expect(counterValue("loopover_d1_size_threshold_alerts_total", { level: "warn" })).toBe(1); + expect(captureSpy).toHaveBeenCalledTimes(1); + expect(captureSpy).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ kind: "infra", alert_level: "warn", percent_of_cap: 75 }), + "d1_size_threshold", + ); + }); + + it("escalates warn -> critical as a fresh alert, but never re-fires within a level", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + checkD1SizeThreshold(7.5 * GB); // warn + checkD1SizeThreshold(9 * GB); // 90% — critical, second alert + checkD1SizeThreshold(9.5 * GB); // still critical — latched + expect(errorSpy).toHaveBeenCalledTimes(2); + const second = JSON.parse(errorSpy.mock.calls[1]![0] as string) as Record; + expect(second).toMatchObject({ alertLevel: "critical", percentOfCap: 90 }); + expect(counterValue("loopover_d1_size_threshold_alerts_total", { level: "critical" })).toBe(1); + }); + + it("logs recovery at info grade, re-arms, and a re-crossing fires again", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + checkD1SizeThreshold(7.5 * GB); // warn — alert 1 + checkD1SizeThreshold(3 * GB); // back to none — recovery, no error + const recovered = JSON.parse(logSpy.mock.calls.at(-1)![0] as string) as Record; + expect(recovered).toMatchObject({ event: "d1_size_threshold_recovered", from: "warn", to: "none" }); + checkD1SizeThreshold(7.5 * GB); // re-crossing — alert 2 + expect(errorSpy).toHaveBeenCalledTimes(2); + }); + + it("a critical -> warn drop logs recovery without an alert, and returning to critical re-fires", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + checkD1SizeThreshold(9 * GB); // critical — alert 1 + checkD1SizeThreshold(7.5 * GB); // down to warn — recovery line, no new alert + expect(errorSpy).toHaveBeenCalledTimes(1); + const recovered = JSON.parse(logSpy.mock.calls.at(-1)![0] as string) as Record; + expect(recovered).toMatchObject({ event: "d1_size_threshold_recovered", from: "critical", to: "warn" }); + checkD1SizeThreshold(9 * GB); // back up — warn latch < critical ⇒ fresh alert + expect(errorSpy).toHaveBeenCalledTimes(2); + }); + + it("runD1SizeProbe feeds a fresh reading into the threshold check, and a failed size fetch does not", async () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + // Fresh reading at 90% of cap → the probe itself raises the alert. + await runD1SizeProbe(FULL_ENV, mockFetch({ + databaseInfo: () => new Response(envelope({ file_size: 9 * GB, num_tables: 3 }), { status: 200 }), + rowsForTable: () => ({ total: 1 }), + })); + const alertCalls = errorSpy.mock.calls.filter((c) => typeof c[0] === "string" && (c[0] as string).includes("d1_size_threshold")); + expect(alertCalls).toHaveLength(1); + // Size endpoint now fails: the carried-forward sample must not re-alert OR recover the latch. + await runD1SizeProbe(FULL_ENV, mockFetch({ + databaseInfo: () => new Response("{}", { status: 500 }), + rowsForTable: () => ({ total: 1 }), + })); + const after = errorSpy.mock.calls.filter((c) => typeof c[0] === "string" && (c[0] as string).includes("d1_size_threshold")); + expect(after).toHaveLength(1); + }); +});