fix(selfhost): alert explicitly when the monitored D1 approaches its 10 GB cap

src/selfhost/d1-size-probe.ts

@@ -27,6 +27,7 @@ import { LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES, RETENTION_POLICY } from "../db/reten
 import { errorMessage } from "../utils/json";
 import { incr } from "./metrics";
 import type { VectorSample } from "./metrics";
+import { capturePostHogReviewFailure } from "./posthog";
 
 export interface D1SizeProbeEnv {
   CLOUDFLARE_D1_MONITOR_ACCOUNT_ID?: string | undefined;
@@ -175,6 +176,61 @@ interface D1ProbeSample {
 
 let lastSample: D1ProbeSample | null = null;
 
+// -- Size-threshold ALERTING (#9435). The gauges below this block only ever reach /metrics, which means an
+// operator with no scrape/dashboard stack gets NOTHING when the database approaches its cap -- and the cap is
+// an outage, not a degradation: at 10 GB every write fails with `D1_ERROR: Exceeded maximum DB size`,
+// including the webhook relay's own INSERT, so inbound delivery stops fleet-wide (observed 2026-07-06 and
+// again 2026-07-26). These thresholds turn the same sample into an explicit structured console.error plus a
+// PostHog capture, both of which land in surfaces the operator already watches, with no metrics stack needed.
+//
+// Latched per level: an alert fires when a sample CROSSES a threshold, not on every probe tick, and re-arms
+// only after the size drops back below that threshold (hysteresis via strict-less-than on reset). -1/absent
+// samples (probe failure keeps the previous reading) never change the latch.
+/** D1's documented per-database maximum; file_size at this value = every write fails. */
+const D1_SIZE_CAP_BYTES = 10 * 1024 ** 3;
+/** Early heads-up: ~3 GB of headroom left. Plenty of time to widen retention or plan a migration. */
+const D1_SIZE_WARN_RATIO = 0.7;
+/** Act NOW: at the fleet's measured write rate the remaining headroom is weeks, not months. */
+const D1_SIZE_CRITICAL_RATIO = 0.85;
+
+type D1SizeAlertLevel = "none" | "warn" | "critical";
+let lastAlertLevel: D1SizeAlertLevel = "none";
+
+function d1SizeAlertLevelFor(fileSizeBytes: number): D1SizeAlertLevel {
+  if (fileSizeBytes >= D1_SIZE_CAP_BYTES * D1_SIZE_CRITICAL_RATIO) return "critical";
+  if (fileSizeBytes >= D1_SIZE_CAP_BYTES * D1_SIZE_WARN_RATIO) return "warn";
+  return "none";
+}
+
+const D1_SIZE_ALERT_RANK: Record<D1SizeAlertLevel, number> = { none: 0, warn: 1, critical: 2 };
+
+/** Evaluate the freshly-sampled file size against the alert thresholds; exported for direct unit testing. */
+export function checkD1SizeThreshold(fileSizeBytes: number): void {
+  const level = d1SizeAlertLevelFor(fileSizeBytes);
+  if (D1_SIZE_ALERT_RANK[level] > D1_SIZE_ALERT_RANK[lastAlertLevel]) {
+    const percentOfCap = Math.round((fileSizeBytes / D1_SIZE_CAP_BYTES) * 100);
+    const message = `Cloudflare D1 database size ${percentOfCap}% of the 10 GB cap (${fileSizeBytes} bytes) — at 100% every write fails and inbound webhook delivery stops fleet-wide`;
+    incr("loopover_d1_size_threshold_alerts_total", { level });
+    console.error(
+      JSON.stringify({
+        level: "error",
+        event: "d1_size_threshold",
+        alertLevel: level,
+        fileSizeBytes,
+        percentOfCap,
+        capBytes: D1_SIZE_CAP_BYTES,
+      }),
+    );
+    capturePostHogReviewFailure(new Error(message), { kind: "infra", alert_level: level, file_size_bytes: fileSizeBytes, percent_of_cap: percentOfCap }, "d1_size_threshold");
+  } else if (D1_SIZE_ALERT_RANK[level] < D1_SIZE_ALERT_RANK[lastAlertLevel]) {
+    // Recovery (retention widened, data migrated): note it once, at plain log grade, and re-arm the latch.
+    console.log(
+      JSON.stringify({ level: "info", event: "d1_size_threshold_recovered", from: lastAlertLevel, to: level, fileSizeBytes }),
+    );
+  }
+  lastAlertLevel = level;
+}
+
 function redactD1ProbeSecret(message: string, apiToken: string): string {
   return message.split(apiToken).join("[redacted]");
 }
@@ -227,6 +283,9 @@ export async function runD1SizeProbe(env: D1SizeProbeEnv, fetchImpl: typeof fetc
     fileSizeBytes: freshInfo?.fileSizeBytes ?? lastSample?.fileSizeBytes ?? -1,
     tableRowCounts: [...tableRowCountsByTable.values()],
   };
+  // Threshold check only on a FRESH size reading: a failed fetch (carried-forward or -1 sample) must neither
+  // fire a duplicate alert nor "recover" the latch on stale data.
+  if (freshInfo) checkD1SizeThreshold(freshInfo.fileSizeBytes);
 }
 
 /** -1 sentinel (matching loopover_host_load_avg1_per_core's convention): distinguishes "probe disabled or
@@ -254,7 +313,8 @@ export function d1SignalSnapshotsRowsPerKeySample(): number {
   return dedup.rowCount / dedup.distinctKeyCount;
 }
 
-/** Test-only: reset the module-level sample between tests. */
+/** Test-only: reset the module-level sample and alert latch between tests. */
 export function resetD1SizeProbeForTest(): void {
   lastSample = null;
+  lastAlertLevel = "none";
 }

src/selfhost/metrics.ts

@@ -148,6 +148,7 @@ export const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [
   ["loopover_d1_table_row_count", { help: "Row count for a monitored D1 table, from the same probe as loopover_d1_database_size_bytes, labeled by table.", type: "gauge" }],
   ["loopover_signal_snapshots_rows_per_key", { help: "signal_snapshots row count divided by its distinct (signal_type, target_key) count, scoped to the latest-only-dedup signal types dedupeSignalSnapshots converges to ~1 row per key; -1 when the probe is disabled or has never completed a successful sample.", type: "gauge" }],
   ["loopover_d1_probe_errors_total", { help: "D1 size/row-count Management API probe failures, by part (database_info/table_row_count).", type: "counter" }],
+  ["loopover_d1_size_threshold_alerts_total", { help: "D1 size-threshold alerts fired on crossing 70%/85% of the 10 GB cap, by level (warn/critical). Latched per level; recovery re-arms (#9435).", type: "counter" }],
   ["loopover_agent_action_permission_denied_total", { help: "Agent actions denied for missing a required GitHub App write permission, by action class.", type: "counter" }],
   ["loopover_agent_action_permission_denied_suppressed_total", { help: "Repeat permission denials suppressed within the cooldown window (still counted here, but not re-audited), by action class.", type: "counter" }],
   ["loopover_ai_review_frozen_reuse_total", { help: "AI review passes that reused a frozen (maintainer-gated) prior verdict instead of re-running.", type: "counter" }],

test/unit/selfhost-d1-size-probe.test.ts

@@ -1,6 +1,7 @@
 import { afterEach, describe, expect, it, vi } from "vitest";
 import { LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES } from "../../src/db/retention";
 import {
+  checkD1SizeThreshold,
   d1DatabaseSizeBytesSample,
   d1SignalSnapshotsRowsPerKeySample,
   d1TableRowCountSamples,
@@ -13,6 +14,7 @@ import {
   type D1SizeProbeConfig,
   type D1SizeProbeEnv,
 } from "../../src/selfhost/d1-size-probe";
+import * as posthogModule from "../../src/selfhost/posthog";
 import { renderMetrics, resetMetrics, gauge, gaugeVector, counterValue } from "../../src/selfhost/metrics";
 
 afterEach(() => {
@@ -368,3 +370,83 @@ describe("D1 metrics end-to-end via renderMetrics()", () => {
     expect(out).not.toContain('loopover_d1_table_row_count{table=');
   });
 });
+
+describe("checkD1SizeThreshold (#9435)", () => {
+  const GB = 1024 ** 3;
+
+  it("stays silent below the warn threshold", () => {
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    checkD1SizeThreshold(6 * GB); // 60% — below the 70% warn line
+    expect(errorSpy).not.toHaveBeenCalled();
+    expect(counterValue("loopover_d1_size_threshold_alerts_total", { level: "warn" })).toBe(0);
+  });
+
+  it("fires ONCE on crossing warn, with the structured log, counter, and PostHog capture", () => {
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const captureSpy = vi.spyOn(posthogModule, "capturePostHogReviewFailure");
+    checkD1SizeThreshold(7.5 * GB); // 75% — warn
+    checkD1SizeThreshold(7.6 * GB); // still warn — latched, no second alert
+    expect(errorSpy).toHaveBeenCalledTimes(1);
+    const logged = JSON.parse(errorSpy.mock.calls[0]![0] as string) as Record<string, unknown>;
+    expect(logged).toMatchObject({ event: "d1_size_threshold", alertLevel: "warn", percentOfCap: 75 });
+    expect(counterValue("loopover_d1_size_threshold_alerts_total", { level: "warn" })).toBe(1);
+    expect(captureSpy).toHaveBeenCalledTimes(1);
+    expect(captureSpy).toHaveBeenCalledWith(
+      expect.any(Error),
+      expect.objectContaining({ kind: "infra", alert_level: "warn", percent_of_cap: 75 }),
+      "d1_size_threshold",
+    );
+  });
+
+  it("escalates warn -> critical as a fresh alert, but never re-fires within a level", () => {
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    checkD1SizeThreshold(7.5 * GB); // warn
+    checkD1SizeThreshold(9 * GB); // 90% — critical, second alert
+    checkD1SizeThreshold(9.5 * GB); // still critical — latched
+    expect(errorSpy).toHaveBeenCalledTimes(2);
+    const second = JSON.parse(errorSpy.mock.calls[1]![0] as string) as Record<string, unknown>;
+    expect(second).toMatchObject({ alertLevel: "critical", percentOfCap: 90 });
+    expect(counterValue("loopover_d1_size_threshold_alerts_total", { level: "critical" })).toBe(1);
+  });
+
+  it("logs recovery at info grade, re-arms, and a re-crossing fires again", () => {
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
+    checkD1SizeThreshold(7.5 * GB); // warn — alert 1
+    checkD1SizeThreshold(3 * GB); // back to none — recovery, no error
+    const recovered = JSON.parse(logSpy.mock.calls.at(-1)![0] as string) as Record<string, unknown>;
+    expect(recovered).toMatchObject({ event: "d1_size_threshold_recovered", from: "warn", to: "none" });
+    checkD1SizeThreshold(7.5 * GB); // re-crossing — alert 2
+    expect(errorSpy).toHaveBeenCalledTimes(2);
+  });
+
+  it("a critical -> warn drop logs recovery without an alert, and returning to critical re-fires", () => {
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
+    checkD1SizeThreshold(9 * GB); // critical — alert 1
+    checkD1SizeThreshold(7.5 * GB); // down to warn — recovery line, no new alert
+    expect(errorSpy).toHaveBeenCalledTimes(1);
+    const recovered = JSON.parse(logSpy.mock.calls.at(-1)![0] as string) as Record<string, unknown>;
+    expect(recovered).toMatchObject({ event: "d1_size_threshold_recovered", from: "critical", to: "warn" });
+    checkD1SizeThreshold(9 * GB); // back up — warn latch < critical ⇒ fresh alert
+    expect(errorSpy).toHaveBeenCalledTimes(2);
+  });
+
+  it("runD1SizeProbe feeds a fresh reading into the threshold check, and a failed size fetch does not", async () => {
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    // Fresh reading at 90% of cap → the probe itself raises the alert.
+    await runD1SizeProbe(FULL_ENV, mockFetch({
+      databaseInfo: () => new Response(envelope({ file_size: 9 * GB, num_tables: 3 }), { status: 200 }),
+      rowsForTable: () => ({ total: 1 }),
+    }));
+    const alertCalls = errorSpy.mock.calls.filter((c) => typeof c[0] === "string" && (c[0] as string).includes("d1_size_threshold"));
+    expect(alertCalls).toHaveLength(1);
+    // Size endpoint now fails: the carried-forward sample must not re-alert OR recover the latch.
+    await runD1SizeProbe(FULL_ENV, mockFetch({
+      databaseInfo: () => new Response("{}", { status: 500 }),
+      rowsForTable: () => ({ total: 1 }),
+    }));
+    const after = errorSpy.mock.calls.filter((c) => typeof c[0] === "string" && (c[0] as string).includes("d1_size_threshold"));
+    expect(after).toHaveLength(1);
+  });
+});