From f20bf9280c7a6004241cab4ba8fa909ff60f6afb Mon Sep 17 00:00:00 2001
From: JSONbored <49853598+JSONbored@users.noreply.github.com>
Date: Sun, 19 Jul 2026 16:25:19 -0700
Subject: [PATCH] fix(selfhost): widen orb-relay-drain checkin margin to match
 redeploy cadence

LOOPOVER-12 fired 1600+ missed-checkin outages over 2+ weeks. Ruled out the
reentrancy-guard overlap-skip (0 occurrences in 14 days of live
instrumentation) and in-process timeouts (~13 events, too few to explain the
volume). The container's own StartedAt resets on every redeploy unlike its
stable sidecars, and boot (image pull + Postgres-readiness wait + init)
routinely exceeds the old 2-minute margin on a fleet redeploying multiple
times a day. Raise checkinMargin from 2 to 6 minutes so a normal redeploy
doesn't trip a false outage, while a genuinely stuck instance still alerts
within ~21 minutes.
---
 src/selfhost/sentry.ts            | 16 +++++++++++++++-
 test/unit/selfhost-sentry.test.ts | 24 ++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/src/selfhost/sentry.ts b/src/selfhost/sentry.ts
index 7e427403f5..b4e96cd64d 100644
--- a/src/selfhost/sentry.ts
+++ b/src/selfhost/sentry.ts
@@ -109,7 +109,21 @@ const SENTRY_MONITORS: Record<SentryMonitorName, { slug: string; config: SentryM
     slug: "orb-relay-drain",
     config: {
       schedule: { type: "interval", value: 1, unit: "minute" },
-      checkinMargin: 2,
+      // #6685-followup-2 (LOOPOVER-12, 1600+ "missed check-in" occurrences over 2+ weeks, still recurring):
+      // checkinMargin: 2 was too tight for a self-host container that gets fully recreated on every redeploy,
+      // not just restarted in place -- both leading in-process hypotheses were investigated and ruled out with
+      // real production evidence: the reentrancy-guard overlap-skip added in #instrument-drain-skip (see
+      // withOrbRelayDrainReentrancyGuard's own header comment) never fired once in 14 days of live coverage
+      // (loopover_orb_relay_drain_skipped_total stayed at 0), and the genuine in-process timeout errors
+      // (LOOPOVER-1Y/1Q, kind: orb_relay_drain) total only ~13 occurrences -- nowhere near enough to explain
+      // 1600+ missed check-ins. What DOES correlate: this container's own StartedAt resets on every redeploy
+      // (unlike its sidecars', which stay up for days), and boot involves an image pull + up to a 30s Postgres-
+      // readiness retry loop (waitForPostgres, server.ts) + app init BEFORE this drain loop's first tick can
+      // even register -- routinely exceeding the old 2-minute margin on a fleet that redeploys multiple times a
+      // day. 6 minutes gives a normal single redeploy cycle room to complete without a false "outage" issue,
+      // while a genuinely stuck/crash-looping instance still alerts within ~3 x (1+6) = ~21 minutes
+      // (failureIssueThreshold below is unchanged -- only the margin, which is what the evidence points at).
+      checkinMargin: 6,
       maxRuntime: 1,
       failureIssueThreshold: 3,
       recoveryThreshold: 1,
diff --git a/test/unit/selfhost-sentry.test.ts b/test/unit/selfhost-sentry.test.ts
index 5f7cfe4361..57542172eb 100644
--- a/test/unit/selfhost-sentry.test.ts
+++ b/test/unit/selfhost-sentry.test.ts
@@ -816,6 +816,30 @@ describe("enabled when SENTRY_DSN is set", () => {
     expect(mocks.captureException).not.toHaveBeenCalled();
   });
 
+  it("gives orb-relay-drain a 6-minute checkin margin so a normal redeploy cycle doesn't false-positive as a missed check-in (LOOPOVER-12 regression)", async () => {
+    await initSentry({
+      SENTRY_DSN: "d",
+      SENTRY_ENVIRONMENT: "prod",
+    } as unknown as NodeJS.ProcessEnv);
+
+    await expect(
+      withSentryMonitor("orb-relay-drain", { jobType: "orb-relay-drain" }, async () => "ok"),
+    ).resolves.toBe("ok");
+
+    expect(mocks.captureCheckIn).toHaveBeenNthCalledWith(
+      1,
+      { monitorSlug: "loopover-selfhost-prod-orb-relay-drain", status: "in_progress" },
+      expect.objectContaining({
+        schedule: { type: "interval", value: 1, unit: "minute" },
+        checkinMargin: 6,
+        maxRuntime: 1,
+        failureIssueThreshold: 3,
+        recoveryThreshold: 1,
+      }),
+    );
+    expect(mocks.captureException).not.toHaveBeenCalled();
+  });
+
   it("records failed Sentry cron monitor check-ins with sanitized context", async () => {
     await initSentry({
       SENTRY_DSN: "d",