diff --git a/.env.example b/.env.example index fa37e5a36d..b3cca1ea12 100644 --- a/.env.example +++ b/.env.example @@ -255,7 +255,15 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # LITESTREAM_REGION=us-east-1 # --- Queue worker (#977/#1201) --- -# QUEUE_CONCURRENCY=4 # max concurrent job-processing loops per instance (default 4; set 1 for strict serial processing) +# QUEUE_CONCURRENCY=4 # max concurrent job-processing loops per instance. Default 4; set 1 +# # for strict serial processing. 8 is a reasonable starting point for +# # a moderate-load instance (several active repos / a steady +# # contributor-PR stream) on a host with a few spare cores -- review +# # jobs are I/O-bound (GitHub + AI awaits dominate), so raising this +# # mostly buys parallelism, not CPU. Watch gittensory_queue_live_pending +# # / gittensory_queue_oldest_live_pending_age_seconds after raising it; +# # if those stay high, the bottleneck is elsewhere (GitHub rate limit, +# # AI latency, Postgres pool -- see PGPOOL_MAX above), not concurrency. # QUEUE_BACKGROUND_CONCURRENCY=1 # max low-priority/background jobs allowed to occupy QUEUE_CONCURRENCY slots # CONTRIBUTOR_EVIDENCE_BATCH_SIZE=150 # logins per build-contributor-evidence job; the scheduled run fans out into # # per-batch jobs above this so the per-login GitHub reads spread across the @@ -285,6 +293,22 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS=14400000 # trickle ceiling: force-admit a maintenance job that has waited this long (4h) # MAINTENANCE_ADMISSION_DRAIN_AGE_MS=600000 # drain ceiling: admit a job despite a backed-up lane once it has waited this long (10m); clamped to MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS +# --- Foreground queue liveness (#selfhost-queue-liveness) --- +# Live contributor-PR-review work (github-webhook, agent-regate-pr, agent-regate-sweep, recapture-preview) must +# always have a BOUNDED runnable trickle. Unlike maintenance jobs (which get an admission trickle floor of their +# own, see above), foreground jobs are never exempted from GitHub rate-limit deferral -- its worst case is +# unbounded (up to ~65 minutes per defer, re-triggered indefinitely under sustained pressure, e.g. right after a +# deploy floods a shared REST budget). This sweep periodically pulls +# back any foreground-priority pending job that has been genuinely waiting (since its own enqueue time, not reset +# by a re-defer) past the ceiling below, regardless of which mechanism deferred it -- mirroring the maintenance +# trickle's own escape hatch. Also runs once at boot, so a restart self-heals inherited over-deferral instead of +# needing manual intervention. All defaults are sane; every value is optional. +# FOREGROUND_LIVENESS_ENABLED=true # set false/0/off to fully disable this sweep +# FOREGROUND_LIVENESS_MAX_DEFER_MS=600000 # trickle ceiling: force-release a foreground job deferred this long (10m) +# FOREGROUND_LIVENESS_CHECK_INTERVAL_MS=60000 # sweep cadence -- deliberately NOT the 1s poll tick, so a job +# # still genuinely rate-limited waits for the next sweep instead +# # of busy-looping (1m) + # --- Caddy HTTPS terminator (#1203; requires --profile caddy) --- # DOMAIN=gittensory.example.com # fully-qualified domain; Caddy auto-obtains a Let's Encrypt cert diff --git a/.gittensory.yml.example b/.gittensory.yml.example index 745542528c..96a3d44070 100644 --- a/.gittensory.yml.example +++ b/.gittensory.yml.example @@ -177,6 +177,19 @@ gate: # Default: null (not configured). checkRunAppSlug: null + # CI check/status context names to treat as REQUIRED when GitHub branch protection returns no + # readable required-status-checks (unconfigured, or a 403 from a token lacking + # administration:read — common for GitHub App installations, especially self-host). Merged with + # any branch-protection required contexts when both are readable; used ALONE when branch + # protection is null/empty. A context listed here that never appears on the commit stays + # pending; a completed red check for a listed context fails the gate; every listed context + # settled clean resolves to a verified "passed" (no completeness warning). List of strings, or + # omit. Default: not configured (keeps today's fold-all fail-closed behavior when branch + # protection is also unreadable). Config-as-code only — no DB column or dashboard toggle. + expectedCiContexts: + - build + - test + # Composite merge-readiness gate (no min score). # off | advisory | block. Default: off. mergeReadiness: off diff --git a/apps/gittensory-ui/public/openapi.json b/apps/gittensory-ui/public/openapi.json index 274be1410d..58e2f4c18a 100644 --- a/apps/gittensory-ui/public/openapi.json +++ b/apps/gittensory-ui/public/openapi.json @@ -9058,6 +9058,12 @@ "claCheckRunAppSlug": { "type": "string", "nullable": true + }, + "expectedCiContexts": { + "type": "array", + "items": { + "type": "string" + } } }, "required": [ diff --git a/apps/gittensory-ui/src/lib/selfhost-env-reference.ts b/apps/gittensory-ui/src/lib/selfhost-env-reference.ts index 12735e807d..eb73bf04ec 100644 --- a/apps/gittensory-ui/src/lib/selfhost-env-reference.ts +++ b/apps/gittensory-ui/src/lib/selfhost-env-reference.ts @@ -7,23 +7,23 @@ export type SelfHostEnvReferenceRow = { export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ { name: "AI_COMBINE", - firstReference: "src/selfhost/ai.ts:930", + firstReference: "src/selfhost/ai.ts:936", }, { name: "AI_EMBED_API_KEY", - firstReference: "src/server.ts:423", + firstReference: "src/server.ts:425", }, { name: "AI_EMBED_BASE_URL", - firstReference: "src/server.ts:420", + firstReference: "src/server.ts:422", }, { name: "AI_EMBED_MODEL", - firstReference: "src/selfhost/ai.ts:826", + firstReference: "src/selfhost/ai.ts:832", }, { name: "AI_ON_MERGE", - firstReference: "src/selfhost/ai.ts:932", + firstReference: "src/selfhost/ai.ts:938", }, { name: "AI_PROVIDER", @@ -31,7 +31,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "ANTHROPIC_AI_BASE_URL", - firstReference: "src/selfhost/ai.ts:830", + firstReference: "src/selfhost/ai.ts:836", }, { name: "ANTHROPIC_AI_MODEL", @@ -39,11 +39,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "ANTHROPIC_API_KEY", - firstReference: "src/selfhost/ai.ts:829", + firstReference: "src/selfhost/ai.ts:835", }, { name: "BACKUP_ACKNOWLEDGED", - firstReference: "src/server.ts:362", + firstReference: "src/server.ts:364", }, { name: "BROWSER_WS_ENDPOINT", @@ -79,11 +79,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "CRON_INTERVAL_MS", - firstReference: "src/server.ts:872", + firstReference: "src/server.ts:885", }, { name: "DATABASE_PATH", - firstReference: "src/server.ts:245", + firstReference: "src/server.ts:247", }, { name: "DATABASE_URL", @@ -97,6 +97,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "DISCORD_WEBHOOK_URL", firstReference: "src/selfhost/discord-notify.ts:40", }, + { + name: "FOREGROUND_LIVENESS_ENABLED", + firstReference: "src/selfhost/foreground-liveness.ts:34", + }, { name: "GITHUB_APP_ID", firstReference: "src/selfhost/orb-collector.ts:59", @@ -107,11 +111,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "GITHUB_CACHE_TTL_SECONDS", - firstReference: "src/server.ts:491", + firstReference: "src/server.ts:493", }, { name: "GITTENSORY_REPO_CONFIG_DIR", - firstReference: "src/server.ts:279", + firstReference: "src/server.ts:281", }, { name: "GITTENSORY_VERSION", @@ -123,11 +127,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "MAINTENANCE_ADMISSION_ENABLED", - firstReference: "src/selfhost/maintenance-admission.ts:99", + firstReference: "src/selfhost/maintenance-admission.ts:111", }, { name: "MIGRATIONS_DIR", - firstReference: "src/server.ts:375", + firstReference: "src/server.ts:377", }, { name: "OBSERVABILITY_SMOKE_POLL_MS", @@ -139,11 +143,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "OLLAMA_AI_API_KEY", - firstReference: "src/selfhost/ai.ts:823", + firstReference: "src/selfhost/ai.ts:829", }, { name: "OLLAMA_AI_BASE_URL", - firstReference: "src/selfhost/ai.ts:819", + firstReference: "src/selfhost/ai.ts:825", }, { name: "OLLAMA_AI_MODEL", @@ -151,7 +155,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "OPENAI_AI_BASE_URL", - firstReference: "src/selfhost/ai.ts:821", + firstReference: "src/selfhost/ai.ts:827", }, { name: "OPENAI_AI_MODEL", @@ -159,15 +163,15 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "OPENAI_API_KEY", - firstReference: "src/selfhost/ai.ts:823", + firstReference: "src/selfhost/ai.ts:829", }, { name: "OPENAI_COMPATIBLE_AI_API_KEY", - firstReference: "src/selfhost/ai.ts:823", + firstReference: "src/selfhost/ai.ts:829", }, { name: "OPENAI_COMPATIBLE_AI_BASE_URL", - firstReference: "src/selfhost/ai.ts:822", + firstReference: "src/selfhost/ai.ts:828", }, { name: "OPENAI_COMPATIBLE_AI_MODEL", @@ -187,7 +191,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "ORB_BROKER_URL", - firstReference: "src/server.ts:921", + firstReference: "src/server.ts:934", }, { name: "ORB_COLLECTOR_TOKEN", @@ -203,7 +207,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "ORB_RELAY_MODE", - firstReference: "src/server.ts:923", + firstReference: "src/server.ts:936", }, { name: "OTEL_EXPORTER_OTLP_ENDPOINT", @@ -235,11 +239,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "PGVECTOR_ENABLED", - firstReference: "src/server.ts:225", + firstReference: "src/server.ts:227", }, { name: "PORT", - firstReference: "src/server.ts:671", + firstReference: "src/server.ts:684", }, { name: "PUBLIC_API_ORIGIN", @@ -255,7 +259,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "QDRANT_URL", - firstReference: "src/server.ts:510", + firstReference: "src/server.ts:512", }, { name: "QUEUE_BACKGROUND_CONCURRENCY", @@ -267,7 +271,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "REVIEW_AUDIT_DIR", - firstReference: "src/server.ts:555", + firstReference: "src/server.ts:557", }, { name: "SELFHOST_BUNDLE_ALL", @@ -303,23 +307,23 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "SETUP_OUTPUT_PATH", - firstReference: "src/server.ts:788", + firstReference: "src/server.ts:801", }, ]; export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| Name | First reference |", "| --- | --- |", - "| `AI_COMBINE` | `src/selfhost/ai.ts:930` |", - "| `AI_EMBED_API_KEY` | `src/server.ts:423` |", - "| `AI_EMBED_BASE_URL` | `src/server.ts:420` |", - "| `AI_EMBED_MODEL` | `src/selfhost/ai.ts:826` |", - "| `AI_ON_MERGE` | `src/selfhost/ai.ts:932` |", + "| `AI_COMBINE` | `src/selfhost/ai.ts:936` |", + "| `AI_EMBED_API_KEY` | `src/server.ts:425` |", + "| `AI_EMBED_BASE_URL` | `src/server.ts:422` |", + "| `AI_EMBED_MODEL` | `src/selfhost/ai.ts:832` |", + "| `AI_ON_MERGE` | `src/selfhost/ai.ts:938` |", "| `AI_PROVIDER` | `src/selfhost/ai-config.ts:43` |", - "| `ANTHROPIC_AI_BASE_URL` | `src/selfhost/ai.ts:830` |", + "| `ANTHROPIC_AI_BASE_URL` | `src/selfhost/ai.ts:836` |", "| `ANTHROPIC_AI_MODEL` | `src/selfhost/ai.ts:57` |", - "| `ANTHROPIC_API_KEY` | `src/selfhost/ai.ts:829` |", - "| `BACKUP_ACKNOWLEDGED` | `src/server.ts:362` |", + "| `ANTHROPIC_API_KEY` | `src/selfhost/ai.ts:835` |", + "| `BACKUP_ACKNOWLEDGED` | `src/server.ts:364` |", "| `BROWSER_WS_ENDPOINT` | `src/selfhost/stubs/puppeteer.ts:11` |", "| `CLAUDE_AI_EFFORT` | `src/selfhost/ai.ts:108` |", "| `CLAUDE_AI_MODEL` | `src/selfhost/ai.ts:49` |", @@ -328,38 +332,39 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `CODEX_AI_MODEL` | `src/selfhost/ai.ts:53` |", "| `CODEX_AI_TIMEOUT_MS` | `src/selfhost/ai.ts:112` |", "| `CODEX_HOME` | `src/selfhost/ai.ts:274` |", - "| `CRON_INTERVAL_MS` | `src/server.ts:872` |", - "| `DATABASE_PATH` | `src/server.ts:245` |", + "| `CRON_INTERVAL_MS` | `src/server.ts:885` |", + "| `DATABASE_PATH` | `src/server.ts:247` |", "| `DATABASE_URL` | `src/selfhost/preflight.ts:201` |", "| `DISCORD_REPO_WEBHOOKS` | `src/selfhost/discord-notify.ts:31` |", "| `DISCORD_WEBHOOK_URL` | `src/selfhost/discord-notify.ts:40` |", + "| `FOREGROUND_LIVENESS_ENABLED` | `src/selfhost/foreground-liveness.ts:34` |", "| `GITHUB_APP_ID` | `src/selfhost/orb-collector.ts:59` |", "| `GITHUB_APP_PRIVATE_KEY` | `src/selfhost/orb-collector.ts:166` |", - "| `GITHUB_CACHE_TTL_SECONDS` | `src/server.ts:491` |", - "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:279` |", + "| `GITHUB_CACHE_TTL_SECONDS` | `src/server.ts:493` |", + "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:281` |", "| `GITTENSORY_VERSION` | `src/selfhost/health.ts:29` |", "| `HOME` | `src/selfhost/ai.ts:274` |", - "| `MAINTENANCE_ADMISSION_ENABLED` | `src/selfhost/maintenance-admission.ts:99` |", - "| `MIGRATIONS_DIR` | `src/server.ts:375` |", + "| `MAINTENANCE_ADMISSION_ENABLED` | `src/selfhost/maintenance-admission.ts:111` |", + "| `MIGRATIONS_DIR` | `src/server.ts:377` |", "| `OBSERVABILITY_SMOKE_POLL_MS` | `scripts/smoke-observability-traces.mjs:8` |", "| `OBSERVABILITY_SMOKE_TIMEOUT_MS` | `scripts/smoke-observability-traces.mjs:6` |", - "| `OLLAMA_AI_API_KEY` | `src/selfhost/ai.ts:823` |", - "| `OLLAMA_AI_BASE_URL` | `src/selfhost/ai.ts:819` |", + "| `OLLAMA_AI_API_KEY` | `src/selfhost/ai.ts:829` |", + "| `OLLAMA_AI_BASE_URL` | `src/selfhost/ai.ts:825` |", "| `OLLAMA_AI_MODEL` | `src/selfhost/ai.ts:61` |", - "| `OPENAI_AI_BASE_URL` | `src/selfhost/ai.ts:821` |", + "| `OPENAI_AI_BASE_URL` | `src/selfhost/ai.ts:827` |", "| `OPENAI_AI_MODEL` | `src/selfhost/ai.ts:62` |", - "| `OPENAI_API_KEY` | `src/selfhost/ai.ts:823` |", - "| `OPENAI_COMPATIBLE_AI_API_KEY` | `src/selfhost/ai.ts:823` |", - "| `OPENAI_COMPATIBLE_AI_BASE_URL` | `src/selfhost/ai.ts:822` |", + "| `OPENAI_API_KEY` | `src/selfhost/ai.ts:829` |", + "| `OPENAI_COMPATIBLE_AI_API_KEY` | `src/selfhost/ai.ts:829` |", + "| `OPENAI_COMPATIBLE_AI_BASE_URL` | `src/selfhost/ai.ts:828` |", "| `OPENAI_COMPATIBLE_AI_MODEL` | `src/selfhost/ai.ts:63` |", "| `ORB_AIR_GAP` | `src/selfhost/orb-collector.ts:161` |", "| `ORB_ANONYMIZE` | `src/selfhost/orb-collector.ts:174` |", "| `ORB_APP_ID` | `src/selfhost/orb-collector.ts:59` |", - "| `ORB_BROKER_URL` | `src/server.ts:921` |", + "| `ORB_BROKER_URL` | `src/server.ts:934` |", "| `ORB_COLLECTOR_TOKEN` | `src/selfhost/orb-collector.ts:205` |", "| `ORB_COLLECTOR_URL` | `src/selfhost/orb-collector.ts:172` |", "| `ORB_ENROLLMENT_SECRET` | `src/selfhost/orb-collector.ts:165` |", - "| `ORB_RELAY_MODE` | `src/server.ts:923` |", + "| `ORB_RELAY_MODE` | `src/server.ts:936` |", "| `OTEL_EXPORTER_OTLP_ENDPOINT` | `src/selfhost/otel.ts:47` |", "| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | `src/selfhost/otel.ts:45` |", "| `OTEL_SERVICE_ENVIRONMENT` | `src/selfhost/otel.ts:60` |", @@ -367,15 +372,15 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `OTEL_TRACES_EXPORTER` | `src/selfhost/otel.ts:40` |", "| `OTEL_TRACES_SAMPLER` | `src/selfhost/otel.ts:74` |", "| `OTEL_TRACES_SAMPLER_ARG` | `src/selfhost/otel.ts:76` |", - "| `PGVECTOR_ENABLED` | `src/server.ts:225` |", - "| `PORT` | `src/server.ts:671` |", + "| `PGVECTOR_ENABLED` | `src/server.ts:227` |", + "| `PORT` | `src/server.ts:684` |", "| `PUBLIC_API_ORIGIN` | `src/selfhost/preflight.ts:192` |", "| `QDRANT_API_KEY` | `src/selfhost/qdrant-vectorize.ts:50` |", "| `QDRANT_DIM` | `src/selfhost/qdrant-vectorize.ts:71` |", - "| `QDRANT_URL` | `src/server.ts:510` |", + "| `QDRANT_URL` | `src/server.ts:512` |", "| `QUEUE_BACKGROUND_CONCURRENCY` | `src/selfhost/queue-common.ts:102` |", "| `REDIS_URL` | `src/selfhost/preflight.ts:144` |", - "| `REVIEW_AUDIT_DIR` | `src/server.ts:555` |", + "| `REVIEW_AUDIT_DIR` | `src/server.ts:557` |", "| `SELFHOST_BUNDLE_ALL` | `scripts/build-selfhost.mjs:13` |", "| `SELFHOST_SERVICE` | `scripts/smoke-observability-traces.mjs:5` |", "| `SELFHOST_SETUP_TOKEN` | `src/selfhost/preflight.ts:186` |", @@ -384,5 +389,5 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `SENTRY_RELEASE` | `src/selfhost/otel.ts:62` |", "| `SENTRY_SERVER_NAME` | `src/selfhost/sentry.ts:383` |", "| `SENTRY_TRACES_SAMPLE_RATE` | `src/selfhost/sentry.ts:171` |", - "| `SETUP_OUTPUT_PATH` | `src/server.ts:788` |", + "| `SETUP_OUTPUT_PATH` | `src/server.ts:801` |", ].join("\n"); diff --git a/config/examples/repo-override.gittensory.yml b/config/examples/repo-override.gittensory.yml index 072e2d4686..90e52bf70b 100644 --- a/config/examples/repo-override.gittensory.yml +++ b/config/examples/repo-override.gittensory.yml @@ -10,10 +10,16 @@ # Replace `owner`/`repo` in the destination path with the real values before using this file. # ============================================================================ -# Overrides ONLY `gate.enabled` for this repo — every other `gate.*` key (e.g. `duplicates`, -# `linkedIssue`) is inherited from the global default untouched. +# Overrides ONLY `gate.enabled` and `gate.expectedCiContexts` for this repo — every other `gate.*` +# key (e.g. `duplicates`, `linkedIssue`) is inherited from the global default untouched. gate: enabled: true + # This repo's own CI job/check names to trust as required when its branch protection is + # unreadable (a common self-host case — see ../../.gittensory.yml.example). Each repo's CI + # naming differs, so this is a per-repo override rather than a global default. + expectedCiContexts: + - build + - test # Array fields REPLACE the global default wholesale, never concatenate with it — this repo's # wanted-paths guidance is exactly this list, not global's list plus this one. diff --git a/grafana/dashboards/gittensory.json b/grafana/dashboards/gittensory.json index f7438ab1ff..cf25cd8992 100644 --- a/grafana/dashboards/gittensory.json +++ b/grafana/dashboards/gittensory.json @@ -2628,6 +2628,164 @@ "refId": "A" } ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 171 }, + "id": 145, + "panels": [], + "title": "Foreground Liveness (#selfhost-queue-liveness)", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 172 }, + "id": 146, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "Runnable Now (all priorities)", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_queue_runnable_now", + "legendFormat": "runnable now" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 172 }, + "id": 147, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "Live Runnable Now", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_queue_live_runnable_now", + "legendFormat": "live runnable now" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 172 }, + "id": 148, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "Processing", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_queue_processing", + "legendFormat": "processing" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 120 }, + { "color": "red", "value": 300 } + ] + }, + "unit": "s" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 172 }, + "id": 149, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "Oldest Live Runnable Age", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_queue_oldest_live_runnable_age_seconds", + "legendFormat": "oldest live runnable age" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 172 }, + "id": 150, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "Foreground Liveness Released (total)", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_jobs_foreground_liveness_released_total or vector(0)", + "legendFormat": "released" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "short" } + }, + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 176 }, + "id": 151, + "options": { + "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "Foreground Liveness Releases by Reason", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (reason) (rate(gittensory_jobs_foreground_liveness_released_by_reason_total[15m])) or vector(0)", + "legendFormat": "{{reason}}", + "refId": "A" + } + ] } ], "refresh": "30s", @@ -2652,5 +2810,5 @@ "timezone": "browser", "title": "Gittensory Self-Host", "uid": "gittensory-selfhost", - "version": 6 + "version": 8 } diff --git a/prometheus/rules/alerts.yml b/prometheus/rules/alerts.yml index fc57e83d9e..c2d3ae06ea 100644 --- a/prometheus/rules/alerts.yml +++ b/prometheus/rules/alerts.yml @@ -133,6 +133,35 @@ groups: description: "The oldest live-priority queue job has been pending for {{ $value | printf \"%.0f\" }}s. Maintenance admission already yields to live work, so check AI latency, GitHub rate limits, DB pressure, or host CPU (gittensory_host_load_avg1_per_core)." runbook: "Open the Runtime Pressure & Maintenance row. If gittensory_host_load_avg1_per_core is elevated, a co-located CI runner or other host process is starving the app -- see docker-compose.yml's runner isolation guidance." + - alert: GittensoryLiveQueueNoRunnableWork + # #selfhost-queue-liveness: the precise "is the foreground lane actually stuck" signal, distinct from + # GittensoryLiveQueueStarved's age-based check -- a large live-pending count on its own is NORMAL (a + # legitimate burst, or work intentionally staggered/rate-deferred); this only fires when there is + # PENDING live work AND NOTHING runnable right now, sustained. That combination is exactly the + # production incident this invariant exists to make impossible: hundreds of pending contributor-PR- + # review jobs, zero processing, zero runnable, previously requiring manual intervention to unstick. + expr: gittensory_queue_live_pending > 0 and gittensory_queue_live_runnable_now == 0 + for: 3m + labels: + severity: critical + annotations: + summary: "gittensory foreground/live queue has pending work but nothing runnable" + description: "{{ $value | printf \"%.0f\" }} live-priority job(s) are pending but gittensory_queue_live_runnable_now has read 0 for over 3m -- contributor PR review work is not making progress." + runbook: "Open the Foreground Liveness row. Check gittensory_jobs_rate_limit_admission_deferred_total / gittensory_jobs_rate_limit_budget_deferred_total for a stuck GitHub rate-limit bucket, and gittensory_jobs_foreground_liveness_released_total for whether the liveness sweep is already recovering it (releases should show up within FOREGROUND_LIVENESS_CHECK_INTERVAL_MS, default 1m). If releases are firing repeatedly with no lasting effect, the underlying rate-limit exhaustion is the real bottleneck, not the queue." + + - alert: GittensoryForegroundLivenessReleasing + # Informational: the liveness sweep IS working (foreground work would otherwise be stuck), but a + # sustained rate means something keeps re-deferring the SAME class of work past the trickle ceiling -- + # worth investigating even though the invariant is holding. + expr: rate(gittensory_jobs_foreground_liveness_released_total[15m]) > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "gittensory foreground-liveness sweep is repeatedly force-releasing deferred work" + description: "The foreground-liveness sweep has released stale-deferred jobs for over 15m — the invariant is holding, but something is repeatedly pushing foreground work past FOREGROUND_LIVENESS_MAX_DEFER_MS." + runbook: "Check gittensory_jobs_rate_limit_admission_deferred_total / gittensory_jobs_rate_limit_budget_deferred_total by key_scope for a chronically exhausted GitHub REST budget (often installation-scoped after a burst)." + - alert: GittensoryMaintenanceStarved # Maintenance admission (maintenance-admission.ts) has TWO age escapes (#selfhost-maintenance-self-pin): # a short maintenanceDrainAgeMs trickle that lets old jobs through even while `maintenance_pending_high` diff --git a/src/github/backfill.ts b/src/github/backfill.ts index 9381290ea4..dc3c71cd14 100644 --- a/src/github/backfill.ts +++ b/src/github/backfill.ts @@ -2373,6 +2373,30 @@ export async function fetchRequiredStatusContexts( return names; } +/** + * Merge a maintainer-configured `expectedCiContexts` allowlist (`settings.expectedCiContexts` / + * `.gittensory.yml` `gate.expectedCiContexts`) with the live branch-protection required-status-check + * contexts from {@link fetchRequiredStatusContexts}. Branch protection stays authoritative when + * readable; `expectedCiContexts` is UNIONED into it when both exist, and becomes the SOLE required-context + * source when branch protection is null/empty (unreadable, or simply not configured) — the generic config + * path the #2137 `ciCompletenessWarning` has always nudged a maintainer toward. A repo with neither + * configured returns null, preserving today's fold-all fail-closed `reduceLiveCiAggregate` behavior + * unchanged. Entries are trimmed and blanks dropped defensively (the focus-manifest parser already + * normalizes `expectedCiContexts`, but this is the single point every caller funnels through). + */ +export function mergeRequiredCiContexts( + branchProtectionContexts: ReadonlySet | null, + expectedCiContexts: ReadonlyArray | null | undefined, +): Set | null { + const expected = (expectedCiContexts ?? []) + .map((context) => context.trim()) + .filter((context) => context.length > 0); + if (branchProtectionContexts && branchProtectionContexts.size > 0) { + return expected.length > 0 ? new Set([...branchProtectionContexts, ...expected]) : new Set(branchProtectionContexts); + } + return expected.length > 0 ? new Set(expected) : null; +} + // A GitHubApiError's own `.rateLimited` flag (set at construction from status/retry-after/remaining/body, see // GitHubApiError below) is ALREADY the correct rate-limit-vs-permission classification -- this just labels the // permission case with its own metric instead of the fetch's `.catch` silently discarding that information. diff --git a/src/github/rate-limit.ts b/src/github/rate-limit.ts index feee4b7c30..a74adb77f1 100644 --- a/src/github/rate-limit.ts +++ b/src/github/rate-limit.ts @@ -1,10 +1,15 @@ import { listLatestGitHubRateLimitObservations } from "../db/repositories"; // All managed repos share ONE GitHub App installation → ONE hourly REST bucket. To keep heavy maintenance work -// from draining the budget real webhook traffic needs, maintenance yields while there is still headroom: -// - backfill yields at LOW_REST_RATE_LIMIT_REMAINING; -// - the re-gate sweep + its per-PR jobs yield EARLIER, at MAINTENANCE_RESERVED_HEADROOM, reserving the budget -// between the two floors for webhooks; +// from draining the budget real-time contributor-PR-review traffic needs, maintenance yields while there is +// still headroom: +// - webhooks AND current-head agent-regate-pr reconciliation (a trailing coalesced re-review, an over-cap +// sibling wake, a linked-issue-change re-review, an outage-repair enqueue -- see +// isScheduledRegateSweepJob/#selfhost-queue-liveness) yield at LOW_REST_RATE_LIMIT_REMAINING: this is +// LIVE work someone is waiting on, not maintenance, regardless of which one of these triggered it; +// - the re-gate SWEEP's own stale-candidate fan-out (deliveryId prefixed "regate-sweep:") yields EARLIER, at +// MAINTENANCE_RESERVED_HEADROOM, reserving the budget between the two floors for the live work above — +// this is genuinely deferrable periodic maintenance, not a response to anything happening on a PR right now; // - historical/scheduled hydration that isn't needed for any CURRENT PR (e.g. backfilling file lists for old // merged pull requests) yields EARLIEST, at HISTORICAL_BACKFILL_RESERVED_HEADROOM — it is the least urgent // GitHub REST consumer, so it must never be the reason a live review or an open-PR convergence pass stalls. diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index 489ac261cb..0adb96b6a6 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -602,6 +602,7 @@ export const RepositorySettingsSchema = z claConsentPhrase: z.string().nullable().optional(), claCheckRunName: z.string().nullable().optional(), claCheckRunAppSlug: z.string().nullable().optional(), + expectedCiContexts: z.array(z.string()).optional(), gateDryRun: z.boolean().optional(), premergeContentRecheck: z.boolean().optional(), requireFreshRebaseWindowMinutes: z.number().int().positive().nullable().optional(), diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 10835a0a7b..e455bd8920 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -99,6 +99,7 @@ import { fetchRequiredStatusContexts, invalidatePrStateCache, isReviewsCacheUpToDate, + mergeRequiredCiContexts, primeDurablePrStateCache, refreshContributorActivity, refreshInstallationHealth, @@ -232,11 +233,13 @@ import { } from "../settings/agent-sweep"; import { selectBacklogConvergenceCandidates } from "../selfhost/backlog-convergence"; import { + LOW_REST_RATE_LIMIT_REMAINING, MAINTENANCE_RESERVED_HEADROOM, delayUntil, shouldWaitForGitHubRateLimit, } from "../github/rate-limit"; import { + isScheduledRegateSweepJob, queueSnapshotBacklog, queueSnapshotFromBinding, } from "../selfhost/queue-common"; @@ -516,21 +519,37 @@ function primeLiveMergeState( ); } +// Stable, order-independent cache-key fragment for settings.expectedCiContexts (#selfhost-ci-verification): +// a config change must never reuse a stale required-contexts/live-CI cache entry from before the change, and +// two equal sets in different orders must hit the SAME cache entry rather than needlessly duplicating fetches. +function expectedCiContextsKeyPart(expectedCiContexts: ReadonlyArray | null | undefined): string { + if (!expectedCiContexts || expectedCiContexts.length === 0) return ""; + return [...expectedCiContexts].sort().join(""); +} + +// RC2 + #selfhost-ci-verification: the EFFECTIVE required-status-check contexts for this repo/baseRef, merging +// live branch-protection required contexts with the maintainer-configured settings.expectedCiContexts fallback +// (mergeRequiredCiContexts — branch protection stays authoritative when readable; expectedCiContexts is the +// SOLE source when it is null/empty). Downstream callers (fetchLiveCiAggregate et al.) never distinguish the +// two sources — they only see one effective required-contexts set, same as before this field existed. function cachedRequiredStatusContexts( env: Env, repoFullName: string, facts: LiveGithubFacts, baseRef: string | null | undefined, token: string | undefined, + expectedCiContexts: ReadonlyArray | null | undefined, admissionKey?: GitHubRateLimitAdmissionKey, ): Promise | null> { - const key = liveFactKey(repoFullName, baseRef, liveFactTokenPart(token)); + const key = liveFactKey(repoFullName, baseRef, liveFactTokenPart(token), expectedCiContextsKeyPart(expectedCiContexts)); const cached = facts.requiredContexts.get(key); if (cached) return cached; const next = evictLiveFactOnReject( facts.requiredContexts, key, - fetchRequiredStatusContexts(env, repoFullName, baseRef, token, admissionKey), + fetchRequiredStatusContexts(env, repoFullName, baseRef, token, admissionKey).then((branchProtectionContexts) => + mergeRequiredCiContexts(branchProtectionContexts, expectedCiContexts), + ), ); facts.requiredContexts.set(key, next); return next; @@ -554,12 +573,13 @@ function fetchLiveCiAggregateWithRequiredContexts( headSha: string | null | undefined, baseRef: string | null | undefined, token: string | undefined, + expectedCiContexts: ReadonlyArray | null | undefined, admissionKey?: GitHubRateLimitAdmissionKey, ): Promise { // CI refresh callers need fresh check/status state; branch protection contexts move slowly enough to stay // request-cached. When the #1941 flag is on, fetchLiveCiAggregatePreferGraphQl collapses the check/status reads // into one GraphQL rollup (reusing these requiredContexts), else it uses the proven REST aggregate. - return cachedRequiredStatusContexts(env, repoFullName, facts, baseRef, token, admissionKey) + return cachedRequiredStatusContexts(env, repoFullName, facts, baseRef, token, expectedCiContexts, admissionKey) .catch(() => null) .then((requiredContexts) => fetchLiveCiAggregatePreferGraphQl(env, repoFullName, headSha, token, requiredContexts, admissionKey), @@ -573,9 +593,10 @@ function cachedLiveCiAggregate( headSha: string | null | undefined, baseRef: string | null | undefined, token: string | undefined, + expectedCiContexts: ReadonlyArray | null | undefined, admissionKey?: GitHubRateLimitAdmissionKey, ): Promise { - const key = liveFactKey(repoFullName, headSha, baseRef, liveFactTokenPart(token)); + const key = liveFactKey(repoFullName, headSha, baseRef, liveFactTokenPart(token), expectedCiContextsKeyPart(expectedCiContexts)); const cached = facts.ciAggregates.get(key); if (cached) return cached; const next = evictLiveFactOnReject( @@ -588,6 +609,7 @@ function cachedLiveCiAggregate( headSha, baseRef, token, + expectedCiContexts, admissionKey, ), ); @@ -602,9 +624,10 @@ function refreshLiveCiAggregate( headSha: string | null | undefined, baseRef: string | null | undefined, token: string | undefined, + expectedCiContexts: ReadonlyArray | null | undefined, admissionKey?: GitHubRateLimitAdmissionKey, ): Promise { - const key = liveFactKey(repoFullName, headSha, baseRef, liveFactTokenPart(token)); + const key = liveFactKey(repoFullName, headSha, baseRef, liveFactTokenPart(token), expectedCiContextsKeyPart(expectedCiContexts)); const next = evictLiveFactOnReject( facts.ciAggregates, key, @@ -615,6 +638,7 @@ function refreshLiveCiAggregate( headSha, baseRef, token, + expectedCiContexts, admissionKey, ), ); @@ -1477,6 +1501,13 @@ async function sweepRepoRegate( const flaggedPulls: number[] = []; const sweepInstallationId = repo?.installationId ?? null; const duplicateWinnerEnabled = env.GITTENSORY_DUPLICATE_WINNER === "true"; + // #selfhost-queue-liveness: priorityPullNumbers (surfaceRepairPriorityPullNumbers, above) are OUTAGE REPAIR -- + // a PR with no current-head Gate check or an unpublished current-head surface -- not routine staleness. A + // repair candidate's fanned-out job must NOT carry the "regate-sweep:" deliveryId prefix, or + // isScheduledRegateSweepJob (queue-common.ts) misclassifies it as background maintenance and it inherits the + // exact starvation this priority mechanism exists to avoid. Ordinary stale candidates keep the sweep prefix + // unchanged. + const priorityPullNumberSet = new Set(priorityPullNumbers); for (const [index, pr] of candidates.entries()) { const others = openPullRequests.filter( (other) => other.number !== pr.number, @@ -1512,7 +1543,9 @@ async function sweepRepoRegate( if (sweepInstallationId != null) { const job: JobMessage = { type: "agent-regate-pr", - deliveryId: `regate-sweep:${repoFullName}#${pr.number}`, + deliveryId: priorityPullNumberSet.has(pr.number) + ? `regate-repair:${repoFullName}#${pr.number}` + : `regate-sweep:${repoFullName}#${pr.number}`, repoFullName, prNumber: pr.number, installationId: sweepInstallationId, @@ -1673,13 +1706,18 @@ async function regatePullRequest( deliveryId: string, force?: boolean, ): Promise { - // Reserve installation rate-limit headroom for real webhooks (#audit-rate-headroom): all repos share ONE GitHub - // App installation = ONE REST bucket, so when the shared budget is at/below the maintenance floor, DEFER this - // re-review until the reset instead of burning budget a webhook's re-review needs. Re-enqueue with the reset - // delay so the PR is still eventually re-reviewed. + // Reserve installation rate-limit headroom (#audit-rate-headroom): all repos share ONE GitHub App installation + // = ONE REST bucket, so when the shared budget is low, DEFER this re-review until the reset instead of + // burning budget other work needs. #selfhost-queue-liveness: the FLOOR depends on WHY this job exists — the + // scheduled sweep's own stale-PR fan-out (isScheduledRegateSweepJob) can wait behind the conservative + // maintenance floor same as any other periodic sweep, but every other trigger (a real webhook event: a + // trailing coalesced re-review, an over-cap sibling wake, a linked-issue-change re-review, a reconciliation- + // repair enqueue) is current-HEAD contributor-PR-review work and gets the SAME low floor a fresh webhook + // gets — it must never be treated as background maintenance and parked behind it. Mirrors the SAME + // reclassification githubRateLimitAdmissionTargetForJob applies at the queue-admission layer. const rateResetAt = await shouldWaitForGitHubRateLimit( env, - MAINTENANCE_RESERVED_HEADROOM, + isScheduledRegateSweepJob(deliveryId) ? MAINTENANCE_RESERVED_HEADROOM : LOW_REST_RATE_LIMIT_REMAINING, ); if (rateResetAt) { await env.JOBS.send( @@ -2001,6 +2039,7 @@ async function runAgentMaintenancePlanAndExecute( args.liveFacts, baseRef, token, + settings.expectedCiContexts, admissionKey, ), // Live mergeable_state after the gate's own publish/review/check mutations. Readiness may have seen the PR as @@ -2019,6 +2058,7 @@ async function runAgentMaintenancePlanAndExecute( pr.headSha, baseRef, token, + settings.expectedCiContexts, admissionKey, ); // #2137: informational-only nudge for the operator — never affects the disposition below (ciState is @@ -2695,7 +2735,7 @@ async function prReadyForReview( } // 2) wait for CI to finish before running the Gittensory review. Required contexts still define which failures // block/close, but hasPending tracks any visible non-bot CI that is not settled yet. - const ci = await cachedLiveCiAggregate(env, repoFullName, liveFacts, pr.headSha, pr.baseRef, token, admissionKey).catch(() => undefined); + const ci = await cachedLiveCiAggregate(env, repoFullName, liveFacts, pr.headSha, pr.baseRef, token, settings.expectedCiContexts, admissionKey).catch(() => undefined); if (ci?.hasPending) { // Staleness cap: inferred or unreadable pending CI can otherwise defer FOREVER (orphaned required context, // transiently unreadable pages, fork check that never reports). Past STUCK_CI_DEFER_MS we stop deferring and @@ -7907,7 +7947,7 @@ async function maybePublishPrPublicSurface( const baseRef = pr.baseRef ?? repo?.defaultBranch; // Required contexts still detect missing/pending required CI, but every visible completed red check/status is // adverse and blocks the PR. - const liveCi = await refreshLiveCiAggregate(env, repoFullName, webhook.liveFacts, pr.headSha, baseRef, token, admissionKey); + const liveCi = await refreshLiveCiAggregate(env, repoFullName, webhook.liveFacts, pr.headSha, baseRef, token, settings.expectedCiContexts, admissionKey); // Live merge-state too — the SAME source the disposition uses (planAgentMaintenanceActions reads liveMergeState). // The stored pr.mergeableState lags GitHub's async recompute, and the gate's own check/review publication can // also advance mergeability after readiness ran, so refresh at this post-publish boundary. diff --git a/src/selfhost/foreground-liveness.ts b/src/selfhost/foreground-liveness.ts new file mode 100644 index 0000000000..320a5f43e2 --- /dev/null +++ b/src/selfhost/foreground-liveness.ts @@ -0,0 +1,58 @@ +// Foreground-liveness invariant (#selfhost-queue-liveness): live contributor-PR-review work (github-webhook, +// agent-regate-pr, agent-regate-sweep, recapture-preview -- everything at or above FOREGROUND_QUEUE_PRIORITY_FLOOR, +// see queue-common.ts) must always have a BOUNDED runnable trickle, mirroring the maintenance lane's own +// maxDeferAgeMs escape hatch (maintenance-admission.ts). Unlike maintenance jobs, foreground jobs never go through +// an admission gate of their own -- only the GitHub rate-limit admission check (processOne, before consume()) and +// the rate-limit BUDGET sweep (deferPendingJobsForRateLimit) can push a foreground job's run_after into the +// future, and NEITHER exempts foreground priority the way maintenance-admission exempts it entirely: a +// GITHUB_BUDGET_BACKGROUND_TYPES job like agent-regate-pr (a literal "contributor PR review", priority 9, +// foreground) is rate-limited with the SAME conservative headroom as genuine maintenance sweeps +// (MAINTENANCE_RESERVED_HEADROOM, see queue-common.ts's githubRateLimitAdmissionTargetForJob), so a shared REST +// budget drained by a post-deploy catch-up burst can defer it for the full rate-limit reset window (up to +// MAX_GITHUB_RATE_LIMIT_RETRY_MS = 65 minutes) with no floor. Without this module, that lane can silently starve +// entirely: hundreds of pending contributor-PR-review jobs, zero processing, zero runnable, requiring manual +// intervention -- the production incident this module exists to make structurally impossible. +// +// The queue backends (pg-queue.ts / sqlite-queue.ts) run releaseStaleForegroundDeferrals() periodically (see +// start()) AND once at boot (init()), so a restart/deploy self-heals inherited over-deferral instead of needing +// manual unsticking. A dedicated slow interval (not the 1s poll tick) bounds retry cost: a job still genuinely +// rate-limited after being released just re-defers and waits for the NEXT sweep, never a busy-loop on every tick. +import { parsePositiveIntEnv } from "./queue-common"; + +const DEFAULT_MAX_DEFER_MS = 10 * 60_000; // 10 minutes -- long enough to not fight a normal rate-limit backoff +// (which typically resolves within DEFAULT_GITHUB_RATE_LIMIT_RETRY_MS + jitter, see queue-common.ts), short +// enough that live contributor-PR-review work is never parked anywhere near the ~65-minute worst case. +const DEFAULT_CHECK_INTERVAL_MS = 60_000; // 1 minute + +export interface ForegroundLivenessConfig { + enabled: boolean; + maxDeferMs: number; + checkIntervalMs: number; +} + +function foregroundLivenessEnabled(): boolean { + const raw = (process.env.FOREGROUND_LIVENESS_ENABLED ?? "").trim().toLowerCase(); + return raw !== "0" && raw !== "false" && raw !== "off" && raw !== "no"; +} + +/** Reads every FOREGROUND_LIVENESS_* knob from process.env, each with a sane, protective default. Resolved ONCE + * per queue instance (mirrors resolveMaintenanceAdmissionConfig / queueBackgroundConcurrency) rather than per + * sweep, so a misconfigured value only warns once at startup instead of on every tick. */ +export function resolveForegroundLivenessConfig(): ForegroundLivenessConfig { + return { + enabled: foregroundLivenessEnabled(), + maxDeferMs: parsePositiveIntEnv("FOREGROUND_LIVENESS_MAX_DEFER_MS", { min: 60_000, fallback: DEFAULT_MAX_DEFER_MS }), + checkIntervalMs: parsePositiveIntEnv("FOREGROUND_LIVENESS_CHECK_INTERVAL_MS", { min: 5_000, fallback: DEFAULT_CHECK_INTERVAL_MS }), + }; +} + +/** PURE decision: is a pending foreground job's deferral stale enough to force-release regardless of its current + * run_after? Mirrors evaluateMaintenanceAdmission's own trickle_max_defer_age condition, but keyed on + * `pendingSinceMs` (the row's created_at -- never reset across a coalesced re-enqueue or an admission-style + * re-defer, see maintenance-admission.ts's own doc comment on the same anchor) rather than run_after, so a job + * repeatedly re-deferred to a fresh future timestamp still gets released once its GENUINE wait time crosses the + * ceiling. `enabled: false` never releases (the operator-disable escape hatch, mirroring + * MAINTENANCE_ADMISSION_ENABLED=false). */ +export function isForegroundDeferralStale(config: ForegroundLivenessConfig, pendingSinceMs: number, nowMs: number): boolean { + return config.enabled && nowMs - pendingSinceMs >= config.maxDeferMs; +} diff --git a/src/selfhost/maintenance-admission.ts b/src/selfhost/maintenance-admission.ts index a68b74bd1f..03336fca0e 100644 --- a/src/selfhost/maintenance-admission.ts +++ b/src/selfhost/maintenance-admission.ts @@ -71,6 +71,17 @@ export function isMaintenanceJobType(type: string): boolean { export interface MaintenancePressureSignals { livePendingCount: number; oldestLivePendingAgeMs: number | null; + /** Foreground-priority pending jobs that are RUNNABLE right now (run_after<=now), i.e. not currently + * deferred by any mechanism -- distinct from livePendingCount, which also includes deferred/processing + * work. #selfhost-queue-liveness's own diagnostic: "queue large but intentionally deferred" (this count can + * be 0 with livePendingCount > 0, transiently, and that is fine) vs. "queue stuck" (this count stays 0 + * while oldestLiveRunnableAgeMs -- once something IS runnable -- climbs, or while releaseStaleForegroundDeferrals + * keeps finding stale work every sweep). */ + liveRunnableNowCount: number; + /** Age in ms of the oldest RUNNABLE (run_after<=now) foreground pending job -- null when none is runnable + * right now. Distinct from oldestLivePendingAgeMs, which is dominated by a job intentionally scheduled far + * in the future and says nothing about how long already-due work has sat unclaimed. */ + oldestLiveRunnableAgeMs: number | null; maintenancePendingCount: number; oldestMaintenancePendingAgeMs: number | null; /** Null when unavailable (see host-pressure.ts) -- a caller must treat null as "skip this check". */ diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts index 4fcfdb1245..52572329e8 100644 --- a/src/selfhost/metrics.ts +++ b/src/selfhost/metrics.ts @@ -26,9 +26,13 @@ const histograms = new Map(); const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ ["gittensory_queue_pending", { help: "Current in-process queue depth.", type: "gauge" }], ["gittensory_queue_dead", { help: "Current in-process dead queue depth.", type: "gauge" }], + ["gittensory_queue_processing", { help: "Jobs currently claimed and mid-flight.", type: "gauge" }], + ["gittensory_queue_runnable_now", { help: "Pending jobs, any priority, currently due (run_after<=now).", type: "gauge" }], ["gittensory_queue_live_pending", { help: "Current live-work queue depth.", type: "gauge" }], + ["gittensory_queue_live_runnable_now", { help: "Live (foreground) pending jobs currently due (run_after<=now).", type: "gauge" }], ["gittensory_queue_maintenance_pending", { help: "Current maintenance-work queue depth.", type: "gauge" }], ["gittensory_queue_oldest_live_pending_age_seconds", { help: "Age in seconds of the oldest live pending job.", type: "gauge" }], + ["gittensory_queue_oldest_live_runnable_age_seconds", { help: "Age in seconds of the oldest live pending job that is currently due.", type: "gauge" }], ["gittensory_queue_oldest_maintenance_pending_age_seconds", { help: "Age in seconds of the oldest maintenance pending job.", type: "gauge" }], ["gittensory_host_load_avg1_per_core", { help: "One-minute host load average normalized by CPU core count.", type: "gauge" }], ["gittensory_uptime_seconds", { help: "Self-host process uptime in seconds.", type: "gauge" }], @@ -59,6 +63,8 @@ const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ ["gittensory_jobs_rate_limited_by_type_total", { help: "Jobs rate-limited by job type.", type: "counter" }], ["gittensory_jobs_maintenance_admission_deferred_by_reason_total", { help: "Maintenance jobs deferred by reason.", type: "counter" }], ["gittensory_jobs_dead_letter_revived_total", { help: "Dead-letter jobs revived for retry.", type: "counter" }], + ["gittensory_jobs_foreground_liveness_released_total", { help: "Foreground-priority jobs force-released from a stale deferral by the liveness sweep.", type: "counter" }], + ["gittensory_jobs_foreground_liveness_released_by_reason_total", { help: "Foreground liveness releases by reason (age vs rate_limit_cleared).", type: "counter" }], ["gittensory_dlq_dead_lettered_total", { help: "Messages moved to a dead-letter queue.", type: "counter" }], ["gittensory_dlq_redriven_total", { help: "Dead-letter queue messages redriven into processing.", type: "counter" }], ["gittensory_github_response_cache_total", { help: "GitHub response cache outcomes by response class.", type: "counter" }], diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index 450131f76b..7ca1d15e75 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -140,6 +140,11 @@ import { pickBacklogRepo, type ForegroundLane, } from "./queue-fairness"; +import { + isForegroundDeferralStale, + resolveForegroundLivenessConfig, + type ForegroundLivenessConfig, +} from "./foreground-liveness"; import type { JobMessage } from "../types"; const TABLE = "_selfhost_jobs"; @@ -187,6 +192,9 @@ export interface PgDurableQueue { drain(): Promise; size(): Promise; deadCount(): Promise; + /** Jobs currently claimed and mid-flight (status='processing') -- distinct from size(), which also + * includes still-pending work. See #selfhost-queue-liveness's own observability additions. */ + processingCount(): Promise; stats(): Promise>; snapshot(): Promise; /** Live-vs-maintenance queue pressure, for the /metrics gauges (see server.ts) -- the SAME signals the @@ -196,6 +204,11 @@ export interface PgDurableQueue { * running (see start()), and exposed directly so tests and an operator-triggered repair path don't have * to wait for the real interval. Returns the number of jobs revived. */ reviveDeadLetterJobs(): Promise; + /** Foreground-liveness invariant (#selfhost-queue-liveness): pulls back any FOREGROUND-priority pending job + * whose deferral has gone stale (see foreground-liveness.ts) regardless of what deferred it. Called once at + * boot and on a timer while running (see init()/start()), and exposed directly so tests and an + * operator-triggered repair path don't have to wait for the real interval. Returns the number released. */ + releaseStaleForegroundDeferrals(): Promise; } interface JobRow { @@ -245,7 +258,9 @@ export function createPgQueue( const activeJobIds = new Set(); let timer: ReturnType | null = null; let deadLetterReviveTimer: ReturnType | null = null; + let foregroundLivenessTimer: ReturnType | null = null; const maintenanceAdmissionConfig: MaintenanceAdmissionConfig = resolveMaintenanceAdmissionConfig(); + const foregroundLivenessConfig: ForegroundLivenessConfig = resolveForegroundLivenessConfig(); async function init(): Promise { await pool.query(DDL); @@ -300,6 +315,10 @@ export function createPgQueue( jitter_ms: queueStartupJitterMs(), }), ); + // Self-heal on boot (#selfhost-queue-liveness): a deploy/restart inherits whatever run_after values were + // already written before it, so a foreground lane over-deferred before the restart must not require manual + // intervention to unstick -- releaseStaleForegroundDeferrals logs + records its own metric when it finds work. + await releaseStaleForegroundDeferrals(); } async function backfillJobPriorities(): Promise { @@ -366,22 +385,35 @@ export function createPgQueue( } /** Cheap aggregate reads behind the maintenance-admission policy (and the observability gauges in - * server.ts): how much LIVE (foreground) work is queued and how old the oldest of it is, and the same for - * the MAINTENANCE lane specifically (not "all background" -- targeted jobs like backfill-repo-segment - * don't count, see maintenance-admission.ts). Host load is an independent, optional signal. */ + * server.ts): how much LIVE (foreground) work is queued and how old the oldest of it is -- both overall + * (pending+processing) and RUNNABLE right now (pending, due) -- and the same PENDING/oldest pair for the + * MAINTENANCE lane specifically (not "all background" -- targeted jobs like backfill-repo-segment don't + * count, see maintenance-admission.ts). The runnable-now split is the #selfhost-queue-liveness diagnostic: + * distinguishes "queue large but intentionally deferred" from "queue stuck, nothing runnable" without + * manual SQL. Host load is an independent, optional signal. */ async function maintenancePressureSignals(now: number): Promise { const liveRes = await pool.query( - `SELECT COUNT(*) AS cnt, MIN(created_at) AS oldest FROM ${TABLE} WHERE status IN ('pending','processing') AND priority>=$1`, - [FOREGROUND_QUEUE_PRIORITY_FLOOR], + `SELECT COUNT(*) AS cnt, MIN(created_at) AS oldest, + COUNT(*) FILTER (WHERE status='pending' AND run_after<=$2) AS runnable_cnt, + MIN(created_at) FILTER (WHERE status='pending' AND run_after<=$2) AS oldest_runnable + FROM ${TABLE} WHERE status IN ('pending','processing') AND priority>=$1`, + [FOREGROUND_QUEUE_PRIORITY_FLOOR, now], ); const maintenanceRes = await pool.query( `SELECT COUNT(*) AS cnt, MIN(created_at) AS oldest FROM ${TABLE} WHERE status IN ('pending','processing') AND is_maintenance=1`, ); - const live = liveRes.rows[0] as { cnt: string | number; oldest: string | number | null }; + const live = liveRes.rows[0] as { + cnt: string | number; + oldest: string | number | null; + runnable_cnt: string | number; + oldest_runnable: string | number | null; + }; const maintenance = maintenanceRes.rows[0] as { cnt: string | number; oldest: string | number | null }; return { livePendingCount: Number(live.cnt), oldestLivePendingAgeMs: live.oldest != null ? now - Number(live.oldest) : null, + liveRunnableNowCount: Number(live.runnable_cnt), + oldestLiveRunnableAgeMs: live.oldest_runnable != null ? now - Number(live.oldest_runnable) : null, maintenancePendingCount: Number(maintenance.cnt), oldestMaintenancePendingAgeMs: maintenance.oldest != null ? now - Number(maintenance.oldest) : null, hostLoadAvg1PerCore: hostLoadAvg1PerCore(), @@ -469,6 +501,91 @@ export function createPgQueue( } } + /** #selfhost-queue-liveness: re-evaluate rate-limit admission for an already-deferred foreground candidate + * against CURRENT observations, independent of how long ago it was deferred. Returns true when it would be + * admitted right now (no longer blocked); false when still blocked OR the payload is unparseable (best- + * effort -- an unparseable payload is left for the normal dead-letter path, never force-released here). */ + async function isRateLimitAdmissionNowClear(payload: string): Promise { + let message: JobMessage; + try { + message = JSON.parse(payload) as JobMessage; + } catch { + return false; + } + return (await rateLimitAdmissionDelayMs(message)) === null; + } + + /** See foreground-liveness.ts for the full rationale. A bounded candidate SELECT (foreground-priority, pending, + * not currently due) then a per-row conditional UPDATE, mirroring reviveEligibleDeadJobs' shape. Each + * candidate is released on EITHER of two independent conditions: it has genuinely been waiting past the + * age-based trickle ceiling (isForegroundDeferralStale, unconditional backstop), OR -- CONDITION-BASED + * recovery (#selfhost-queue-liveness VPS incident) -- re-evaluating rateLimitAdmissionDelayMs against + * CURRENT observations right now says it would be admitted immediately. The age floor alone can leave a job + * pinned to a stale reset timestamp for up to its full original delay (observed up to ~15m) even when a + * fresher, healthier observation arrived moments after it was deferred; the condition check recovers it on + * the NEXT sweep tick instead (bounded by FOREGROUND_LIVENESS_CHECK_INTERVAL_MS, default 60s) whenever the + * underlying rate-limit pressure has actually cleared, regardless of job age. Logs + records a metric ONCE + * per sweep (aggregate count), not per row, so a large release batch cannot spam the log. */ + async function releaseStaleForegroundDeferrals(): Promise { + if (!foregroundLivenessConfig.enabled) return 0; + const now = Date.now(); + const res = await pool.query( + `SELECT id, payload, created_at FROM ${TABLE} WHERE status='pending' AND priority>=$1 AND run_after>$2`, + [FOREGROUND_QUEUE_PRIORITY_FLOOR, now], + ); + let released = 0; + let releasedByAge = 0; + let releasedByRateLimitClear = 0; + for (const row of res.rows as Array<{ id: string; payload: string; created_at: number | string }>) { + const ageStale = isForegroundDeferralStale(foregroundLivenessConfig, Number(row.created_at), now); + if (!ageStale && !(await isRateLimitAdmissionNowClear(row.payload))) continue; + const update = await pool.query( + `UPDATE ${TABLE} SET run_after=$1 WHERE id=$2 AND status='pending' AND run_after>$1`, + [now, row.id], + ); + const rowsChanged = update.rowCount ?? 0; + released += rowsChanged; + if (ageStale) releasedByAge += rowsChanged; + else releasedByRateLimitClear += rowsChanged; + } + if (released) { + await recordQueueMetric("gittensory_jobs_foreground_liveness_released_total", released); + if (releasedByAge) incr("gittensory_jobs_foreground_liveness_released_by_reason_total", { reason: "age" }, releasedByAge); + if (releasedByRateLimitClear) incr("gittensory_jobs_foreground_liveness_released_by_reason_total", { reason: "rate_limit_cleared" }, releasedByRateLimitClear); + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_foreground_liveness_released", + count: released, + released_by_age: releasedByAge, + released_by_rate_limit_cleared: releasedByRateLimitClear, + max_defer_ms: foregroundLivenessConfig.maxDeferMs, + }), + ); + kickAll(); + } + return released; + } + + /** Wraps releaseStaleForegroundDeferrals() for the setInterval callback below, mirroring + * reviveDeadLetterJobsSafely's own rationale: an uncaught rejection here would surface as an unhandled + * promise rejection and can terminate the process when SENTRY_DSN is unset. A failed sweep just waits for + * the next interval, same as a failed poll tick waits for the next poll. */ + async function releaseStaleForegroundDeferralsSafely(): Promise { + try { + await releaseStaleForegroundDeferrals(); + } catch (error) { + console.error( + JSON.stringify({ + level: "error", + event: "selfhost_queue_foreground_liveness_release_crashed", + error: errorMessageWithCause(error), + }), + ); + captureError(error, { kind: "queue_foreground_liveness_release_crashed" }); + } + } + async function spreadDueJobsOnStartup(): Promise { const now = Date.now(); const res = await pool.query( @@ -1101,11 +1218,18 @@ export function createPgQueue( // recreate the retry storm this feature exists to bound. The interval itself is the cooldown between // auto-retry rounds for any one job. deadLetterReviveTimer = setInterval(() => void reviveDeadLetterJobsSafely(), queueDeadLetterReviveIntervalMs()); + // Foreground-liveness sweep (#selfhost-queue-liveness): also a separate, slow interval -- see + // foreground-liveness.ts for why a per-tick check would busy-loop under sustained rate-limit pressure. + foregroundLivenessTimer = setInterval( + () => void releaseStaleForegroundDeferralsSafely(), + foregroundLivenessConfig.checkIntervalMs, + ); }, async stop() { running = false; if (timer) clearTimeout(timer); if (deadLetterReviveTimer) clearInterval(deadLetterReviveTimer); + if (foregroundLivenessTimer) clearInterval(foregroundLivenessTimer); while (active > 0) await new Promise((r) => setTimeout(r, 10)); }, async drain() { @@ -1130,11 +1254,21 @@ export function createPgQueue( ).rows[0].c, ); }, + async processingCount() { + return Number( + ( + await pool.query( + `SELECT COUNT(*) AS c FROM ${TABLE} WHERE status='processing'`, + ) + ).rows[0].c, + ); + }, async stats() { return readQueueStats(); }, snapshot: binding.snapshot, reviveDeadLetterJobs, + releaseStaleForegroundDeferrals, pressureSignals() { return maintenancePressureSignals(Date.now()); }, diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index ffbb3d50f9..b4fbac68cd 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -123,6 +123,18 @@ export function isGitHubBudgetBackgroundJob(message: JobMessage): boolean { return GITHUB_BUDGET_BACKGROUND_TYPES.has(message.type); } +// The scheduled sweep's own per-PR fan-out (sweepRepoRegate, #audit-sweep-fanout) tags its synthetic delivery +// id with this prefix -- the ONLY agent-regate-pr trigger that is genuinely stale/scheduled maintenance, not a +// response to something happening on the PR right now. EVERY other agent-regate-pr producer (a trailing +// coalesced re-review, an over-cap sibling wake, a linked-issue-change re-review, a reconciliation-repair +// enqueue) carries the REAL webhook/event delivery id that caused it -- current-HEAD contributor-PR-review +// work, not background maintenance (#selfhost-queue-liveness, VPS incident: agent-regate-pr jobs were treated +// as background admission and parked behind a conservative maintenance floor even though they were reconciling +// a live contributor PR someone was waiting on). +export function isScheduledRegateSweepJob(deliveryId: string | null | undefined): boolean { + return typeof deliveryId === "string" && deliveryId.startsWith("regate-sweep:"); +} + export function buildSelfHostQueueSnapshot( rows: Iterable<{ payload?: unknown; status?: unknown; run_after?: unknown; runAfter?: unknown }>, nowMs = Date.now(), @@ -378,6 +390,18 @@ export function githubRateLimitAdmissionTargetForJob( admissionKey: githubRateLimitAdmissionKeyForJob(message), }; } + // Current-head contributor-PR-review reconciliation (#selfhost-queue-liveness): every agent-regate-pr EXCEPT + // the scheduled sweep's own fan-out (isGitHubBudgetBackgroundJob already fully exempts the manual-regate + // operator override above that check) is a response to something happening on the PR right now, so it gets + // the SAME floor as a fresh webhook -- never the conservative maintenance floor a stale/scheduled sweep + // reserves. Checked BEFORE isGitHubBudgetBackgroundJob (which would otherwise classify it "background") so + // this branch wins for every non-sweep, non-manual agent-regate-pr job. + if (message.type === "agent-regate-pr" && isGitHubBudgetBackgroundJob(message) && !isScheduledRegateSweepJob(message.deliveryId)) { + return { + kind: "webhook", + admissionKey: githubRateLimitAdmissionKeyForJob(message) ?? githubRateLimitAdmissionKeyForPublicToken(), + }; + } if (!isGitHubBudgetBackgroundJob(message)) return null; const admissionKey = githubRateLimitAdmissionKeyForJob(message); return { diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index e14409b655..190be38f7e 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -56,6 +56,11 @@ import { pickBacklogRepo, type ForegroundLane, } from "./queue-fairness"; +import { + isForegroundDeferralStale, + resolveForegroundLivenessConfig, + type ForegroundLivenessConfig, +} from "./foreground-liveness"; import type { JobMessage } from "../types"; const TABLE = "_selfhost_jobs"; @@ -104,6 +109,9 @@ export interface DurableQueue { drain(): Promise; size(): number; deadCount(): number; + /** Jobs currently claimed and mid-flight (status='processing') -- distinct from size(), which also + * includes still-pending work. See #selfhost-queue-liveness's own observability additions. */ + processingCount(): number; stats(): Record; snapshot(): SelfHostQueueSnapshot; /** Live-vs-maintenance queue pressure, for the /metrics gauges (see server.ts) -- the SAME signals the @@ -113,6 +121,11 @@ export interface DurableQueue { * running (see start()), and exposed directly so tests and an operator-triggered repair path don't have * to wait for the real interval. Returns the number of jobs revived. */ reviveDeadLetterJobs(): number; + /** Foreground-liveness invariant (#selfhost-queue-liveness): pulls back any FOREGROUND-priority pending job + * whose deferral has gone stale (see foreground-liveness.ts) regardless of what deferred it. Called once at + * boot and on a timer while running (see the module-init block/start()), and exposed directly so tests and + * an operator-triggered repair path don't have to wait for the real interval. Returns the number released. */ + releaseStaleForegroundDeferrals(): number; } interface JobRow { @@ -220,6 +233,7 @@ export function createSqliteQueue( }), ); const maintenanceAdmissionConfig: MaintenanceAdmissionConfig = resolveMaintenanceAdmissionConfig(); + const foregroundLivenessConfig: ForegroundLivenessConfig = resolveForegroundLivenessConfig(); // Recover jobs a crashed previous run left mid-flight → make them claimable again. const recovered = recoverProcessingJobs(driver); if (recovered) { @@ -237,13 +251,21 @@ export function createSqliteQueue( jitter_ms: queueStartupJitterMs(), }), ); - let running = false; let active = 0; // number of concurrent pump() loops currently draining jobs let activeBackground = 0; const activeJobIds = new Set(); let timer: ReturnType | null = null; let deadLetterReviveTimer: ReturnType | null = null; + let foregroundLivenessTimer: ReturnType | null = null; + + // Self-heal on boot (#selfhost-queue-liveness): a deploy/restart inherits whatever run_after values were + // already written before it, so a foreground lane over-deferred before the restart must not require manual + // intervention to unstick. releaseStaleForegroundDeferrals is declared below (function-hoisted, see + // foreground-liveness.ts) and logs + records its own metric when it finds work. MUST run after `active`/ + // `activeBackground` above are initialized -- a release calls kickAll(), which reads them, and both are + // still in the temporal dead zone before this point (#selfhost-queue-liveness-tdz). + releaseStaleForegroundDeferrals(); function reviveDeadLetterJobs(): number { const revived = reviveEligibleDeadJobs(driver, maxRetries); @@ -275,6 +297,90 @@ export function createSqliteQueue( } } + /** #selfhost-queue-liveness: re-evaluate rate-limit admission for an already-deferred foreground candidate + * against CURRENT observations, independent of how long ago it was deferred. Returns true when it would be + * admitted right now (no longer blocked); false when still blocked OR the payload is unparseable (best- + * effort -- an unparseable payload is left for the normal dead-letter path, never force-released here). */ + function isRateLimitAdmissionNowClear(payload: string): boolean { + let message: JobMessage; + try { + message = JSON.parse(payload) as JobMessage; + } catch { + return false; + } + return rateLimitAdmissionDelayMs(driver, message) === null; + } + + /** See foreground-liveness.ts for the full rationale. A bounded candidate SELECT (foreground-priority, pending, + * not currently due) then a per-row conditional UPDATE, mirroring reviveEligibleDeadJobs' shape. Each + * candidate is released on EITHER of two independent conditions: it has genuinely been waiting past the + * age-based trickle ceiling (isForegroundDeferralStale, unconditional backstop), OR -- CONDITION-BASED + * recovery (#selfhost-queue-liveness VPS incident) -- re-evaluating rate-limit admission against CURRENT + * observations right now says it would be admitted immediately. The age floor alone can leave a job pinned + * to a stale reset timestamp for up to its full original delay (observed up to ~15m) even when a fresher, + * healthier observation arrived moments after it was deferred; the condition check recovers it on the NEXT + * sweep tick instead (bounded by FOREGROUND_LIVENESS_CHECK_INTERVAL_MS, default 60s) whenever the underlying + * rate-limit pressure has actually cleared, regardless of job age. Logs + records a metric ONCE per sweep + * (aggregate count), not per row, so a large release batch cannot spam the log. */ + function releaseStaleForegroundDeferrals(): number { + if (!foregroundLivenessConfig.enabled) return 0; + const now = Date.now(); + const { rows } = driver.query( + `SELECT id, payload, created_at FROM ${TABLE} WHERE status='pending' AND priority>=? AND run_after>?`, + [FOREGROUND_QUEUE_PRIORITY_FLOOR, now], + ); + let released = 0; + let releasedByAge = 0; + let releasedByRateLimitClear = 0; + for (const row of rows as Array<{ id: number; payload: string; created_at: number }>) { + const ageStale = isForegroundDeferralStale(foregroundLivenessConfig, row.created_at, now); + if (!ageStale && !isRateLimitAdmissionNowClear(row.payload)) continue; + const { changes } = driver.query( + `UPDATE ${TABLE} SET run_after=? WHERE id=? AND status='pending' AND run_after>?`, + [now, row.id, now], + ); + released += changes; + if (ageStale) releasedByAge += changes; + else releasedByRateLimitClear += changes; + } + if (released) { + recordQueueMetric(driver, "gittensory_jobs_foreground_liveness_released_total", released); + if (releasedByAge) incr("gittensory_jobs_foreground_liveness_released_by_reason_total", { reason: "age" }, releasedByAge); + if (releasedByRateLimitClear) incr("gittensory_jobs_foreground_liveness_released_by_reason_total", { reason: "rate_limit_cleared" }, releasedByRateLimitClear); + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_foreground_liveness_released", + count: released, + released_by_age: releasedByAge, + released_by_rate_limit_cleared: releasedByRateLimitClear, + max_defer_ms: foregroundLivenessConfig.maxDeferMs, + }), + ); + kickAll(); + } + return released; + } + + /** Wraps releaseStaleForegroundDeferrals() for the setInterval callback below, mirroring + * reviveDeadLetterJobsSafely's own rationale: an uncaught exception here would surface as an unhandled + * exception and can terminate the process when SENTRY_DSN is unset. A failed sweep just waits for the next + * interval, same as a failed poll tick waits for the next poll. */ + function releaseStaleForegroundDeferralsSafely(): void { + try { + releaseStaleForegroundDeferrals(); + } catch (error) { + console.error( + JSON.stringify({ + level: "error", + event: "selfhost_queue_foreground_liveness_release_crashed", + error: errorMessageWithCause(error), + }), + ); + captureError(error, { kind: "queue_foreground_liveness_release_crashed" }); + } + } + function enqueue(message: JobMessage, delaySeconds: number): void { const now = Date.now(); const payload = JSON.stringify(message); @@ -829,11 +935,15 @@ export function createSqliteQueue( // recreate the retry storm this feature exists to bound. The interval itself is the cooldown between // auto-retry rounds for any one job. deadLetterReviveTimer = setInterval(reviveDeadLetterJobsSafely, queueDeadLetterReviveIntervalMs()); + // Foreground-liveness sweep (#selfhost-queue-liveness): also a separate, slow interval -- see + // foreground-liveness.ts for why a per-tick check would busy-loop under sustained rate-limit pressure. + foregroundLivenessTimer = setInterval(releaseStaleForegroundDeferralsSafely, foregroundLivenessConfig.checkIntervalMs); }, async stop() { running = false; if (timer) clearTimeout(timer); if (deadLetterReviveTimer) clearInterval(deadLetterReviveTimer); + if (foregroundLivenessTimer) clearInterval(foregroundLivenessTimer); while (active > 0) await new Promise((r) => setTimeout(r, 10)); // let in-flight pumps finish }, async drain() { @@ -861,11 +971,22 @@ export function createSqliteQueue( ).c, ); }, + processingCount() { + return Number( + ( + driver.query( + `SELECT COUNT(*) AS c FROM ${TABLE} WHERE status='processing'`, + [], + ).rows[0] as { c: number } + ).c, + ); + }, stats() { return readQueueStats(driver); }, snapshot: binding.snapshot, reviveDeadLetterJobs, + releaseStaleForegroundDeferrals, pressureSignals() { return maintenancePressureSignals(driver, Date.now()); }, @@ -937,14 +1058,20 @@ function backfillJobForegroundLanes(driver: SqliteDriver): number { } /** Cheap aggregate reads behind the maintenance-admission policy (and the observability gauges in server.ts): - * how much LIVE (foreground) work is queued and how old the oldest of it is, and the same for the + * how much LIVE (foreground) work is queued and how old the oldest of it is -- both overall + * (pending+processing) and RUNNABLE right now (pending, due) -- and the same PENDING/oldest pair for the * MAINTENANCE lane specifically (not "all background" -- targeted jobs like backfill-repo-segment don't - * count, see maintenance-admission.ts). Host load is an independent, optional signal (see host-pressure.ts). */ + * count, see maintenance-admission.ts). The runnable-now split is the #selfhost-queue-liveness diagnostic: + * distinguishes "queue large but intentionally deferred" from "queue stuck, nothing runnable" without manual + * SQL. Host load is an independent, optional signal (see host-pressure.ts). */ function maintenancePressureSignals(driver: SqliteDriver, now: number): MaintenancePressureSignals { const live = driver.query( - `SELECT COUNT(*) as cnt, MIN(created_at) as oldest FROM ${TABLE} WHERE status IN ('pending','processing') AND priority>=?`, - [FOREGROUND_QUEUE_PRIORITY_FLOOR], - ).rows[0] as { cnt: number; oldest: number | null }; + `SELECT COUNT(*) as cnt, MIN(created_at) as oldest, + SUM(CASE WHEN status='pending' AND run_after<=? THEN 1 ELSE 0 END) as runnable_cnt, + MIN(CASE WHEN status='pending' AND run_after<=? THEN created_at ELSE NULL END) as oldest_runnable + FROM ${TABLE} WHERE status IN ('pending','processing') AND priority>=?`, + [now, now, FOREGROUND_QUEUE_PRIORITY_FLOOR], + ).rows[0] as { cnt: number; oldest: number | null; runnable_cnt: number | null; oldest_runnable: number | null }; const maintenance = driver.query( `SELECT COUNT(*) as cnt, MIN(created_at) as oldest FROM ${TABLE} WHERE status IN ('pending','processing') AND is_maintenance=1`, [], @@ -952,6 +1079,8 @@ function maintenancePressureSignals(driver: SqliteDriver, now: number): Maintena return { livePendingCount: Number(live.cnt), oldestLivePendingAgeMs: live.oldest != null ? now - Number(live.oldest) : null, + liveRunnableNowCount: Number(live.runnable_cnt ?? 0), + oldestLiveRunnableAgeMs: live.oldest_runnable != null ? now - Number(live.oldest_runnable) : null, maintenancePendingCount: Number(maintenance.cnt), oldestMaintenancePendingAgeMs: maintenance.oldest != null ? now - Number(maintenance.oldest) : null, hostLoadAvg1PerCore: hostLoadAvg1PerCore(), diff --git a/src/server.ts b/src/server.ts index 3225d4d09a..08de549a3e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -55,7 +55,7 @@ import { runSelfHostMigrations } from "./selfhost/migrate"; import { createPgAdapter, tuneGithubRateLimitObservationsAutovacuum } from "./selfhost/pg-adapter"; import { createPgQueue } from "./selfhost/pg-queue"; import { createPgVectorize, initPgVectorize } from "./selfhost/pg-vectorize"; -import { resolvePostgresPoolMax } from "./selfhost/queue-common"; +import { resolvePostgresPoolMax, type SelfHostQueueSnapshot } from "./selfhost/queue-common"; import type { MaintenancePressureSignals } from "./selfhost/maintenance-admission"; import { createSqliteQueue } from "./selfhost/sqlite-queue"; import { createSqliteVectorize } from "./selfhost/vectorize"; @@ -130,8 +130,10 @@ interface Backend { stop(): Promise; size(): number | Promise; deadCount(): number | Promise; + processingCount(): number | Promise; stats(): Record | Promise>; pressureSignals(): MaintenancePressureSignals | Promise; + snapshot(): SelfHostQueueSnapshot | Promise; }; vectorize?: Vectorize; shutdown(): Promise; @@ -603,6 +605,7 @@ async function main(): Promise { gauge("gittensory_queue_pending", () => backend.queue.size()); gauge("gittensory_queue_dead", () => backend.queue.deadCount()); + gauge("gittensory_queue_processing", () => backend.queue.processingCount()); const durableJobMetric = async (name: string): Promise => Number((await backend.queue.stats())[name] ?? 0); for (const name of [ @@ -635,6 +638,16 @@ async function main(): Promise { gauge("gittensory_queue_oldest_maintenance_pending_age_seconds", async () => Math.floor(((await maintenancePressure()).oldestMaintenancePendingAgeMs ?? 0) / 1000), ); + // #selfhost-queue-liveness: runnable-now is the "is anything actually due right now" signal the incident + // this module fixes required manual SQL to answer (processing=0, runnable_now=0 with hundreds pending). + // gittensory_queue_runnable_now covers every priority; the live-scoped pair narrows to foreground work + // specifically and adds the oldest-RUNNABLE age, distinct from oldest-PENDING age (which a job intentionally + // scheduled far out can inflate without indicating anything is stuck). + gauge("gittensory_queue_runnable_now", async () => (await backend.queue.snapshot()).totals.due); + gauge("gittensory_queue_live_runnable_now", async () => (await maintenancePressure()).liveRunnableNowCount); + gauge("gittensory_queue_oldest_live_runnable_age_seconds", async () => + Math.floor(((await maintenancePressure()).oldestLiveRunnableAgeMs ?? 0) / 1000), + ); // -1 (not 0) when unavailable -- a genuine idle host reads 0, so a dashboard can tell "known idle" apart // from "no signal on this platform" (see host-pressure.ts). gauge("gittensory_host_load_avg1_per_core", async () => (await maintenancePressure()).hostLoadAvg1PerCore ?? -1); diff --git a/src/signals/focus-manifest.ts b/src/signals/focus-manifest.ts index 1357c21eb4..29aba3f86f 100644 --- a/src/signals/focus-manifest.ts +++ b/src/signals/focus-manifest.ts @@ -97,6 +97,11 @@ export type FocusManifestGateConfig = { /** `gate.cla.checkRunAppSlug`: the trusted GitHub App slug that must produce `checkRunName`. null (unset) ⇒ * check-run detection remains unresolved rather than trusting a spoofable name-only match. */ claCheckRunAppSlug: string | null; + /** `gate.expectedCiContexts` (#selfhost-ci-verification): CI check/status context names to treat as + * required when GitHub branch-protection required-status-checks are unreadable or unconfigured. null + * (unset) ⇒ no generic fallback configured — the live-CI aggregate keeps today's fold-all behavior + * when branch protection is also unreadable. See {@link RepositorySettings.expectedCiContexts}. */ + expectedCiContexts: ReadonlyArray | null; }; // The converged per-PR review features a self-host operator toggles PER-REPO under `features:` in the private @@ -383,6 +388,7 @@ const EMPTY_GATE_CONFIG: FocusManifestGateConfig = { claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null, + expectedCiContexts: null, }; const EMPTY_FEATURES_CONFIG: FocusManifestFeaturesConfig = { @@ -483,6 +489,15 @@ function normalizeStringList(value: JsonValue | undefined, field: string, warnin return result; } +/** Like {@link normalizeStringList}, but returns `null` (not `[]`) when unset or when nothing survives + * validation — the convention every OTHER `FocusManifestGateConfig` field uses for "not configured", so + * the resolver's `!== null` overlay checks work uniformly. */ +function normalizeOptionalStringList(value: JsonValue | undefined, field: string, warnings: string[]): ReadonlyArray | null { + if (value === undefined || value === null) return null; + const list = normalizeStringList(value, field, warnings); + return list.length > 0 ? list : null; +} + function normalizeEnum(value: JsonValue | undefined, field: string, allowed: readonly T[], fallback: T, warnings: string[]): T { if (value === undefined || value === null) return fallback; if (typeof value !== "string" || !allowed.includes(value as T)) { @@ -662,6 +677,7 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu claConsentPhrase: parsePublicSafeText(claRecord?.consentPhrase, "gate.cla.consentPhrase", warnings), claCheckRunName: parsePublicSafeText(claRecord?.checkRunName, "gate.cla.checkRunName", warnings), claCheckRunAppSlug: parsePublicSafeText(claRecord?.checkRunAppSlug, "gate.cla.checkRunAppSlug", warnings), + expectedCiContexts: normalizeOptionalStringList(record.expectedCiContexts, "gate.expectedCiContexts", warnings), }; // #2266: the flag is parsed, clamped, and threaded end-to-end, but the gate evaluator never reads it — a // maintainer who sets it to true believing it softens a blocker for newcomers gets no such effect. Surface @@ -701,7 +717,8 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu gate.claMode !== null || gate.claConsentPhrase !== null || gate.claCheckRunName !== null || - gate.claCheckRunAppSlug !== null; + gate.claCheckRunAppSlug !== null || + gate.expectedCiContexts !== null; return gate; } @@ -773,6 +790,7 @@ export function gateConfigToJson(gate: FocusManifestGateConfig): JsonValue { if (gate.claCheckRunAppSlug !== null) cla.checkRunAppSlug = gate.claCheckRunAppSlug; out.cla = cla; } + if (gate.expectedCiContexts !== null) out.expectedCiContexts = gate.expectedCiContexts as JsonValue; return out; } @@ -1556,6 +1574,7 @@ export function resolveEffectiveSettings( if (gate.claConsentPhrase !== null) effective.claConsentPhrase = gate.claConsentPhrase; if (gate.claCheckRunName !== null) effective.claCheckRunName = gate.claCheckRunName; if (gate.claCheckRunAppSlug !== null) effective.claCheckRunAppSlug = gate.claCheckRunAppSlug; + if (gate.expectedCiContexts !== null) effective.expectedCiContexts = gate.expectedCiContexts; // The dashboard "Require linked issue" toggle must not silently diverge from gate blocking: when the // boolean is on but linkedIssueGateMode is still off, treat it as a block requirement (#797). if (effective.requireLinkedIssue && effective.linkedIssueGateMode === "off") { diff --git a/src/types.ts b/src/types.ts index b891a6edf8..5b322ad269 100644 --- a/src/types.ts +++ b/src/types.ts @@ -25,9 +25,16 @@ export type JobMessage = attempt: number; } | { - // One bounded re-gate unit fanned out by the scheduled sweep (#audit-sweep-fanout): re-review + stamp a - // single PR. Each candidate becomes its own individually-retryable, rate-limited queue message so the heavy - // re-review work interleaves with other jobs instead of monopolizing the consumer for all 25 at once. + // One bounded re-gate unit: re-review + stamp a single PR. Each candidate becomes its own individually- + // retryable, rate-limited queue message so the heavy re-review work interleaves with other jobs instead + // of monopolizing the consumer. Producers: the scheduled sweep's stale-candidate fan-out + // (#audit-sweep-fanout, deliveryId prefixed "regate-sweep:" — genuinely deferrable maintenance) and the + // sweep's own outage-repair fan-out (deliveryId prefixed "regate-repair:" — a PR missing a current-head + // Gate check or public-surface publish); a trailing coalesced re-review after a webhook burst; an + // over-cap sibling wake; a linked-issue-change re-review. EXCEPT for the "regate-sweep:" prefix, every + // producer carries the real webhook/event deliveryId that caused it — current-HEAD contributor-PR-review + // work, never background maintenance (isScheduledRegateSweepJob / githubRateLimitAdmissionTargetForJob in + // ../selfhost/queue-common.ts, #selfhost-queue-liveness). type: "agent-regate-pr"; deliveryId: string; repoFullName: string; @@ -599,6 +606,15 @@ export type RepositorySettings = { /** `gate.cla.checkRunAppSlug`: the trusted GitHub App slug that must have produced `claCheckRunName`. Required * for check-run detection so contributor-controlled same-name runs cannot satisfy a blocking CLA gate. */ claCheckRunAppSlug?: string | null | undefined; + /** `gate.expectedCiContexts` (#selfhost-ci-verification): maintainer-declared CI check/status context names to + * treat as required when GitHub branch protection returns no readable required-status-checks (unconfigured, + * or a 403 from a token lacking `administration:read` — common for GitHub App installations). Merged with any + * branch-protection required contexts when both exist; used ALONE when branch protection is null/empty; a + * repo with neither configured keeps the existing fold-all fail-closed behavior. A context missing from the + * commit ⇒ pending; a completed red check for a listed context ⇒ failed; every listed context settled clean + * ⇒ verified passed (no `ciCompletenessWarning`). Config-as-code only — no DB column; set via + * `.gittensory.yml gate.expectedCiContexts`. */ + expectedCiContexts?: ReadonlyArray | null | undefined; /** Dry-run disposition (#gate-dryrun). When true, the gate renders the would-be merge/close/manual verdict (every * advisory sub-gate promoted to block) WITHOUT enforcing — the posted check stays non-blocking. Lets advisory mode * preview exactly what it would do before the maintainer flips to real enforcement. Default off. */ diff --git a/test/unit/backfill.test.ts b/test/unit/backfill.test.ts index 2cb5b7a31b..4fb33f7256 100644 --- a/test/unit/backfill.test.ts +++ b/test/unit/backfill.test.ts @@ -42,6 +42,7 @@ import { fetchRequiredStatusContexts, isOwnReviewThreadAuthor, isRateLimitedGitHubFailure, + mergeRequiredCiContexts, refreshContributorActivity, refreshInstallationHealth, refreshPullRequestDetails, @@ -4334,6 +4335,74 @@ describe("GitHub backfill", () => { expect(aggregate.failingDetails).toEqual([expect.objectContaining({ name: "ci/overflow" })]); }); + describe("expectedCiContexts fallback (#selfhost-ci-verification)", () => { + it("passes with no completeness warning when branch protection is unreadable but an expected context settles clean", async () => { + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/check-runs?")) return Response.json({ check_runs: [{ name: "build", status: "completed", conclusion: "success" }] }); + if (url.includes("/status?")) return Response.json({ statuses: [] }); + return new Response("not found", { status: 404 }); + }); + + const requiredContexts = mergeRequiredCiContexts(null, ["build"]); + const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "abc123", "public-token", requiredContexts); + + // The key regression: an expectedCiContexts fallback (used when branch protection can't be read) + // resolves to enforce-required mode, so a clean settle is "passed" with NO completeness warning — + // unlike the fold-all path, which would warn (#2137). + expect(aggregate.ciState).toBe("passed"); + expect(aggregate.ciCompletenessWarning).toBeNull(); + }); + + it("stays pending when branch protection is unreadable and the expected context never appears on the commit", async () => { + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/check-runs?")) return Response.json({ check_runs: [] }); + if (url.includes("/status?")) return Response.json({ statuses: [] }); + return new Response("not found", { status: 404 }); + }); + + const requiredContexts = mergeRequiredCiContexts(null, ["build"]); + const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "abc123", "public-token", requiredContexts); + + expect(aggregate.ciState).toBe("pending"); + }); + + it("fails when branch protection is unreadable and the expected context completes red", async () => { + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/check-runs?")) return Response.json({ check_runs: [{ name: "build", status: "completed", conclusion: "failure" }] }); + if (url.includes("/status?")) return Response.json({ statuses: [] }); + return new Response("not found", { status: 404 }); + }); + + const requiredContexts = mergeRequiredCiContexts(null, ["build"]); + const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "abc123", "public-token", requiredContexts); + + expect(aggregate.ciState).toBe("failed"); + expect(aggregate.failingDetails).toEqual([expect.objectContaining({ name: "build" })]); + }); + + it("does not regress the no-config case: no branch protection and no expected contexts still fold-all warns on pass", async () => { + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/check-runs?")) return Response.json({ check_runs: [{ name: "test", status: "completed", conclusion: "success", app: { slug: "github-actions" } }] }); + if (url.includes("/status?")) return Response.json({ statuses: [] }); + if (url.includes("/check-suites?")) return Response.json({ check_suites: [{ status: "completed", app: { slug: "github-actions" } }] }); + return new Response("not found", { status: 404 }); + }); + + const requiredContexts = mergeRequiredCiContexts(null, undefined); + const aggregate = await fetchLiveCiAggregate(env, "JSONbored/gittensory", "abc123", "public-token", requiredContexts); + + expect(aggregate.ciState).toBe("passed"); + expect(aggregate.ciCompletenessWarning).toMatch(/branch-protection required checks/i); + }); + }); }); describe("fetchLiveReviewThreadBlockers", () => { @@ -5018,6 +5087,61 @@ describe("GitHub backfill", () => { }); }); + describe("mergeRequiredCiContexts", () => { + it("unions branch-protection contexts with expectedCiContexts when both have entries", () => { + const merged = mergeRequiredCiContexts(new Set(["build"]), ["test", "lint"]); + expect([...(merged as Set)].sort()).toEqual(["build", "lint", "test"]); + }); + + it("returns branch-protection contexts unchanged when expectedCiContexts is undefined", () => { + const merged = mergeRequiredCiContexts(new Set(["build", "test"]), undefined); + expect(merged).toBeInstanceOf(Set); + expect([...(merged as Set)].sort()).toEqual(["build", "test"]); + }); + + it("returns branch-protection contexts unchanged when expectedCiContexts is an empty array", () => { + const merged = mergeRequiredCiContexts(new Set(["build", "test"]), []); + expect([...(merged as Set)].sort()).toEqual(["build", "test"]); + }); + + it("returns branch-protection contexts unchanged when expectedCiContexts is null", () => { + const merged = mergeRequiredCiContexts(new Set(["build", "test"]), null); + expect([...(merged as Set)].sort()).toEqual(["build", "test"]); + }); + + it("returns just the expected set when branch protection is null and expectedCiContexts has entries", () => { + const merged = mergeRequiredCiContexts(null, ["build"]); + expect([...(merged as Set)]).toEqual(["build"]); + }); + + it("returns null when branch protection is null and expectedCiContexts is undefined", () => { + expect(mergeRequiredCiContexts(null, undefined)).toBeNull(); + }); + + it("returns null when branch protection is null and expectedCiContexts is null", () => { + expect(mergeRequiredCiContexts(null, null)).toBeNull(); + }); + + it("returns null when branch protection is null and expectedCiContexts is an empty array", () => { + expect(mergeRequiredCiContexts(null, [])).toBeNull(); + }); + + it("returns just the expected set when branch protection is an empty (non-null) Set and expectedCiContexts has entries", () => { + const merged = mergeRequiredCiContexts(new Set(), ["build"]); + expect([...(merged as Set)]).toEqual(["build"]); + }); + + it("drops blank/whitespace-only expectedCiContexts entries while keeping real entries", () => { + const merged = mergeRequiredCiContexts(null, [" ", "", "build"]); + expect([...(merged as Set)]).toEqual(["build"]); + }); + + it("trims leading/trailing whitespace from expectedCiContexts entries in the result", () => { + const merged = mergeRequiredCiContexts(null, [" build "]); + expect([...(merged as Set)]).toEqual(["build"]); + }); + }); + describe("fetchRequiredStatusContexts", () => { it("returns null without fetching when baseRef is missing", async () => { const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "public-token" }); diff --git a/test/unit/focus-manifest.test.ts b/test/unit/focus-manifest.test.ts index cc313c9528..1b0764aa31 100644 --- a/test/unit/focus-manifest.test.ts +++ b/test/unit/focus-manifest.test.ts @@ -516,7 +516,7 @@ describe("compileFocusManifestPolicy", () => { issueDiscoveryPolicy: "neutral", maintainerNotes: [], publicNotes: ["Keep PRs focused.", "Maximize your reward payout"], - gate: { present: false, enabled: null, pack: null, linkedIssue: null, duplicates: null, readinessMode: null, readinessMinScore: null, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, lockfileIntegrityMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, aiReviewCombine: null, aiReviewOnMerge: null, aiReviewReviewers: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, manifestPolicy: null, dryRun: null, firstTimeContributorGrace: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, claMode: null, claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null }, + gate: { present: false, enabled: null, pack: null, linkedIssue: null, duplicates: null, readinessMode: null, readinessMinScore: null, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, lockfileIntegrityMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, aiReviewCombine: null, aiReviewOnMerge: null, aiReviewReviewers: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, manifestPolicy: null, dryRun: null, firstTimeContributorGrace: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, claMode: null, claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null, expectedCiContexts: null }, settings: {}, review: { present: false, footerText: null, note: null, fields: {}, profile: null, securityFocus: null, inlineComments: null, pathInstructions: [], instructions: null, excludePaths: [], preMergeChecks: [] }, features: { present: false, rag: null, reputation: null, unifiedComment: null, safety: null }, @@ -824,7 +824,7 @@ describe("parseFocusManifest gate config", () => { // the block→advisory deprecation-downgrade behavior itself is covered separately below. const m = parseFocusManifest({ gate: { linkedIssue: "block", duplicates: "advisory", readiness: { mode: "advisory", minScore: 70 } } }); expect(m.present).toBe(true); - expect(m.gate).toEqual({ present: true, enabled: null, pack: null, linkedIssue: "block", duplicates: "advisory", readinessMode: "advisory", readinessMinScore: 70, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, lockfileIntegrityMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, aiReviewCombine: null, aiReviewOnMerge: null, aiReviewReviewers: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, manifestPolicy: null, dryRun: null, firstTimeContributorGrace: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, claMode: null, claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null }); + expect(m.gate).toEqual({ present: true, enabled: null, pack: null, linkedIssue: "block", duplicates: "advisory", readinessMode: "advisory", readinessMinScore: 70, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, lockfileIntegrityMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, aiReviewCombine: null, aiReviewOnMerge: null, aiReviewReviewers: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, manifestPolicy: null, dryRun: null, firstTimeContributorGrace: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, claMode: null, claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null, expectedCiContexts: null }); }); it("parses gate.mergeReadiness + gate.firstTimeContributorGrace, round-trips them, and warns on bad values (#822)", () => { @@ -2397,3 +2397,96 @@ describe("gate.claMode / gate.cla CLA / license-compatibility gate config (#2564 expect(eff.claConsentPhrase).toBe("agree to the CLA"); }); }); + +describe("gate.expectedCiContexts (#selfhost-ci-verification)", () => { + it("parses a clean list, sets present, and preserves order", () => { + const m = parseFocusManifest({ gate: { expectedCiContexts: ["build", "test"] } }); + expect(m.gate.expectedCiContexts).toEqual(["build", "test"]); + expect(m.gate.present).toBe(true); + }); + + it("trims whitespace from each entry", () => { + const m = parseFocusManifest({ gate: { expectedCiContexts: [" build ", "test"] } }); + expect(m.gate.expectedCiContexts).toEqual(["build", "test"]); + }); + + it("drops a non-string entry, keeps the valid ones, and warns naming the field", () => { + const m = parseFocusManifest({ gate: { expectedCiContexts: ["build", 42, "test"] as never } }); + expect(m.gate.expectedCiContexts).toEqual(["build", "test"]); + expect(m.warnings.some((w) => w.includes("gate.expectedCiContexts") && /non-string entry/i.test(w))).toBe(true); + }); + + it("silently drops blank/whitespace-only entries with no warning (matches normalizeStringList's blank-skip branch)", () => { + const m = parseFocusManifest({ gate: { expectedCiContexts: ["build", "", " "] } }); + expect(m.gate.expectedCiContexts).toEqual(["build"]); + expect(m.warnings).toEqual([]); + }); + + it("is null when gate.expectedCiContexts is absent, and gate.present is not forced true by an otherwise-empty gate block", () => { + const withEmptyGate = parseFocusManifest({ gate: {} }); + expect(withEmptyGate.gate.expectedCiContexts).toBeNull(); + expect(withEmptyGate.gate.present).toBe(false); + + const withNoGateKey = parseFocusManifest({}); + expect(withNoGateKey.gate.expectedCiContexts).toBeNull(); + expect(withNoGateKey.gate.present).toBe(false); + }); + + it("is null when gate.expectedCiContexts is explicitly null", () => { + const m = parseFocusManifest({ gate: { expectedCiContexts: null } }); + expect(m.gate.expectedCiContexts).toBeNull(); + expect(m.gate.present).toBe(false); + }); + + it("normalizes an entirely blank/invalid list back to null, not an empty array (normalizeOptionalStringList's empty-after-normalization branch)", () => { + const m = parseFocusManifest({ gate: { expectedCiContexts: ["", " ", 123] as never } }); + expect(m.gate.expectedCiContexts).toBeNull(); + // Distinct from the "absent" case: this run DID produce warnings (the non-string 123 entry) even + // though the final normalized value collapses to null just like the absent case does. + expect(m.warnings.some((w) => w.includes("gate.expectedCiContexts"))).toBe(true); + }); + + it("warns and drops a non-array value (mirrors normalizeStringList's own non-array warning branch)", () => { + const nonArrayString = parseFocusManifest({ gate: { expectedCiContexts: "build" as never } }); + expect(nonArrayString.gate.expectedCiContexts).toBeNull(); + expect(nonArrayString.warnings.some((w) => w.includes("gate.expectedCiContexts") && /must be a list/i.test(w))).toBe(true); + + const nonArrayObject = parseFocusManifest({ gate: { expectedCiContexts: { build: true } as never } }); + expect(nonArrayObject.gate.expectedCiContexts).toBeNull(); + expect(nonArrayObject.warnings.some((w) => w.includes("gate.expectedCiContexts") && /must be a list/i.test(w))).toBe(true); + }); + + it("round-trips a set expectedCiContexts through gateConfigToJson and back through parseFocusManifest", () => { + const m = parseFocusManifest({ gate: { expectedCiContexts: ["build", "lint"] } }); + const json = gateConfigToJson(m.gate); + expect(json).toMatchObject({ expectedCiContexts: ["build", "lint"] }); + const round = parseFocusManifest({ gate: json }); + expect(round.gate.expectedCiContexts).toEqual(["build", "lint"]); + }); + + it("omits the expectedCiContexts key from gateConfigToJson output when unset", () => { + const m = parseFocusManifest({ gate: { claMode: "block" } }); + expect(m.gate.expectedCiContexts).toBeNull(); + const json = gateConfigToJson(m.gate); + expect(json).not.toBeNull(); + expect("expectedCiContexts" in (json as Record)).toBe(false); + }); + + it("overlay wins over the DB value when the manifest sets expectedCiContexts", () => { + const db = { expectedCiContexts: ["old"] } as unknown as RepositorySettings; + const m = parseFocusManifest({ gate: { expectedCiContexts: ["new"] } }); + const eff = resolveEffectiveSettings(db, m); + expect(eff.expectedCiContexts).toEqual(["new"]); + }); + + it("lets the DB value pass through when the manifest doesn't configure expectedCiContexts", () => { + const db = { expectedCiContexts: ["from-db"] } as unknown as RepositorySettings; + const eff = resolveEffectiveSettings(db, parseFocusManifest(null)); + expect(eff.expectedCiContexts).toEqual(["from-db"]); + }); + + it("is undefined when neither the DB nor the manifest sets expectedCiContexts (no DB column for this field)", () => { + const eff = resolveEffectiveSettings({} as unknown as RepositorySettings, parseFocusManifest(null)); + expect(eff.expectedCiContexts).toBeUndefined(); + }); +}); diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index a382a05528..e0475a8781 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -1579,6 +1579,112 @@ describe("queue processors", () => { } }); + // #selfhost-ci-verification: settings.expectedCiContexts must actually change the live-CI disposition, not just + // get threaded through as an inert parameter. Branch protection is unreadable (empty) on BOTH calls, so without + // expectedCiContexts folded into mergeRequiredCiContexts every check-run folds to "passed" (fold-all); WITH + // expectedCiContexts naming a context that never appears in check-runs, mergeRequiredCiContexts makes it the + // SOLE required context and reduceLiveCiAggregate's "a required context that never appeared is not safe to + // treat as passed" rule (backfill.ts) forces ciState to "pending" — deferring the review before auto-maintain + // ever runs. Two full processJob passes (each gets its own request-scoped LiveGithubFacts, so this is a + // same-repo/baseRef/headSha comparison of the MERGED outcome, not a same-cache-hit test) prove the config is + // live, not stale/ignored. + it("REGRESSION (#selfhost-ci-verification): expectedCiContexts turns an otherwise-passing fold-all CI aggregate into a deferred pending review", async () => { + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); + await upsertInstallation(env, { action: "created", installation: { id: 9001, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: { pull_requests: "write", checks: "write" }, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9001); + await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { merge: "auto", update_branch: "auto" }, aiReviewMode: "off", gatePack: "oss-anti-slop", gateCheckMode: "enabled", checkRunMode: "off", commentMode: "off", publicSurface: "off" }); + await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 7, title: "Fold-all vs configured", state: "open", user: { login: "contributor" }, head: { sha: "a7" }, base: { ref: "main" }, labels: [], body: "Closes #1" }); + let branchProtectionGets = 0; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = (init?.method ?? "GET").toUpperCase(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (/\/pulls\/7(?:\?|$)/.test(url) && method === "GET") return Response.json({ number: 7, title: "Fold-all vs configured", state: "open", user: { login: "contributor" }, head: { sha: "a7" }, mergeable_state: "clean", labels: [], body: "Closes #1" }); + if (url.includes("/pulls/7/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = true;" }]); + // Neither call's check-runs/status ever mentions "required-build" — only expectedCiContexts makes that matter. + if (url.includes("/commits/a7/check-runs")) return Response.json({ total_count: 1, check_runs: [{ name: "lint", status: "completed", conclusion: "success", app: { slug: "github-actions" } }] }); + if (url.includes("/commits/a7/status")) return Response.json({ state: "success", statuses: [] }); + if (url.includes("/issues/1")) return Response.json({ number: 1, title: "Issue", state: "open", labels: [], user: { login: "reporter" } }); + // Branch protection unreadable on both calls — expectedCiContexts is the ONLY source of a required context. + if (url.includes("/branches/")) { + branchProtectionGets += 1; + return new Response("forbidden", { status: 403 }); + } + return Response.json({}); + }); + + // Call A: no expectedCiContexts configured — fold-all mode, nothing pending, review proceeds normally. + await processJob(env, { type: "agent-regate-pr", deliveryId: "no-expected-contexts", repoFullName: "owner/agent-repo", prNumber: 7, installationId: 9001 }); + const deferredBefore = await env.DB.prepare("select count(*) as n from audit_events where event_type = ? and metadata_json like ?") + .bind("github_app.review_deferred_ci_pending", '%"no-expected-contexts"%') + .first<{ n: number }>(); + expect(deferredBefore?.n).toBe(0); + expect(branchProtectionGets).toBe(1); + + // Config change: gate.expectedCiContexts now names a context absent from every check-run/status above. + await upsertRepoFocusManifest(env, "owner/agent-repo", { gate: { expectedCiContexts: ["required-build"] } }); + + // Call B: SAME repo/baseRef/headSha/check-run state — only settings.expectedCiContexts changed. + await processJob(env, { type: "agent-regate-pr", deliveryId: "with-expected-contexts", repoFullName: "owner/agent-repo", prNumber: 7, installationId: 9001 }); + + // The branch-protection endpoint was fetched again for call B (a fresh per-job LiveGithubFacts always misses), + // proving the merged result was actually RE-DERIVED against the new config rather than reused from call A. + expect(branchProtectionGets).toBe(2); + const deferredAfter = await env.DB.prepare("select count(*) as n from audit_events where event_type = ? and metadata_json like ?") + .bind("github_app.review_deferred_ci_pending", '%"with-expected-contexts"%') + .first<{ n: number }>(); + // "required-build" never appears in check-runs/status ⇒ mergeRequiredCiContexts(null, ["required-build"]) makes + // it the sole required context ⇒ reduceLiveCiAggregate treats the unseen required context as pending ⇒ + // prReadyForReview defers BEFORE auto-maintain runs — the opposite disposition of call A on identical CI data. + expect(deferredAfter?.n).toBe(1); + }); + + // #selfhost-ci-verification: within a SINGLE processJob pass, cachedRequiredStatusContexts is reached from THREE + // call sites sharing one request-scoped LiveGithubFacts — prReadyForReview (via cachedLiveCiAggregate), + // maybePublishPrPublicSurface (via refreshLiveCiAggregate), and runAgentMaintenancePlanAndExecute (directly, and + // again via refreshLiveCiAggregate). All three now fold expectedCiContextsKeyPart(settings.expectedCiContexts) + // into their cache key. Since settings is resolved ONCE per job, expectedCiContexts is constant across the three + // call sites within this one pass — this proves folding it into the key did NOT reintroduce a redundant fetch: + // the branch-protection endpoint is still hit exactly once for the whole job, exactly like before expectedCiContexts + // existed (see the sibling "#audit-rate-headroom: the per-PR re-review refreshes..." dedup test above). + it("REGRESSION (#selfhost-ci-verification): expectedCiContexts in the cache key does not defeat within-job required-contexts memoization", async () => { + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); + await upsertInstallation(env, { action: "created", installation: { id: 9001, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: { pull_requests: "write", checks: "write" }, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9001); + await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { merge: "auto", update_branch: "auto" }, autoMaintain: { requireApprovals: 0, mergeMethod: "squash" }, aiReviewMode: "off", gatePack: "oss-anti-slop", gateCheckMode: "enabled", checkRunMode: "off", commentMode: "off", publicSurface: "off" }); + await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 7, title: "Configured + clean", state: "open", user: { login: "contributor" }, head: { sha: "a7" }, base: { ref: "main" }, labels: [], body: "Closes #1" }); + await upsertPullRequestDetailSyncState(env, { repoFullName: "owner/agent-repo", pullNumber: 7, status: "complete", reviewsSyncedAt: new Date().toISOString() }); + // gate.expectedCiContexts is satisfied by a real, passing check-run — so CI resolves cleanly and the pass + // proceeds all the way through readiness, public-surface publish, AND auto-maintain (unlike the deferred-pending + // test above, which deliberately stops at readiness to prove the disposition changes). + await upsertRepoFocusManifest(env, "owner/agent-repo", { gate: { expectedCiContexts: ["required-build"] } }); + let branchProtectionGets = 0; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = (init?.method ?? "GET").toUpperCase(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (/\/pulls\/7(?:\?|$)/.test(url) && method === "GET") return Response.json({ number: 7, title: "Configured + clean", state: "open", user: { login: "contributor" }, head: { sha: "a7" }, mergeable_state: "clean", labels: [], body: "Closes #1" }); + if (url.includes("/pulls/7/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = true;" }]); + if (url.includes("/commits/a7/check-runs")) return Response.json({ total_count: 1, check_runs: [{ name: "required-build", status: "completed", conclusion: "success", app: { slug: "github-actions" } }] }); + if (url.includes("/commits/a7/status")) return Response.json({ state: "success", statuses: [] }); + if (url.includes("/issues/1")) return Response.json({ number: 1, title: "Issue", state: "open", labels: [], user: { login: "reporter" } }); + if (url.includes("/branches/")) { + branchProtectionGets += 1; + return new Response("forbidden", { status: 403 }); + } + return Response.json({}); + }); + + await processJob(env, { type: "agent-regate-pr", deliveryId: "configured-memoized", repoFullName: "owner/agent-repo", prNumber: 7, installationId: 9001 }); + + // One fetch for the whole job despite three internal call sites sharing the config-aware cache key. + expect(branchProtectionGets).toBe(1); + const deferred = await env.DB.prepare("select count(*) as n from audit_events where event_type = ?") + .bind("github_app.review_deferred_ci_pending") + .first<{ n: number }>(); + expect(deferred?.n).toBe(0); + }); + it("#sweep-resync: a failing resync upsert is swallowed (fail-open) — the sweep never throws", async () => { const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); await upsertInstallation(env, { action: "created", installation: { id: 9001, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); @@ -4073,6 +4179,53 @@ describe("queue processors", () => { expect(fanned.map((job) => job.prNumber)).toEqual([1]); // only the priority repair, not PRs 2-5 }); + it("REGRESSION: the sweep tags a priority-repair fan-out with 'regate-repair:' and an ordinary candidate with 'regate-sweep:' (#selfhost-queue-liveness)", async () => { + const sent: import("../../src/types").JobMessage[] = []; + const env = createTestEnv({ JOBS: { async send(m: import("../../src/types").JobMessage) { sent.push(m); } } as unknown as Queue }); + await upsertInstallation(env, { action: "created", installation: { id: 9404, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9404); + await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { merge: "auto" }, gateCheckMode: "enabled", checkRunMode: "off", commentMode: "off", publicSurface: "off" }); + // PR 1: missing its current Gate check for its current head -- surfaceRepairPriorityPullNumbers flags this as + // outage-repair priority (no completed Gittensory Gate check run at the live head SHA). + await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 1, title: "Repair 1", state: "open", user: { login: "c" }, head: { sha: "repair-1" }, labels: [], body: "" }); + await repositoriesModule.markPullRequestSurfacePublished(env, "owner/agent-repo", 1, "repair-1"); + // PR 2: ordinary PR with a completed current-head Gate check -- NOT priority. + await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 2, title: "Ordinary 2", state: "open", user: { login: "c" }, head: { sha: "ordinary-2" }, labels: [], body: "" }); + await repositoriesModule.markPullRequestSurfacePublished(env, "owner/agent-repo", 2, "ordinary-2"); + await upsertCheckSummary(env, { + id: "gate-current-2", + repoFullName: "owner/agent-repo", + pullNumber: 2, + headSha: "ordinary-2", + name: "Gittensory Orb Review Agent", + status: "completed", + conclusion: "success", + payload: {}, + }); + vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z")); + + await processJob(env, { type: "agent-regate-sweep", requestedBy: "test", repoFullName: "owner/agent-repo" }); + + const fanned = sent.filter((job): job is Extract => job.type === "agent-regate-pr"); + expect(fanned).toHaveLength(2); + const repairJob = fanned.find((job) => job.prNumber === 1); + const ordinaryJob = fanned.find((job) => job.prNumber === 2); + expect(repairJob).toMatchObject({ + type: "agent-regate-pr", + deliveryId: "regate-repair:owner/agent-repo#1", + repoFullName: "owner/agent-repo", + prNumber: 1, + installationId: 9404, + }); + expect(ordinaryJob).toMatchObject({ + type: "agent-regate-pr", + deliveryId: "regate-sweep:owner/agent-repo#2", + repoFullName: "owner/agent-repo", + prNumber: 2, + installationId: 9404, + }); + }); + it("agent re-gate sweep fail-opens when current Gate check reads fail during repair priority selection", async () => { const sent: import("../../src/types").JobMessage[] = []; const env = createTestEnv({ JOBS: { async send(m: import("../../src/types").JobMessage) { sent.push(m); } } as unknown as Queue }); @@ -4854,6 +5007,9 @@ describe("queue processors", () => { await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9203); await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { merge: "auto" } }); await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 9, title: "PR9", state: "open", user: { login: "c" }, head: { sha: "a9" }, labels: [], body: "" }); + // Published at the current head so this is an ORDINARY (non-priority-repair) candidate -- this test is about + // backlog-row-type filtering, not the priority-repair "regate-repair:" tagging (covered separately above). + await repositoriesModule.markPullRequestSurfacePublished(env, "owner/agent-repo", 9, "a9"); vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z")); await processJob(env, { type: "agent-regate-sweep", requestedBy: "schedule", repoFullName: "owner/agent-repo" }); @@ -4885,6 +5041,9 @@ describe("queue processors", () => { await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9202); await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { merge: "auto" } }); await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 8, title: "PR8", state: "open", user: { login: "c" }, head: { sha: "a8" }, labels: [], body: "" }); + // Published at the current head so this is an ORDINARY (non-priority-repair) candidate -- this test is about + // queue-introspection independence, not the priority-repair "regate-repair:" tagging (covered separately above). + await repositoriesModule.markPullRequestSurfacePublished(env, "owner/agent-repo", 8, "a8"); vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z")); await processJob(env, { type: "agent-regate-sweep", requestedBy: "schedule", repoFullName: "owner/agent-repo" }); @@ -4914,6 +5073,40 @@ describe("queue processors", () => { stamp.mockRestore(); }); + it("REGRESSION: a 'regate-sweep:' per-PR job DEFERS at the maintenance floor even with headroom above the lower live floor (#selfhost-queue-liveness)", async () => { + const sent: import("../../src/types").JobMessage[] = []; + const env = createTestEnv({ JOBS: { async send(m: import("../../src/types").JobMessage) { sent.push(m); } } as unknown as Queue }); + vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z")); + // 100 remaining sits BELOW the 150 maintenance floor but ABOVE the 75 live floor -- isScheduledRegateSweepJob + // must route this "regate-sweep:"-prefixed job to the higher (150) floor, so it still defers here. + await repositoriesModule.recordGitHubRateLimitObservation(env, { repoFullName: "owner/agent-repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 100, resetAt: "2026-05-28T02:30:00.000Z", observedAt: "2026-05-28T02:00:00.000Z" }); + const stamp = vi.spyOn(repositoriesModule, "markPullRequestsRegated"); + + await processJob(env, { type: "agent-regate-pr", deliveryId: "regate-sweep:owner/agent-repo#7", repoFullName: "owner/agent-repo", prNumber: 7, installationId: 9200 }); + + expect(sent.filter((m) => m.type === "agent-regate-pr")).toHaveLength(1); // re-queued for after the reset + expect(stamp).not.toHaveBeenCalled(); + stamp.mockRestore(); + }); + + it("REGRESSION: a non-'regate-sweep:' per-PR job (current-head trigger) does NOT defer at the maintenance floor, only at the lower live floor (#selfhost-queue-liveness)", async () => { + const sent: import("../../src/types").JobMessage[] = []; + const env = createTestEnv({ JOBS: { async send(m: import("../../src/types").JobMessage) { sent.push(m); } } as unknown as Queue }); + vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z")); + // Same 100-remaining observation as the sibling "regate-sweep:" test above, but this deliveryId does NOT carry + // the "regate-sweep:" prefix (e.g. a repair-priority fan-out, or a real webhook-triggered re-review), so + // isScheduledRegateSweepJob is false and shouldWaitForGitHubRateLimit is called with the lower 75 floor: + // 100 > 75, so this job proceeds instead of deferring. + await repositoriesModule.recordGitHubRateLimitObservation(env, { repoFullName: "owner/agent-repo", resource: "rest", path: "/x", statusCode: 200, limitValue: 5000, remaining: 100, resetAt: "2026-05-28T02:30:00.000Z", observedAt: "2026-05-28T02:00:00.000Z" }); + + // No stored PR row for prNumber 7 -- reReviewStoredPullRequest reaches its `getPullRequest` read (proving the + // rate-limit gate did not short-circuit it) and then returns immediately with no re-enqueue, since there is + // nothing to review. A deferral would instead re-enqueue this exact job (asserted absent below). + await processJob(env, { type: "agent-regate-pr", deliveryId: "regate-repair:owner/agent-repo#7", repoFullName: "owner/agent-repo", prNumber: 7, installationId: 9200 }); + + expect(sent.filter((m) => m.type === "agent-regate-pr")).toEqual([]); // proceeded — no rate-limit re-enqueue + }); + it("routes repo-scoped backfill jobs into resumable segment and detail processors", async () => { const sent: import("../../src/types").JobMessage[] = []; const env = createTestEnv({ diff --git a/test/unit/selfhost-foreground-liveness.test.ts b/test/unit/selfhost-foreground-liveness.test.ts new file mode 100644 index 0000000000..d0f7e18067 --- /dev/null +++ b/test/unit/selfhost-foreground-liveness.test.ts @@ -0,0 +1,107 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + isForegroundDeferralStale, + resolveForegroundLivenessConfig, + type ForegroundLivenessConfig, +} from "../../src/selfhost/foreground-liveness"; + +describe("resolveForegroundLivenessConfig", () => { + const envKeys = [ + "FOREGROUND_LIVENESS_ENABLED", + "FOREGROUND_LIVENESS_MAX_DEFER_MS", + "FOREGROUND_LIVENESS_CHECK_INTERVAL_MS", + ] as const; + const saved: Record = {}; + + beforeEach(() => { + for (const key of envKeys) { + saved[key] = process.env[key]; + delete process.env[key]; + } + }); + + afterEach(() => { + for (const key of envKeys) { + if (saved[key] === undefined) delete process.env[key]; + else process.env[key] = saved[key]; + } + }); + + it("returns protective defaults with no env overrides", () => { + expect(resolveForegroundLivenessConfig()).toEqual({ + enabled: true, + maxDeferMs: 600_000, + checkIntervalMs: 60_000, + }); + }); + + it("reads a custom FOREGROUND_LIVENESS_MAX_DEFER_MS and FOREGROUND_LIVENESS_CHECK_INTERVAL_MS when set", () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "120000"; + process.env.FOREGROUND_LIVENESS_CHECK_INTERVAL_MS = "10000"; + const config = resolveForegroundLivenessConfig(); + expect(config.maxDeferMs).toBe(120_000); + expect(config.checkIntervalMs).toBe(10_000); + }); + + it.each(["0", "false", "off", "no"])("treats FOREGROUND_LIVENESS_ENABLED=%s as disabled", (value) => { + process.env.FOREGROUND_LIVENESS_ENABLED = value; + expect(resolveForegroundLivenessConfig().enabled).toBe(false); + }); + + it.each(["1", "true", "on", "yes", "anything-else"])( + "treats FOREGROUND_LIVENESS_ENABLED=%s as enabled", + (value) => { + process.env.FOREGROUND_LIVENESS_ENABLED = value; + expect(resolveForegroundLivenessConfig().enabled).toBe(true); + }, + ); + + it("keeps liveness enabled when the env var is unset/empty", () => { + delete process.env.FOREGROUND_LIVENESS_ENABLED; + expect(resolveForegroundLivenessConfig().enabled).toBe(true); + process.env.FOREGROUND_LIVENESS_ENABLED = ""; + expect(resolveForegroundLivenessConfig().enabled).toBe(true); + }); + + it("falls back to the default max defer when the value is non-numeric", () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "not-a-number"; + expect(resolveForegroundLivenessConfig().maxDeferMs).toBe(600_000); + }); + + it("falls back to the default max defer when the value is below the min (60_000)", () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "59999"; + expect(resolveForegroundLivenessConfig().maxDeferMs).toBe(600_000); + }); + + it("falls back to the default check interval when the value is non-numeric", () => { + process.env.FOREGROUND_LIVENESS_CHECK_INTERVAL_MS = "not-a-number"; + expect(resolveForegroundLivenessConfig().checkIntervalMs).toBe(60_000); + }); + + it("falls back to the default check interval when the value is below the min (5_000)", () => { + process.env.FOREGROUND_LIVENESS_CHECK_INTERVAL_MS = "4999"; + expect(resolveForegroundLivenessConfig().checkIntervalMs).toBe(60_000); + }); +}); + +describe("isForegroundDeferralStale", () => { + const now = 1_000_000_000; + const config: ForegroundLivenessConfig = { enabled: true, maxDeferMs: 600_000, checkIntervalMs: 60_000 }; + + it("is stale once the pending age is at or beyond maxDeferMs", () => { + expect(isForegroundDeferralStale(config, now - config.maxDeferMs - 1, now)).toBe(true); + }); + + it("is stale exactly AT the boundary (>=, not >)", () => { + expect(isForegroundDeferralStale(config, now - config.maxDeferMs, now)).toBe(true); + }); + + it("is not stale when the pending age is below maxDeferMs", () => { + expect(isForegroundDeferralStale(config, now - (config.maxDeferMs - 1), now)).toBe(false); + }); + + it("is never stale when disabled, even for a huge age (config.enabled && short-circuit)", () => { + const disabled: ForegroundLivenessConfig = { ...config, enabled: false }; + expect(isForegroundDeferralStale(disabled, now - config.maxDeferMs * 100, now)).toBe(false); + }); +}); diff --git a/test/unit/selfhost-maintenance-admission.test.ts b/test/unit/selfhost-maintenance-admission.test.ts index cdef7d5028..05ae991f20 100644 --- a/test/unit/selfhost-maintenance-admission.test.ts +++ b/test/unit/selfhost-maintenance-admission.test.ts @@ -14,6 +14,8 @@ import { const CLEAR_SIGNALS: MaintenancePressureSignals = { livePendingCount: 0, oldestLivePendingAgeMs: null, + liveRunnableNowCount: 0, + oldestLiveRunnableAgeMs: null, maintenancePendingCount: 0, oldestMaintenancePendingAgeMs: null, hostLoadAvg1PerCore: null, diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index cb0015a04e..5dad20193f 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -70,11 +70,26 @@ interface MockPool { setReviveUpdateRowCounts(rowCounts: number[]): void; setRateLimitRows(rows: Array<{ admission_key?: string | null; repo_full_name?: string | null; remaining: number | string | null; reset_at: string | null; observed_at?: string | null }>): void; /** Configures the two maintenance-admission pressure aggregate queries (live + maintenance lane). Defaults - * to zero pending / null oldest in both lanes (pressure clear) until set. */ + * to zero pending / null oldest in both lanes (pressure clear) until set. `runnableCnt`/`oldestRunnable` + * back the #selfhost-queue-liveness runnable-now split (see maintenancePressureSignals's FILTER columns); + * they default to 0/null (nothing runnable) so existing tests that never set them keep working. */ setPressureSignals(signals: { - live?: { cnt: number; oldest: number | null }; + live?: { cnt: number; oldest: number | null; runnableCnt?: number; oldestRunnable?: number | null }; maintenance?: { cnt: number; oldest: number | null }; }): void; + /** Programs the exact rows returned by releaseStaleForegroundDeferrals' candidate SELECT + * (`WHERE status='pending' AND priority>=$1 AND run_after>$2`), and the per-row rowCount its conditional + * UPDATE reports (defaults to 1 -- the row still matched at UPDATE time -- when not otherwise queued). + * `payload` defaults to a "recapture-preview" message -- a foreground type NOT in + * GITHUB_BUDGET_BACKGROUND_TYPES / not "github-webhook"/"agent-regate-pr" -- so + * githubRateLimitAdmissionTargetForJob returns null for it and the rate-limit-clear OR-condition is + * trivially/always true, isolating the AGE condition cleanly for tests that aren't specifically about + * rate-limit clearing. Pass an explicit `payload` (e.g. a github-webhook message) plus `setRateLimitRows` + * to test the rate-limit-clear condition itself, or "not valid json" to test the unparseable-payload path. */ + setForegroundLivenessCandidates( + rows: Array<{ id: string; created_at: number; payload?: string }>, + updateRowCounts?: number[], + ): void; } function makePool(): MockPool { @@ -82,13 +97,54 @@ function makePool(): MockPool { let deferUpdateRowCount = 1; const reviveUpdateRowCounts: number[] = []; let rateLimitRows: Array<{ admission_key?: string | null; repo_full_name?: string | null; remaining: number | string | null; reset_at: string | null; observed_at?: string | null }> = []; - let pressureLive: { cnt: number; oldest: number | null } = { cnt: 0, oldest: null }; + let pressureLive: { cnt: number; oldest: number | null; runnableCnt?: number; oldestRunnable?: number | null } = { + cnt: 0, + oldest: null, + }; let pressureMaintenance: { cnt: number; oldest: number | null } = { cnt: 0, oldest: null }; + let foregroundLivenessCandidates: Array<{ id: string; created_at: number; payload?: string }> = []; + const foregroundLivenessUpdateRowCounts: number[] = []; + const DEFAULT_FOREGROUND_LIVENESS_PAYLOAD = JSON.stringify({ + type: "recapture-preview", + deliveryId: "seed", + repoFullName: "o/r", + prNumber: 1, + attempt: 1, + }); const fn = vi.fn().mockImplementation(async (sql: unknown, params?: unknown[]) => { const q = String(sql); + if (q.includes("SELECT id, payload, created_at FROM") && q.includes("priority>=$1 AND run_after>$2")) { + return { + rows: foregroundLivenessCandidates.map((row) => ({ + id: row.id, + payload: row.payload ?? DEFAULT_FOREGROUND_LIVENESS_PAYLOAD, + created_at: row.created_at, + })), + rowCount: foregroundLivenessCandidates.length, + }; + } + if (q.includes("SET run_after=$1 WHERE id=$2 AND status='pending' AND run_after>$1")) { + const rowCount = + foregroundLivenessUpdateRowCounts.length > 0 ? (foregroundLivenessUpdateRowCounts.shift() ?? 1) : 1; + return { rows: [], rowCount }; + } if (q.includes("AS cnt, MIN(created_at) AS oldest")) { const signal = q.includes("is_maintenance=1") ? pressureMaintenance : pressureLive; - return { rows: [{ cnt: String(signal.cnt), oldest: signal.oldest }], rowCount: 1 }; + return { + rows: [ + { + cnt: String(signal.cnt), + oldest: signal.oldest, + ...(signal === pressureLive + ? { + runnable_cnt: String(pressureLive.runnableCnt ?? 0), + oldest_runnable: pressureLive.oldestRunnable ?? null, + } + : {}), + }, + ], + rowCount: 1, + }; } if (q.includes("FROM github_rate_limit_observations")) { const admissionKey = typeof params?.[0] === "string" ? params[0] : null; @@ -154,6 +210,11 @@ function makePool(): MockPool { setRateLimitRows(rows) { rateLimitRows = rows; }, + setForegroundLivenessCandidates(rows, updateRowCounts) { + foregroundLivenessCandidates = rows; + foregroundLivenessUpdateRowCounts.length = 0; + if (updateRowCounts) foregroundLivenessUpdateRowCounts.push(...updateRowCounts); + }, }; } @@ -1105,7 +1166,7 @@ describe("createPgQueue (durable #977)", () => { m.setRateLimitRows([{ admission_key: "installation:123", repo_full_name: "owner/other-repo", remaining: "120", reset_at: "2026-06-24T12:10:00.000Z", observed_at: "2026-06-24T11:59:30.000Z" }]); m.enqueueJob("background", { type: "agent-regate-pr", - deliveryId: "sweep:owner/repo#7", + deliveryId: "regate-sweep:owner/repo#7", repoFullName: "owner/repo", prNumber: 7, installationId: 123, @@ -1145,7 +1206,7 @@ describe("createPgQueue (durable #977)", () => { m.setRateLimitRows([{ admission_key: "installation:123", repo_full_name: "owner/other-repo", remaining: "120", reset_at: "2026-06-24T12:10:00.000Z", observed_at: "2026-06-24T11:59:30.000Z" }]); m.enqueueJob("background", { type: "agent-regate-pr", - deliveryId: "sweep:owner/repo#7", + deliveryId: "regate-sweep:owner/repo#7", repoFullName: "owner/repo", prNumber: 7, installationId: 123, @@ -1572,6 +1633,233 @@ describe("createPgQueue (durable #977)", () => { }); }); + describe("releaseStaleForegroundDeferrals (#selfhost-queue-liveness)", () => { + afterEach(() => { + delete process.env.FOREGROUND_LIVENESS_ENABLED; + delete process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS; + }); + + it("releases a foreground-priority pending row deferred far into the future once its created_at is stale", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "60000"; // 1m (parsePositiveIntEnv floor) + const m = makePool(); + const now = Date.now(); + // priority>=8 (foreground), run_after far in the future, created_at old enough to cross the 1m ceiling. + m.setForegroundLivenessCandidates([{ id: "fg-1", created_at: now - 61_000 }]); + const q = createPgQueue(m.pool, async () => undefined); + + const released = await q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(1); + expect(m.fn).toHaveBeenCalledWith( + expect.stringContaining("SELECT id, payload, created_at FROM _selfhost_jobs WHERE status='pending' AND priority>=$1 AND run_after>$2"), + expect.arrayContaining([8]), + ); + expect(m.fn).toHaveBeenCalledWith( + expect.stringContaining("SET run_after=$1 WHERE id=$2 AND status='pending' AND run_after>$1"), + expect.arrayContaining(["fg-1"]), + ); + expect(await renderMetrics()).toContain("gittensory_jobs_foreground_liveness_released_total 1"); + }); + + // Isolates the AGE condition from the OR'd rate-limit-clear condition: the default candidate payload + // (recapture-preview) is always rate-limit-"clear" (see setForegroundLivenessCandidates' own doc comment), + // so this test uses a github-webhook payload WITH a genuinely exhausted, still-future-reset observation for + // its admission key, ensuring isRateLimitAdmissionNowClear() returns false and only the age check governs. + it("does NOT release a foreground row whose created_at is still recent (not yet stale) AND is still genuinely rate-limited", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "600000"; // default 10m + const m = makePool(); + const now = Date.now(); + m.setRateLimitRows([ + { + admission_key: "installation:123", + remaining: 1, + reset_at: new Date(now + 30 * 60_000).toISOString(), + observed_at: new Date(now).toISOString(), + }, + ]); + const payload = JSON.stringify({ + type: "github-webhook", + deliveryId: "still-blocked", + eventName: "x", + payload: { installation: { id: 123 } }, + }); + // Same shape as the stale case (foreground priority, future run_after) but created_at is only 1s old. + m.setForegroundLivenessCandidates([{ id: "fg-fresh", created_at: now - 1_000, payload }]); + const q = createPgQueue(m.pool, async () => undefined); + + const released = await q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(0); + expect(m.fn).not.toHaveBeenCalledWith( + expect.stringContaining("SET run_after=$1 WHERE id=$2 AND status='pending' AND run_after>$1"), + expect.arrayContaining(["fg-fresh"]), + ); + expect(await renderMetrics()).not.toContain("gittensory_jobs_foreground_liveness_released_total"); + }); + + // CONDITION-BASED recovery (the second OR arm): a foreground job whose created_at is nowhere near stale but + // whose rate-limit observation has since cleared (no blocking observation seeded here) is released anyway -- + // the whole point of pairing the age floor with a rate-limit-aware re-check (see the source's own doc + // comment on releaseStaleForegroundDeferrals). + it("releases a foreground row that is NOT yet age-stale once rate-limit admission for it reads clear", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "600000"; // default 10m -- nowhere near stale by age + const m = makePool(); + const now = Date.now(); + const payload = JSON.stringify({ + type: "github-webhook", + deliveryId: "now-clear", + eventName: "x", + payload: { installation: { id: 123 } }, + }); + // No rate-limit rows seeded at all -- rateLimitAdmissionDelayMs degrades to "clear". + m.setForegroundLivenessCandidates([{ id: "fg-clear", created_at: now - 1_000, payload }]); + const q = createPgQueue(m.pool, async () => undefined); + + const released = await q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(1); + expect(await renderMetrics()).toContain("gittensory_jobs_foreground_liveness_released_total 1"); + }); + + // The payload is unparseable -- isRateLimitAdmissionNowClear's own catch(){ return false } branch -- so ONLY + // the age condition can release it; while young, it must stay parked exactly like any other not-yet-stale, + // still-blocked row. + it("does NOT release a foreground row with an unparseable payload before it is age-stale", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "600000"; + const m = makePool(); + const now = Date.now(); + m.setForegroundLivenessCandidates([{ id: "fg-bad-json", created_at: now - 1_000, payload: "not valid json" }]); + const q = createPgQueue(m.pool, async () => undefined); + + const released = await q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(0); + }); + + // The priority>=$1 filter is enforced by the candidate SELECT's own WHERE clause (bound to + // FOREGROUND_QUEUE_PRIORITY_FLOOR=8), not by any application-side check on the returned rows -- mirrors how + // the maintenance-admission pressure tests in this file assert the is_maintenance=1 predicate is present in + // the issued SQL rather than re-deriving it from mock row shapes. Asserting the bind param here is the + // faithful way to prove a background-priority (<8) row is structurally excluded from ever being considered. + it("scopes the candidate SELECT to foreground priority via the FOREGROUND_QUEUE_PRIORITY_FLOOR bind param", async () => { + const m = makePool(); + m.setForegroundLivenessCandidates([]); // the real WHERE priority>=8 excludes a background row; assert the bind param enforces it + const q = createPgQueue(m.pool, async () => undefined); + + const released = await q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(0); + expect(m.fn).toHaveBeenCalledWith( + expect.stringContaining("priority>=$1"), + expect.arrayContaining([8]), + ); + }); + + it("returns 0 immediately without issuing the candidate SELECT when FOREGROUND_LIVENESS_ENABLED=false", async () => { + process.env.FOREGROUND_LIVENESS_ENABLED = "false"; + const m = makePool(); + const now = Date.now(); + m.setForegroundLivenessCandidates([{ id: "fg-1", created_at: now - 10 * 60_000 }]); + const q = createPgQueue(m.pool, async () => undefined); + + const released = await q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(0); + expect(m.fn).not.toHaveBeenCalledWith( + expect.stringContaining("SELECT id, payload, created_at FROM _selfhost_jobs WHERE status='pending' AND priority>=$1 AND run_after>$2"), + expect.anything(), + ); + expect(await renderMetrics()).not.toContain("gittensory_jobs_foreground_liveness_released_total"); + }); + + // REGRESSION (#selfhost-queue-liveness): the production incident this module exists to make structurally + // impossible -- a GitHub rate-limit sweep pushes MANY foreground-priority jobs' run_after far into the + // future at once (a shared REST budget drained by a post-deploy catch-up burst), and without this release + // path they'd sit deferred for up to the ~65-minute worst-case rate-limit window with zero runnable work, + // requiring manual intervention. Assert releaseStaleForegroundDeferrals() releases ALL stale rows in ONE + // sweep (not just the first) and records the metric as a SINGLE aggregate increment, not one per row -- + // matching the source's own "logs + records a metric ONCE per sweep (aggregate count), not per row" doc + // comment, which exists specifically so a large release batch cannot spam the log/metric. + it("releases every stale foreground deferral in one sweep and records one aggregate metric increment (regression for #selfhost-queue-liveness)", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "60000"; // 1m floor + const m = makePool(); + const now = Date.now(); + const staleAge = now - 5 * 60_000; // 5m old -- well past the 1m ceiling + m.setForegroundLivenessCandidates([ + { id: "stuck-1", created_at: staleAge }, + { id: "stuck-2", created_at: staleAge }, + { id: "stuck-3", created_at: staleAge }, + ]); + const q = createPgQueue(m.pool, async () => undefined); + + const released = await q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(3); + for (const id of ["stuck-1", "stuck-2", "stuck-3"]) { + expect(m.fn).toHaveBeenCalledWith( + expect.stringContaining("SET run_after=$1 WHERE id=$2 AND status='pending' AND run_after>$1"), + expect.arrayContaining([id]), + ); + } + // Exactly one aggregate increment of 3, not three separate increments of 1. + expect(await renderMetrics()).toContain("gittensory_jobs_foreground_liveness_released_total 3"); + }); + + // A stale candidate can lose the UPDATE race (another instance/tick already moved it) -- mirrors + // reviveDeadLetterJobs' own "AND status='dead'" re-check pattern: only rows whose UPDATE actually matched + // (rowCount 1) count toward the release total, never the raw SELECT candidate count. + it("counts only rows whose conditional UPDATE actually matched, not the raw candidate count", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "60000"; + const m = makePool(); + const now = Date.now(); + const staleAge = now - 5 * 60_000; + m.setForegroundLivenessCandidates( + [ + { id: "won-race", created_at: staleAge }, + { id: "lost-race", created_at: staleAge }, + ], + [1, 0], // second row's UPDATE matches zero rows -- already released/claimed by someone else + ); + const q = createPgQueue(m.pool, async () => undefined); + + const released = await q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(1); + }); + + it("handles a null rowCount from the release UPDATE (rowCount ?? 0 nullish arm)", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "60000"; + const now = Date.now(); + // pg's driver can report a null rowCount for some UPDATEs; the release count must tolerate it rather + // than propagate NaN (mirrors init()'s own "handles null rowCount from the recovery query" test). + const fn = vi.fn().mockImplementation(async (sql: unknown) => { + const q = String(sql); + if (q.includes("SELECT id, payload, created_at FROM") && q.includes("priority>=$1 AND run_after>$2")) { + return { rows: [{ id: "fg-null", created_at: now - 5 * 60_000 }], rowCount: 1 }; + } + if (q.includes("SET run_after=$1 WHERE id=$2 AND status='pending' AND run_after>$1")) { + return { rows: [], rowCount: null }; + } + return { rows: [], rowCount: 0 }; + }); + const q = createPgQueue({ query: fn } as unknown as Pool, async () => undefined); + + const released = await q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(0); // null ?? 0 -- no metric recorded, no crash + expect(await renderMetrics()).not.toContain("gittensory_jobs_foreground_liveness_released_total"); + }); + }); + + describe("processingCount (#selfhost-queue-liveness)", () => { + it("returns the count of status='processing' jobs", async () => { + const { pool } = makePool(); + // makePool returns { c: "3" } for any COUNT(*) query, including WHERE status='processing'. + const q = createPgQueue(pool, async () => undefined); + expect(await q.processingCount()).toBe(3); + }); + }); + it("reschedules GitHub rate-limit failures without consuming the dead-letter budget", async () => { const m = makePool(); m.enqueueJob("1", { type: "github-webhook" }, 4); @@ -2289,10 +2577,39 @@ describe("createPgQueue (durable #977)", () => { expect(signals).toEqual({ livePendingCount: 2, oldestLivePendingAgeMs: expect.any(Number), + liveRunnableNowCount: 0, + oldestLiveRunnableAgeMs: null, maintenancePendingCount: 4, oldestMaintenancePendingAgeMs: expect.any(Number), hostLoadAvg1PerCore: null, }); }); + + // #selfhost-queue-liveness: liveRunnableNowCount/oldestLiveRunnableAgeMs must reflect only the SUBSET of + // live pending jobs that are currently DUE (run_after<=now) -- distinct from livePendingCount/ + // oldestLivePendingAgeMs, which are dominated by whatever row is OLDEST by created_at regardless of + // whether it is runnable right now. A stale/deferred oldest row must not mask a younger due row. + it("pressureSignals() reports the runnable-now subset distinctly from the overall oldest-pending age", async () => { + const m = makePool(); + m.setPressureSignals({ + live: { cnt: 3, oldest: now - 500_000, runnableCnt: 1, oldestRunnable: now - 10_000 }, + }); + const q = createPgQueue(m.pool, async () => undefined); + const signals = await q.pressureSignals(); + expect(signals.livePendingCount).toBe(3); + expect(signals.oldestLivePendingAgeMs).toBeGreaterThanOrEqual(500_000); + expect(signals.liveRunnableNowCount).toBe(1); + expect(signals.oldestLiveRunnableAgeMs).toBeGreaterThanOrEqual(10_000); + expect(signals.oldestLiveRunnableAgeMs).toBeLessThan(signals.oldestLivePendingAgeMs as number); + }); + + it("pressureSignals() reports zero runnable-now and a null oldest-runnable age when every live row is deferred to the future", async () => { + const m = makePool(); + m.setPressureSignals({ live: { cnt: 5, oldest: now - 100_000, runnableCnt: 0, oldestRunnable: null } }); + const q = createPgQueue(m.pool, async () => undefined); + const signals = await q.pressureSignals(); + expect(signals.liveRunnableNowCount).toBe(0); + expect(signals.oldestLiveRunnableAgeMs).toBeNull(); + }); }); }); diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index e4319350de..bbc39a73ac 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -15,6 +15,7 @@ import { githubRateLimitRetryDelayMs, githubWebhookRateLimitDelayMs, isGitHubBudgetBackgroundJob, + isScheduledRegateSweepJob, isForegroundJobPriority, jobCoalesceAbsorbedByKey, jobCoalesceKey, @@ -134,6 +135,30 @@ describe("self-host queue common helpers", () => { expect(isGitHubBudgetBackgroundJob({ type: "refresh-installation-health", requestedBy: "schedule" })).toBe(false); }); + describe("isScheduledRegateSweepJob", () => { + it("returns true only for the scheduled sweep fan-out's own delivery id prefix", () => { + expect(isScheduledRegateSweepJob("regate-sweep:JSONbored/gittensory#42")).toBe(true); + }); + + it("returns false for the outage-repair priority prefix (a live-PR reconciliation, not stale/scheduled)", () => { + expect(isScheduledRegateSweepJob("regate-repair:JSONbored/gittensory#42")).toBe(false); + }); + + it("returns false for a real GitHub webhook delivery id", () => { + expect(isScheduledRegateSweepJob("a1b2c3d4-e5f6-7890-abcd-ef1234567890")).toBe(false); + }); + + it("returns false for the explicit manual-regate operator override prefix", () => { + expect(isScheduledRegateSweepJob("manual-regate:JSONbored/gittensory#42")).toBe(false); + }); + + it("returns false for undefined, null, and empty-string deliveryId", () => { + expect(isScheduledRegateSweepJob(undefined)).toBe(false); + expect(isScheduledRegateSweepJob(null)).toBe(false); + expect(isScheduledRegateSweepJob("")).toBe(false); + }); + }); + it("normalizes GitHub rate-limit metric labels without leaking raw admission keys", () => { const webhookJob = { type: "github-webhook", @@ -192,7 +217,7 @@ describe("self-host queue common helpers", () => { kind: "background", admissionKey: githubRateLimitAdmissionKeyForPublicToken(), }); - expect(githubRateLimitAdmissionTargetForJob({ type: "agent-regate-pr", deliveryId: "sweep:owner/repo#1", repoFullName: "owner/repo", prNumber: 1, installationId: 123 })).toEqual({ + expect(githubRateLimitAdmissionTargetForJob({ type: "agent-regate-pr", deliveryId: "regate-sweep:owner/repo#1", repoFullName: "owner/repo", prNumber: 1, installationId: 123 })).toEqual({ kind: "background", admissionKey: "installation:123", }); @@ -202,6 +227,119 @@ describe("self-host queue common helpers", () => { }); }); + describe("githubRateLimitAdmissionTargetForJob: agent-regate-pr classification (#selfhost-queue-liveness)", () => { + it("REGRESSION: manual-regate operator override stays a full admission bypass (null) — unchanged by the fix", () => { + expect( + githubRateLimitAdmissionTargetForJob({ + type: "agent-regate-pr", + deliveryId: "manual-regate:repo#1", + repoFullName: "owner/repo", + prNumber: 1, + installationId: 123, + }), + ).toBeNull(); + }); + + it("keeps the scheduled sweep fan-out on the conservative background floor — unchanged by the fix", () => { + expect( + githubRateLimitAdmissionTargetForJob({ + type: "agent-regate-pr", + deliveryId: "regate-sweep:repo#1", + repoFullName: "owner/repo", + prNumber: 1, + installationId: 123, + }), + ).toEqual({ + kind: "background", + admissionKey: "installation:123", + }); + }); + + it("FIX: an outage-repair priority candidate now gets webhook-level admission, not the maintenance floor", () => { + expect( + githubRateLimitAdmissionTargetForJob({ + type: "agent-regate-pr", + deliveryId: "regate-repair:repo#1", + repoFullName: "owner/repo", + prNumber: 1, + installationId: 123, + }), + ).toEqual({ + kind: "webhook", + admissionKey: "installation:123", + }); + }); + + it("FIX: a real webhook-originated delivery id (trailing re-review / sibling wake / linked-issue re-review) gets webhook-level admission", () => { + expect( + githubRateLimitAdmissionTargetForJob({ + type: "agent-regate-pr", + deliveryId: "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + repoFullName: "owner/repo", + prNumber: 1, + installationId: 123, + }), + ).toEqual({ + kind: "webhook", + admissionKey: "installation:123", + }); + }); + + it("FIX: the webhook-level admissionKey falls back to the public-token key when no installationId resolves", () => { + // installationId is a required, non-optional `number` on the agent-regate-pr message shape, so the + // only way to reach githubRateLimitAdmissionKeyForJob's null result here (without violating the + // type) is a non-finite numeric value -- exactly what Number.isFinite in that helper rejects. + expect( + githubRateLimitAdmissionTargetForJob({ + type: "agent-regate-pr", + deliveryId: "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + repoFullName: "owner/repo", + prNumber: 1, + installationId: Number.NaN, + }), + ).toEqual({ + kind: "webhook", + admissionKey: githubRateLimitAdmissionKeyForPublicToken(), + }); + }); + + it("leaves github-webhook jobs on their own first branch, not the new agent-regate-pr branch", () => { + expect( + githubRateLimitAdmissionTargetForJob({ + type: "github-webhook", + deliveryId: "d3", + eventName: "pull_request", + payload: { installation: { id: 123 } }, + }), + ).toEqual({ + kind: "webhook", + admissionKey: "installation:123", + }); + }); + + it("leaves other GITHUB_BUDGET_BACKGROUND_TYPES jobs on the background floor -- the new branch is guarded to agent-regate-pr only", () => { + expect( + githubRateLimitAdmissionTargetForJob({ + type: "rag-index-repo", + repoFullName: "owner/repo", + requestedBy: "schedule", + } as JobMessage), + ).toEqual({ + kind: "background", + admissionKey: githubRateLimitAdmissionKeyForPublicToken(), + }); + expect( + githubRateLimitAdmissionTargetForJob({ + type: "backfill-registered-repos", + requestedBy: "schedule", + } as JobMessage), + ).toEqual({ + kind: "background", + admissionKey: githubRateLimitAdmissionKeyForPublicToken(), + }); + }); + }); + it("computes background admission delays from persisted GitHub REST observations", () => { const now = Date.parse("2026-06-24T12:00:00.000Z"); expect(githubBackgroundRateLimitDelayMs(null, now)).toBeNull(); @@ -412,6 +550,38 @@ describe("self-host queue common helpers", () => { ), ).toBeNull(); }); + + // The production VPS incident (#selfhost-queue-liveness) specifically involved mixed-freshness + // observations for the SAME admission key (not an exact-vs-fallback mismatch, which the tests + // above already cover) -- these two pin that newerRateLimitObservation's own same-key comparison + // picks the observation with the later observed_at correctly, regardless of array order. + it("REGRESSION: an older exact-key exhausted observation is superseded by a newer exact-key healthy observation for the SAME key", () => { + expect( + githubRateLimitAdmissionDelayMs( + "webhook", + key, + [ + { admission_key: key, remaining: 0, reset_at: "2026-06-24T12:10:00.000Z", observed_at: "2026-06-24T11:59:00.000Z" }, + { admission_key: key, remaining: 4000, reset_at: "2026-06-24T12:20:00.000Z", observed_at: "2026-06-24T12:00:00.000Z" }, + ], + now, + ), + ).toBeNull(); + }); + + it("REGRESSION: a newer exact-key exhausted observation supersedes an older exact-key healthy observation for the SAME key", () => { + expect( + githubRateLimitAdmissionDelayMs( + "webhook", + key, + [ + { admission_key: key, remaining: 4000, reset_at: "2026-06-24T12:20:00.000Z", observed_at: "2026-06-24T11:59:00.000Z" }, + { admission_key: key, remaining: 0, reset_at: "2026-06-24T12:10:00.000Z", observed_at: "2026-06-24T12:00:00.000Z" }, + ], + now, + ), + ).toBe(615_000); + }); }); describe("matchesGitHubRateLimitAdmissionTarget", () => { diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 4e6d396ef7..ea3e34043c 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -210,7 +210,7 @@ describe("createSqliteQueue (durable #980)", () => { const seen: string[] = []; const q = createSqliteQueue(driver, async (m) => void seen.push(typeOf(m))); - await q.binding.send({ type: "agent-regate-pr", deliveryId: "sweep:owner/repo#7", repoFullName: "owner/repo", prNumber: 7, installationId: 123 }); + await q.binding.send({ type: "agent-regate-pr", deliveryId: "regate-sweep:owner/repo#7", repoFullName: "owner/repo", prNumber: 7, installationId: 123 }); await q.binding.send({ type: "rag-index-repo", requestedBy: "schedule", repoFullName: "owner/repo" }); await q.binding.send({ type: "github-webhook", deliveryId: "fresh", eventName: "pull_request", payload: {} }); await q.drain(); @@ -1668,6 +1668,288 @@ describe("createSqliteQueue (durable #980)", () => { }); }); + describe("releaseStaleForegroundDeferrals (#selfhost-queue-liveness)", () => { + afterEach(() => { + delete process.env.FOREGROUND_LIVENESS_ENABLED; + delete process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS; + }); + + /** Directly inserts a foreground-priority (>=8 by default) pending row with an explicit created_at/run_after, + * bypassing enqueue()'s own jitter/coalescing so the row's age and deferral are fully deterministic. Uses + * "recapture-preview" by default -- a foreground type NOT in GITHUB_BUDGET_BACKGROUND_TYPES and not + * "github-webhook"/"agent-regate-pr", so githubRateLimitAdmissionTargetForJob returns null for it and the + * rate-limit-clear condition (isRateLimitAdmissionNowClear) is trivially/always true for these rows -- + * isolating the AGE-based condition cleanly for tests that aren't specifically about rate-limit clearing. */ + function seedForegroundPendingRow( + driver: ReturnType, + opts: { createdAt: number; runAfter: number; priority?: number; type?: string }, + ): void { + driver.query( + `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) + VALUES (?, 'pending', 0, ?, ?, ?, NULL, 0)`, + [ + JSON.stringify({ type: opts.type ?? "recapture-preview", deliveryId: `seed:${opts.createdAt}`, repoFullName: "o/r", prNumber: 1, attempt: 1 }), + opts.runAfter, + opts.createdAt, + opts.priority ?? 9, + ], + ); + } + + /** Creates github_rate_limit_observations (mirrors the existing "pre-yields ..." tests' inline DDL) and seeds + * ONE exhausted observation for the given admission key, so isRateLimitAdmissionNowClear() genuinely returns + * false for a job routed to that key -- letting a test isolate the AGE-only release condition even for a + * rate-limit-tracked job type (github-webhook / agent-regate-pr). */ + function seedExhaustedRateLimitObservation( + driver: ReturnType, + admissionKey: string, + resetAtIso: string, + ): void { + driver.query( + `CREATE TABLE IF NOT EXISTS github_rate_limit_observations ( + id TEXT PRIMARY KEY, + repo_full_name TEXT NOT NULL, + admission_key TEXT, + resource TEXT NOT NULL, + path TEXT NOT NULL, + status_code INTEGER NOT NULL, + limit_value INTEGER, + remaining INTEGER, + reset_at TEXT, + observed_at TEXT NOT NULL + )`, + [], + ); + driver.query( + `INSERT INTO github_rate_limit_observations (id, repo_full_name, admission_key, resource, path, status_code, limit_value, remaining, reset_at, observed_at) + VALUES (?, 'o/r', ?, 'rest', '/x', 200, 5000, 1, ?, ?)`, + [`rl-${admissionKey}`, admissionKey, resetAtIso, new Date().toISOString()], + ); + } + + it("releases a foreground-priority pending row deferred far into the future once its created_at is stale", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "60000"; // 1m (parsePositiveIntEnv floor) + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + const now = Date.now(); + seedForegroundPendingRow(driver, { createdAt: now - 5 * 60_000, runAfter: now + 60 * 60_000 }); + + const released = q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(1); + const row = driver.query("SELECT run_after FROM _selfhost_jobs", []).rows[0] as { run_after: number }; + expect(row.run_after).toBeLessThanOrEqual(Date.now()); + expect(await renderMetrics()).toContain("gittensory_jobs_foreground_liveness_released_total 1"); + }); + + // Isolates the AGE condition from the OR'd rate-limit-clear condition: uses a github-webhook row (which IS + // rate-limit-tracked) with a genuinely exhausted, still-future-reset observation for its admission key, so + // isRateLimitAdmissionNowClear() returns false and only the age check governs release. + it("does NOT release a foreground row whose created_at is still recent (not yet stale) AND is still genuinely rate-limited", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "600000"; // default 10m + const driver = makeDriver(); + const now = Date.now(); + // Keyed to installation:123, matching the job's own payload.installation.id below -- githubRateLimitAdmissionKeyForJob + // only resolves an admission key for a github-webhook job from payload.installation.id (an unkeyed webhook + // payload resolves to a null admissionKey, which this exact/fallback-keyed observation would NOT match). + seedExhaustedRateLimitObservation(driver, "installation:123", new Date(now + 30 * 60_000).toISOString()); + const q = createSqliteQueue(driver, async () => undefined); + const futureRunAfter = now + 60 * 60_000; + driver.query( + `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) + VALUES (?, 'pending', 0, ?, ?, 10, NULL, 0)`, + [ + JSON.stringify({ type: "github-webhook", deliveryId: "still-blocked", eventName: "x", payload: { installation: { id: 123 } } }), + futureRunAfter, + now - 1_000, + ], + ); + + const released = q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(0); + const row = driver.query("SELECT run_after FROM _selfhost_jobs", []).rows[0] as { run_after: number }; + expect(row.run_after).toBe(futureRunAfter); + expect(await renderMetrics()).not.toContain("gittensory_jobs_foreground_liveness_released_total"); + }); + + // CONDITION-BASED recovery (the second OR arm): a foreground job whose created_at is nowhere near stale but + // whose rate-limit observation has since cleared (no blocking observation at all here) is released anyway -- + // this is the whole point of pairing the age floor with a rate-limit-aware re-check (see the source's own + // doc comment on releaseStaleForegroundDeferrals). + it("releases a foreground row that is NOT yet age-stale once rate-limit admission for it reads clear", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "600000"; // default 10m -- nowhere near stale by age + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + const now = Date.now(); + const futureRunAfter = now + 60 * 60_000; + // No github_rate_limit_observations table/row at all -- rateLimitAdmissionDelayMs degrades to "clear". + driver.query( + `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) + VALUES (?, 'pending', 0, ?, ?, 10, NULL, 0)`, + [JSON.stringify({ type: "github-webhook", deliveryId: "now-clear", eventName: "x", payload: {} }), futureRunAfter, now - 1_000], + ); + + const released = q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(1); + expect(await renderMetrics()).toContain("gittensory_jobs_foreground_liveness_released_total 1"); + }); + + // The payload is unparseable -- isRateLimitAdmissionNowClear's own catch(){ return false } branch -- so ONLY + // the age condition can release it; while young, it must stay parked exactly like any other not-yet-stale, + // still-blocked row. + it("does NOT release a foreground row with an unparseable payload before it is age-stale", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "600000"; + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + const now = Date.now(); + const futureRunAfter = now + 60 * 60_000; + driver.query( + `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) + VALUES (?, 'pending', 0, ?, ?, 10, NULL, 0)`, + ["not valid json", futureRunAfter, now - 1_000], + ); + + const released = q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(0); + }); + + // The priority>=? filter is enforced by the candidate SELECT's own WHERE clause (bound to + // FOREGROUND_QUEUE_PRIORITY_FLOOR=8), not by any application-side check on the returned rows -- mirrors how + // the maintenance-admission pressure tests in this file seed real rows and rely on the actual SQL predicate + // rather than re-deriving it. A background-priority (<8) row, however old and however far-future its + // run_after, must never be touched by this sweep. + it("does NOT release a BACKGROUND-priority row even with an old created_at and future run_after", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "60000"; + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + const now = Date.now(); + const futureRunAfter = now + 60 * 60_000; + seedForegroundPendingRow(driver, { + createdAt: now - 5 * 60_000, + runAfter: futureRunAfter, + priority: 0, // background -- below FOREGROUND_QUEUE_PRIORITY_FLOOR (8) + type: "build-contributor-evidence", + }); + + const released = q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(0); + const row = driver.query("SELECT run_after FROM _selfhost_jobs", []).rows[0] as { run_after: number }; + expect(row.run_after).toBe(futureRunAfter); // untouched + }); + + it("returns 0 immediately without releasing anything when FOREGROUND_LIVENESS_ENABLED=false", async () => { + process.env.FOREGROUND_LIVENESS_ENABLED = "false"; + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); // creates the table + const now = Date.now(); + seedForegroundPendingRow(driver, { createdAt: now - 60 * 60_000, runAfter: now + 60 * 60_000 }); + + const released = q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(0); + const row = driver.query("SELECT run_after FROM _selfhost_jobs", []).rows[0] as { run_after: number }; + expect(row.run_after).toBeGreaterThan(Date.now()); // still deferred -- the escape hatch never touched it + expect(await renderMetrics()).not.toContain("gittensory_jobs_foreground_liveness_released_total"); + }); + + // REGRESSION (#selfhost-queue-liveness): the production incident this module exists to make structurally + // impossible -- a GitHub rate-limit sweep pushes MANY foreground-priority jobs' run_after far into the + // future at once (a shared REST budget drained by a post-deploy catch-up burst), and without this release + // path they'd sit deferred for up to the ~65-minute worst-case rate-limit window with zero runnable work, + // requiring manual intervention. Assert releaseStaleForegroundDeferrals() releases ALL stale rows in ONE + // sweep (not just the first) and records the metric as a SINGLE aggregate increment, not one per row -- + // matching the source's own "logs + records a metric ONCE per sweep (aggregate count), not per row" doc + // comment, which exists specifically so a large release batch cannot spam the log/metric. Also asserts that + // kickAll()-driven pump activity actually finds the released rows runnable afterward. + it("releases every stale foreground deferral in one sweep and records one aggregate metric increment (regression for #selfhost-queue-liveness)", async () => { + process.env.FOREGROUND_LIVENESS_MAX_DEFER_MS = "60000"; // 1m floor + const driver = makeDriver(); + const started: string[] = []; + const q = createSqliteQueue(driver, async (m) => void started.push(typeOf(m))); + const now = Date.now(); + const staleAge = now - 5 * 60_000; // 5m old -- well past the 1m ceiling + const farFuture = now + 60 * 60_000; + for (let i = 0; i < 3; i += 1) { + seedForegroundPendingRow(driver, { createdAt: staleAge, runAfter: farFuture }); + } + + const released = q.releaseStaleForegroundDeferrals(); + + expect(released).toBe(3); + expect(await renderMetrics()).toContain("gittensory_jobs_foreground_liveness_released_total 3"); + // kickAll() (called internally once released > 0) means pump activity picks these up without waiting for + // the next poll tick -- drain() confirms all three are now genuinely runnable. + await q.drain(); + expect(started.length).toBe(3); + }); + + // Mirrors reviveDeadLetterJobsSafely's own regression test: the foreground-liveness interval had no error + // handler of its own, so a thrown driver/metric failure on that tick would surface as an uncaught exception + // and could terminate the process -- exactly the failure mode pump()'s own try/catch already guards against + // for the main poll loop. + it("survives a releaseStaleForegroundDeferrals() driver failure on the interval tick instead of crashing the process", async () => { + process.env.FOREGROUND_LIVENESS_CHECK_INTERVAL_MS = "5000"; // parsePositiveIntEnv floor + vi.useFakeTimers(); + try { + const driver = makeDriver(); + // Let the boot-time self-heal call (inside createSqliteQueue's own constructor, unguarded) run against + // the real driver first -- only start throwing on the candidate SELECT once the queue is fully + // constructed, so this test isolates the INTERVAL tick's own error handling rather than a constructor-time + // crash (a distinct concern, already covered by the TDZ-crash tests elsewhere in this describe block). + let armed = false; + const realQuery = driver.query.bind(driver); + vi.spyOn(driver, "query").mockImplementation((sql: string, params: unknown[]) => { + if (armed && sql.includes("SELECT id, payload, created_at FROM") && sql.includes("priority>=? AND run_after>?")) { + throw new Error("disk I/O error"); + } + return realQuery(sql, params); + }); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const q = createSqliteQueue(driver, async () => undefined); + armed = true; + + q.start(); + await vi.advanceTimersByTimeAsync(5000); // the foreground-liveness interval fires once -- would throw here if uncaught + + const logged = errorSpy.mock.calls.map(([line]) => String(line)); + expect( + logged.some( + (line) => line.includes("selfhost_queue_foreground_liveness_release_crashed") && line.includes("disk I/O error"), + ), + ).toBe(true); + await q.stop(); + } finally { + delete process.env.FOREGROUND_LIVENESS_CHECK_INTERVAL_MS; + } + }); + }); + + describe("processingCount (#selfhost-queue-liveness)", () => { + it("returns the count of status='processing' jobs", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'processing', 0, 0, 0, 0)", + [JSON.stringify(msg("x"))], + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, 0, 0, 0)", + [JSON.stringify(msg("y"))], + ); + expect(q.processingCount()).toBe(1); + }); + + it("returns 0 when no job is processing", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + expect(q.processingCount()).toBe(0); + }); + }); + it("reschedules GitHub rate-limit failures without consuming the dead-letter budget", async () => { const driver = makeDriver(); let calls = 0; @@ -2647,6 +2929,60 @@ describe("createSqliteQueue (durable #980)", () => { expect(signals.oldestLivePendingAgeMs).toBeNull(); expect(signals.maintenancePendingCount).toBe(0); expect(signals.oldestMaintenancePendingAgeMs).toBeNull(); + // Zero foreground rows at all -- SQLite's SUM(CASE...)/MIN(CASE...) return NULL (not 0) over a zero-row + // aggregate group, exercising the `?? 0` nullish arm on runnable_cnt (see maintenancePressureSignals). + expect(signals.liveRunnableNowCount).toBe(0); + expect(signals.oldestLiveRunnableAgeMs).toBeNull(); + }); + + // #selfhost-queue-liveness: liveRunnableNowCount/oldestLiveRunnableAgeMs must reflect only the SUBSET of + // live pending jobs that are currently DUE (run_after<=now) -- distinct from livePendingCount/ + // oldestLivePendingAgeMs, which are dominated by whatever row is OLDEST by created_at regardless of + // whether it is runnable right now. Constructed so the OLDEST-by-created_at foreground row is NOT yet due + // (a large future run_after) while a NEWER foreground row IS due -- the oldest-runnable age must come from + // the newer, due row, not the older, not-due one. + it("pressureSignals() reports the runnable-now subset distinctly from the overall oldest-pending age", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + const now = Date.now(); + // Oldest by created_at, but deferred far into the future -- NOT runnable now. + driver.query( + `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) + VALUES (?, 'pending', 0, ?, ?, 9, NULL, 0)`, + [JSON.stringify(msg("agent-regate-pr")), now + 3_600_000, now - 500_000], + ); + // Newer by created_at, but already due -- runnable right now. + driver.query( + `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) + VALUES (?, 'pending', 0, ?, ?, 9, NULL, 0)`, + [JSON.stringify(msg("agent-regate-pr")), now - 1_000, now - 10_000], + ); + const signals = q.pressureSignals(); + expect(signals.livePendingCount).toBe(2); + expect(signals.oldestLivePendingAgeMs).toBeGreaterThanOrEqual(500_000); + expect(signals.liveRunnableNowCount).toBe(1); + expect(signals.oldestLiveRunnableAgeMs).toBeGreaterThanOrEqual(10_000); + expect(signals.oldestLiveRunnableAgeMs).toBeLessThan(signals.oldestLivePendingAgeMs as number); + }); + + it("pressureSignals() reports zero runnable-now with a null oldest-runnable age when foreground jobs exist but none are due yet", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + const now = Date.now(); + // Foreground rows exist (outer WHERE matches), but every one is deferred to the future -- the inner CASE + // never matches, so SUM(CASE...) is a real 0 here (a set exists, just none due), a DIFFERENT code path + // from the "zero foreground rows at all" NULL-aggregate case covered above. + for (let i = 0; i < 3; i += 1) { + driver.query( + `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key, is_maintenance) + VALUES (?, 'pending', 0, ?, ?, 9, NULL, 0)`, + [JSON.stringify(msg("agent-regate-pr")), now + 3_600_000, now - 60_000], + ); + } + const signals = q.pressureSignals(); + expect(signals.livePendingCount).toBe(3); + expect(signals.liveRunnableNowCount).toBe(0); + expect(signals.oldestLiveRunnableAgeMs).toBeNull(); }); it("backfills the is_maintenance flag on startup for jobs enqueued by an older version", async () => {