Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .loopover.yml.example
Original file line number Diff line number Diff line change
Expand Up @@ -1276,13 +1276,15 @@ settings:
# upstreamDriftIssues:
# enabled: true # Bool. Default: false (the env var decides instead).

# Fleet-wide sweep-liveness watchdog cron (#6558 / #6275): config-as-code override for the
# Fleet-wide sweep-liveness watchdog cron (#6558 / #6275 / #6594): config-as-code override for the
# LOOPOVER_SWEEP_WATCHDOG flag that gates the hourly sweep-liveness check. Operator-level, not per-repo --
# only meaningful on the loopover self-repo's own manifest; a present block there wins over the env var.
# Distinct from the per-repo FORCE-OFF under `review.sweepWatchdog` (which only excludes one repo from the
# scan set once the fleet gate is ON).
# sweepWatchdog:
# enabled: true # Bool. Default: false (the env var decides instead).
# enabled: true # Bool. Default: false (the env var decides instead).
# staleAfterMinutes: 90 # Optional positive int (#6594). Minutes without a sweep-marker advance before
# # a repo with open PRs is treated as stale. Omit ⇒ hardcoded 45-minute default.

# Fleet-wide open-PR reconciliation cron (#6558 / #6275): config-as-code override for the
# LOOPOVER_PR_RECONCILIATION flag that gates the fast open-PR reconciliation pass. Same shape and
Expand Down
6 changes: 4 additions & 2 deletions config/examples/loopover.full.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1290,13 +1290,15 @@ settings:
# upstreamDriftIssues:
# enabled: true # Bool. Default: false (the env var decides instead).

# Fleet-wide sweep-liveness watchdog cron (#6558 / #6275): config-as-code override for the
# Fleet-wide sweep-liveness watchdog cron (#6558 / #6275 / #6594): config-as-code override for the
# LOOPOVER_SWEEP_WATCHDOG flag that gates the hourly sweep-liveness check. Operator-level, not per-repo --
# only meaningful on the loopover self-repo's own manifest; a present block there wins over the env var.
# Distinct from the per-repo FORCE-OFF under `review.sweepWatchdog` (which only excludes one repo from the
# scan set once the fleet gate is ON).
# sweepWatchdog:
# enabled: true # Bool. Default: false (the env var decides instead).
# enabled: true # Bool. Default: false (the env var decides instead).
# staleAfterMinutes: 90 # Optional positive int (#6594). Minutes without a sweep-marker advance before
# # a repo with open PRs is treated as stale. Omit ⇒ hardcoded 45-minute default.

# Fleet-wide open-PR reconciliation cron (#6558 / #6275): config-as-code override for the
# LOOPOVER_PR_RECONCILIATION flag that gates the fast open-PR reconciliation pass. Same shape and
Expand Down
19 changes: 15 additions & 4 deletions packages/loopover-engine/src/focus-manifest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,8 @@ export type FocusManifestUpstreamDriftIssuesConfig = {
export type FocusManifestSweepWatchdogConfig = {
present: boolean;
enabled: boolean;
/** Optional staleness window in minutes (#6594). null ⇒ caller keeps the hardcoded 45-minute default. */
staleAfterMinutes: number | null;
};

/**
Expand Down Expand Up @@ -1245,6 +1247,7 @@ const EMPTY_UPSTREAM_DRIFT_ISSUES_CONFIG: FocusManifestUpstreamDriftIssuesConfig
const EMPTY_SWEEP_WATCHDOG_CONFIG: FocusManifestSweepWatchdogConfig = {
present: false,
enabled: false,
staleAfterMinutes: null,
};

const EMPTY_PR_RECONCILIATION_CONFIG: FocusManifestPrReconciliationConfig = {
Expand Down Expand Up @@ -2134,8 +2137,9 @@ export function upstreamDriftIssuesConfigToJson(config: FocusManifestUpstreamDri
}

/**
* Parse the optional top-level `sweepWatchdog:` mapping (#6558 / #6275). Mirrors {@link parseOpsConfig}
* exactly — `enabled` is the only field. Distinct from per-repo `review.sweepWatchdog`.
* Parse the optional top-level `sweepWatchdog:` mapping (#6558 / #6275 / #6594). Mirrors {@link parseOpsConfig}
* for `enabled`, plus an optional `staleAfterMinutes` positive integer. Invalid / non-positive values warn and
* fall back to null (caller keeps the hardcoded 45-minute default). Distinct from per-repo `review.sweepWatchdog`.
*/
function parseSweepWatchdogConfig(value: JsonValue | undefined, warnings: string[]): FocusManifestSweepWatchdogConfig {
if (value === undefined || value === null) return { ...EMPTY_SWEEP_WATCHDOG_CONFIG };
Expand All @@ -2145,14 +2149,21 @@ function parseSweepWatchdogConfig(value: JsonValue | undefined, warnings: string
}
const record = value as Record<string, JsonValue>;
const enabled = normalizeOptionalBoolean(record.enabled, "sweepWatchdog.enabled", warnings) ?? false;
return { present: true, enabled };
const staleAfterMinutes = normalizeOptionalPositiveInteger(
record.staleAfterMinutes,
"sweepWatchdog.staleAfterMinutes",
warnings,
);
return { present: true, enabled, staleAfterMinutes };
}

/** Serialize a sweepWatchdog config back into the parse-compatible shape so a cached snapshot round-trips
* through {@link parseSweepWatchdogConfig} unchanged. Returns null when nothing is configured. */
export function sweepWatchdogConfigToJson(config: FocusManifestSweepWatchdogConfig): JsonValue {
if (!config.present) return null;
return { enabled: config.enabled };
const out: Record<string, JsonValue> = { enabled: config.enabled };
if (config.staleAfterMinutes !== null) out.staleAfterMinutes = config.staleAfterMinutes;
return out;
}

/**
Expand Down
4 changes: 3 additions & 1 deletion src/queue/job-dispatch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,9 @@ export async function processJob(env: Env, message: JobMessage): Promise<void> {
// no-op, so disabled does zero work here too. Fails safe internally — never throws into the queue.
{
const sweepWatchdogManifestOverride = await resolveSweepWatchdogManifestOverride(env);
if (isSweepWatchdogEnabled(env, sweepWatchdogManifestOverride)) await runSweepLivenessWatchdog(env);
if (isSweepWatchdogEnabled(env, sweepWatchdogManifestOverride)) {
await runSweepLivenessWatchdog(env, sweepWatchdogManifestOverride);
}
}
return;
case "loop-escalation-sweep":
Expand Down
55 changes: 45 additions & 10 deletions src/review/sweep-watchdog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,16 @@ import type { JobMessage } from "../types";
import { errorMessage, nowIso } from "../utils/json";
import { isConvergenceRepoAllowed, listConvergenceRepos } from "./cutover-gate";

/** A manifest-sourced enable override (#6558 / #6275) -- the top-level `sweepWatchdog` block of the
* loopover self-repo's `.loopover.yml` (see FocusManifestSweepWatchdogConfig). Distinct from the
/** A manifest-sourced enable/threshold override (#6558 / #6275 / #6594) -- the top-level `sweepWatchdog`
* block of the loopover self-repo's `.loopover.yml` (see FocusManifestSweepWatchdogConfig). Distinct from the
* per-repo FORCE-OFF under `review.sweepWatchdog`. `present: false` means "no override configured",
* not "disabled" -- the caller falls through to the env var in that case. Mirrors OpsManifestOverride. */
export type SweepWatchdogManifestOverride = { present: boolean; enabled: boolean };
* not "disabled" -- the caller falls through to the env var in that case. `staleAfterMinutes: null` means
* keep the hardcoded {@link SWEEP_STALENESS_THRESHOLD_MS} default. Mirrors OpsManifestOverride. */
export type SweepWatchdogManifestOverride = {
present: boolean;
enabled: boolean;
staleAfterMinutes: number | null;
};

/** True when the sweep-liveness watchdog is enabled. Config-as-code (#6558 / #6275): a present top-level
* `sweepWatchdog` manifest block on the loopover self-repo wins outright; otherwise falls back to the
Expand Down Expand Up @@ -58,12 +63,16 @@ export async function resolveSweepWatchdogManifestOverride(env: Env, nowMs: numb
try {
const manifest = await loadRepoFocusManifest(env, resolveLoopOverSelfRepoFullName(env));
const config = manifest.sweepWatchdog;
const override = { present: config.present, enabled: config.enabled };
const override: SweepWatchdogManifestOverride = {
present: config.present,
enabled: config.enabled,
staleAfterMinutes: config.staleAfterMinutes,
};
sweepWatchdogManifestOverrideCache = { override, at: nowMs };
return override;
} catch (error) {
console.warn(JSON.stringify({ event: "sweep_watchdog_manifest_override_error", message: errorMessage(error).slice(0, 200) }));
const override = { present: false, enabled: false };
const override: SweepWatchdogManifestOverride = { present: false, enabled: false, staleAfterMinutes: null };
sweepWatchdogManifestOverrideCache = { override, at: nowMs };
return override;
}
Expand All @@ -79,11 +88,31 @@ export function clearSweepWatchdogManifestOverrideCacheForTest(): void {
* the sweep to do, so a `null` marker there means "nothing to regate," not "the sweep stopped working." */
export const SWEEP_STALENESS_THRESHOLD_MS = 45 * 60 * 1000;

export function isSweepStale(input: { openPullRequestCount: number; lastRegatedAt: string | null; nowMs: number }): boolean {
/** Resolve the effective staleness threshold in ms from a manifest override (#6594). Absent / null minutes
* keep {@link SWEEP_STALENESS_THRESHOLD_MS}; never returns zero/negative/NaN. */
export function resolveSweepStalenessThresholdMs(
manifestOverride?: SweepWatchdogManifestOverride | undefined,
): number {
const minutes = manifestOverride?.present ? manifestOverride.staleAfterMinutes : null;
if (typeof minutes === "number" && Number.isFinite(minutes) && minutes > 0) return minutes * 60_000;
return SWEEP_STALENESS_THRESHOLD_MS;
}

export function isSweepStale(input: {
openPullRequestCount: number;
lastRegatedAt: string | null;
nowMs: number;
/** Optional override in milliseconds (#6594); omitted ⇒ {@link SWEEP_STALENESS_THRESHOLD_MS}. */
staleAfterMs?: number;
}): boolean {
if (input.openPullRequestCount === 0) return false;
const lastMs = input.lastRegatedAt ? Date.parse(input.lastRegatedAt) : NaN;
if (!Number.isFinite(lastMs)) return true;
return input.nowMs - lastMs > SWEEP_STALENESS_THRESHOLD_MS;
const thresholdMs =
typeof input.staleAfterMs === "number" && Number.isFinite(input.staleAfterMs) && input.staleAfterMs > 0
? input.staleAfterMs
: SWEEP_STALENESS_THRESHOLD_MS;
return input.nowMs - lastMs > thresholdMs;
}

/** The same acting-autonomy repo set fanOutAgentRegateSweepJobs sweeps: the convergence allowlist
Expand Down Expand Up @@ -149,16 +178,22 @@ export interface StaleSweepRepo {
* Caller MUST gate this on {@link isSweepWatchdogEnabled} — it is invoked only from the flag-ON cron path, so
* flag-OFF this function is never reached and the cron does zero new work.
*/
export async function runSweepLivenessWatchdog(env: Env): Promise<StaleSweepRepo[]> {
export async function runSweepLivenessWatchdog(
env: Env,
/** Optional pre-resolved override (#6594); when omitted, looks up the self-repo manifest (cached). */
manifestOverride?: SweepWatchdogManifestOverride,
): Promise<StaleSweepRepo[]> {
const found: StaleSweepRepo[] = [];
const nowMs = Date.parse(nowIso());
const override = manifestOverride ?? (await resolveSweepWatchdogManifestOverride(env, nowMs));
const staleAfterMs = resolveSweepStalenessThresholdMs(override);
try {
const repos = await watchedRepos(env);
for (const repo of repos) {
try {
if (typeof repo.installationId !== "number") continue;
const [openPullRequestCount, lastRegatedAt] = await Promise.all([countOpenPullRequests(env, repo.fullName), getLatestRegatedAt(env, repo.fullName)]);
if (!isSweepStale({ openPullRequestCount, lastRegatedAt, nowMs })) continue;
if (!isSweepStale({ openPullRequestCount, lastRegatedAt, nowMs, staleAfterMs })) continue;
const lastMs = lastRegatedAt ? Date.parse(lastRegatedAt) : NaN;
const ageMs = Number.isFinite(lastMs) ? nowMs - lastMs : Number.POSITIVE_INFINITY;
found.push({ repoFullName: repo.fullName, installationId: repo.installationId, openPullRequestCount, lastRegatedAt, ageMs });
Expand Down
51 changes: 44 additions & 7 deletions test/unit/focus-manifest.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,7 @@ describe(".loopover.yml.example field-exhaustiveness (#1670)", () => {

const SWEEP_WATCHDOG_FIELD_TOKENS = {
enabled: "enabled:",
staleAfterMinutes: "staleAfterMinutes:",
} satisfies Record<Exclude<keyof FocusManifestSweepWatchdogConfig, "present">, string>;

it.each(Object.entries(SWEEP_WATCHDOG_FIELD_TOKENS))("documents sweepWatchdog.%s", (_field, token) => {
Expand Down Expand Up @@ -945,7 +946,7 @@ describe("compileFocusManifestPolicy", () => {
publicStats: { present: false, enabled: false },
draftFlow: { present: false, enabled: false },
upstreamDriftIssues: { present: false, enabled: false },
sweepWatchdog: { present: false, enabled: false },
sweepWatchdog: { present: false, enabled: false, staleAfterMinutes: null },
prReconciliation: { present: false, enabled: false },
federatedIntelligence: { present: false, enabled: false, collectorUrl: null, collectorMode: null },
warnings: [],
Expand Down Expand Up @@ -2143,15 +2144,19 @@ describe("parseFocusManifest gate config", () => {
});
});

describe("sweepWatchdog: (#6558 / #6275, fleet-wide sweep-liveness watchdog config-as-code override)", () => {
describe("sweepWatchdog: (#6558 / #6275 / #6594, fleet-wide sweep-liveness watchdog config-as-code override)", () => {
it("defaults to fully disabled/absent when the key is omitted, and does not make the manifest present on its own", () => {
const m = parseFocusManifest({});
expect(m.sweepWatchdog).toEqual({ present: false, enabled: false });
expect(m.sweepWatchdog).toEqual({ present: false, enabled: false, staleAfterMinutes: null });
expect(m.present).toBe(false);
});

it("treats an explicit null the same as an omitted key", () => {
expect(parseFocusManifest({ sweepWatchdog: null }).sweepWatchdog).toEqual({ present: false, enabled: false });
expect(parseFocusManifest({ sweepWatchdog: null }).sweepWatchdog).toEqual({
present: false,
enabled: false,
staleAfterMinutes: null,
});
});

it("warns and falls back to the default when the value is a non-mapping type (string or array)", () => {
Expand All @@ -2165,13 +2170,13 @@ describe("parseFocusManifest gate config", () => {

it("parses enabled: true, making the manifest present", () => {
const m = parseFocusManifest({ sweepWatchdog: { enabled: true } });
expect(m.sweepWatchdog).toEqual({ present: true, enabled: true });
expect(m.sweepWatchdog).toEqual({ present: true, enabled: true, staleAfterMinutes: null });
expect(m.present).toBe(true);
});

it("parses enabled: false explicitly, still marking the manifest present (present is a real override, off)", () => {
const m = parseFocusManifest({ sweepWatchdog: { enabled: false } });
expect(m.sweepWatchdog).toEqual({ present: true, enabled: false });
expect(m.sweepWatchdog).toEqual({ present: true, enabled: false, staleAfterMinutes: null });
expect(m.present).toBe(true);
});

Expand All @@ -2181,9 +2186,41 @@ describe("parseFocusManifest gate config", () => {
expect(m.warnings.some((w) => /sweepWatchdog\.enabled/.test(w))).toBe(true);
});

it("parses a valid staleAfterMinutes positive integer (#6594)", () => {
const m = parseFocusManifest({ sweepWatchdog: { enabled: true, staleAfterMinutes: 90 } });
expect(m.sweepWatchdog).toEqual({ present: true, enabled: true, staleAfterMinutes: 90 });
});

it("leaves staleAfterMinutes null when the field is absent (#6594)", () => {
const m = parseFocusManifest({ sweepWatchdog: { enabled: true } });
expect(m.sweepWatchdog.staleAfterMinutes).toBeNull();
});

it.each([
["string", "ninety" as unknown],
["zero", 0],
["negative", -5],
["NaN", Number.NaN],
["float", 1.5],
] as const)("warns and falls back to null when staleAfterMinutes is %s (#6594)", (_label, value) => {
const m = parseFocusManifest({ sweepWatchdog: { enabled: true, staleAfterMinutes: value as number } });
expect(m.sweepWatchdog.staleAfterMinutes).toBeNull();
expect(m.warnings.some((w) => /sweepWatchdog\.staleAfterMinutes/.test(w))).toBe(true);
});

it("round-trips through sweepWatchdogConfigToJson → parseFocusManifest unchanged", () => {
const m = parseFocusManifest({ sweepWatchdog: { enabled: true, staleAfterMinutes: 90 } });
expect(parseFocusManifest({ sweepWatchdog: sweepWatchdogConfigToJson(m.sweepWatchdog) }).sweepWatchdog).toEqual(
m.sweepWatchdog,
);
});

it("round-trips enabled-only configs without inventing a staleAfterMinutes key", () => {
const m = parseFocusManifest({ sweepWatchdog: { enabled: true } });
expect(parseFocusManifest({ sweepWatchdog: sweepWatchdogConfigToJson(m.sweepWatchdog) }).sweepWatchdog).toEqual(m.sweepWatchdog);
expect(sweepWatchdogConfigToJson(m.sweepWatchdog)).toEqual({ enabled: true });
expect(parseFocusManifest({ sweepWatchdog: sweepWatchdogConfigToJson(m.sweepWatchdog) }).sweepWatchdog).toEqual(
m.sweepWatchdog,
);
});

it("sweepWatchdogConfigToJson returns null for an absent config", () => {
Expand Down
Loading