From 97329195e4a5d4c57f7007eb74bd665f11816e04 Mon Sep 17 00:00:00 2001 From: jaso0n0818 Date: Sun, 21 Jun 2026 10:11:10 +0000 Subject: [PATCH] feat(scoring): warn when the served scoring snapshot is stale (#810) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getOrCreateScoringModelSnapshot served the last-good (or DEFAULT) scoring snapshot with no staleness signal, so a failed or long-stale upstream refresh silently kept previews on old constants — Pipeline B (upstream/ruleset.ts) has UPSTREAM_STALE_MS but the scoring side had none. Add SCORING_SNAPSHOT_STALE_MS (mirrors Pipeline B's 2h window) and a pure scoringSnapshotStalenessWarning(snapshot, now) helper; getOrCreateScoringModelSnapshot now appends that warning when the served snapshot is older than the window, so previews surface it. Pure helper + serve-path covered by tests (fresh/stale boundary, served-old, freshly-refreshed). Completes #810 (the parser-robustness deliverable landed in #969). Closes #810. --- src/scoring/model.ts | 15 ++++++++++++++- test/unit/scoring.test.ts | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/scoring/model.ts b/src/scoring/model.ts index 35f930064b..c088a2abcd 100644 --- a/src/scoring/model.ts +++ b/src/scoring/model.ts @@ -133,8 +133,21 @@ export async function refreshScoringModelSnapshot(env: Env): Promise, now: number = Date.now()): string | null { + if (Date.parse(snapshot.fetchedAt) + SCORING_SNAPSHOT_STALE_MS >= now) return null; + return "Scoring constants snapshot is stale: the last upstream refresh is older than the freshness window, so scoring may be using last-good or default constants and be behind upstream."; +} + export async function getOrCreateScoringModelSnapshot(env: Env): Promise { - return (await getLatestScoringModelSnapshot(env)) ?? refreshScoringModelSnapshot(env); + const snapshot = (await getLatestScoringModelSnapshot(env)) ?? (await refreshScoringModelSnapshot(env)); + // Surface staleness so previews do not silently use last-good/DEFAULT constants after a failed/old refresh (#810). + const stalenessWarning = scoringSnapshotStalenessWarning(snapshot); + return stalenessWarning ? { ...snapshot, warnings: [...snapshot.warnings, stalenessWarning] } : snapshot; } export function parsePythonNumberConstants(source: string, options: { knownOnly?: boolean } = { knownOnly: true }): Record { diff --git a/test/unit/scoring.test.ts b/test/unit/scoring.test.ts index 2502255382..1445bc85c4 100644 --- a/test/unit/scoring.test.ts +++ b/test/unit/scoring.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { getLatestScoringModelSnapshot, listUpstreamDriftReports } from "../../src/db/repositories"; -import { DEFAULT_SCORING_CONSTANTS, detectActiveModel, findUnmodeledUpstreamConstants, isTimeDecayEnabled, parsePythonNumberConstants, refreshScoringModelSnapshot } from "../../src/scoring/model"; +import { getLatestScoringModelSnapshot, listUpstreamDriftReports, persistScoringModelSnapshot } from "../../src/db/repositories"; +import { DEFAULT_SCORING_CONSTANTS, detectActiveModel, findUnmodeledUpstreamConstants, getOrCreateScoringModelSnapshot, isTimeDecayEnabled, parsePythonNumberConstants, refreshScoringModelSnapshot, SCORING_SNAPSHOT_STALE_MS, scoringSnapshotStalenessWarning } from "../../src/scoring/model"; import { buildScorePreview, calculateTimeDecay, makeScorePreviewRecord, resolveTimeDecay } from "../../src/scoring/preview"; import { unmodeledScoringConstantsFingerprint } from "../../src/upstream/unmodeled-scoring-drift"; import type { ScorePreviewInput } from "../../src/scoring/preview"; @@ -86,6 +86,34 @@ OSS_EMISSION_SHARE = 0.90 expect(parsed.OSS_EMISSION_SHARE).toBe(0.9); }); + it("flags only scoring snapshots older than the freshness window as stale (#810)", () => { + const now = Date.parse("2026-06-21T12:00:00.000Z"); + const justFresh = new Date(now - SCORING_SNAPSHOT_STALE_MS + 60_000).toISOString(); + const clearlyStale = new Date(now - SCORING_SNAPSHOT_STALE_MS - 60_000).toISOString(); + expect(scoringSnapshotStalenessWarning({ fetchedAt: justFresh }, now)).toBeNull(); + expect(scoringSnapshotStalenessWarning({ fetchedAt: clearlyStale }, now)).toMatch(/stale/i); + }); + + it("appends a staleness warning when getOrCreateScoringModelSnapshot serves an old snapshot (#810)", async () => { + const env = createTestEnv(); + await persistScoringModelSnapshot(env, snapshot); + const served = await getOrCreateScoringModelSnapshot(env); + expect(served.id).toBe(snapshot.id); + expect(served.warnings.some((warning) => /stale/i.test(warning))).toBe(true); + }); + + it("does not add a staleness warning when getOrCreate refreshes a fresh snapshot (#810)", async () => { + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "token" }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n"); + if (url.includes("programming_languages.json")) return Response.json({}); + return new Response("not found", { status: 404 }); + }); + const served = await getOrCreateScoringModelSnapshot(env); + expect(served.warnings.some((warning) => /stale/i.test(warning))).toBe(false); + }); + it("prefers exponential saturation when mixed upstream constants are present", () => { const parsed = parsePythonNumberConstants(` MERGED_PR_BASE_SCORE = 25