From fdd97838e56751927b82dfd3f7f94328cf84d5a8 Mon Sep 17 00:00:00 2001
From: JSONbored <49853598+JSONbored@users.noreply.github.com>
Date: Wed, 24 Jun 2026 05:04:55 -0700
Subject: [PATCH] fix(scoring): fail closed on a semantically-garbage 200
 constants body
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#1192 pinned the upstream constants fetch and froze last-good on a non-2xx
response, but a 200 with a semantically-garbage body — an HTML interstitial,
a Git-LFS pointer, or a truncated file — parsed to ~0 recognized constants
and silently reverted every constant to DEFAULT_SCORING_CONSTANTS while
keeping sourceKind="raw-github", moving live scoring with no staleness
signal.

The refresh now applies a sanity floor: a 200 body must parse at least
MIN_RECOGNIZED_SCORING_CONSTANTS recognized constants to be trusted.
Otherwise it is handled exactly like a failed fetch — freeze the last-good
snapshot, or bootstrap to a clearly-labeled "fallback" when there is no
verified last-good. Also surfaces a warning when the upstream ref can't be
resolved to an immutable SHA (the fetch falls back to the mutable ref), and
records the resolved fetch ref in the drift-sync source.
---
 src/scoring/model.ts      | 41 ++++++++++++++++------
 test/unit/scoring.test.ts | 71 ++++++++++++++++++++++++++++++++-------
 2 files changed, 89 insertions(+), 23 deletions(-)

diff --git a/src/scoring/model.ts b/src/scoring/model.ts
index 767f9ea052..a6297a9970 100644
--- a/src/scoring/model.ts
+++ b/src/scoring/model.ts
@@ -83,6 +83,11 @@ async function fetchUpstreamRefSha(upstream: { repo: string; ref: string }, toke
 
 const SCORING_CONSTANT_NAMES = new Set([...Object.keys(DEFAULT_SCORING_CONSTANTS), "MIN_TOKEN_SCORE_FOR_BASE_SCORE", "MAX_CODE_DENSITY_MULTIPLIER"]);
 
+// Sanity floor for a 200 constants.py body. A real upstream file defines ~30 recognized constants; an HTML
+// interstitial, a Git-LFS pointer, or a truncated body parses to ~0. Below this, treat the body as non-source
+// and fail closed rather than reverting live scoring to defaults under a "raw-github" label. (#audit-3.6)
+const MIN_RECOGNIZED_SCORING_CONSTANTS = 8;
+
 export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringModelSnapshotRecord> {
   const warnings: string[] = [];
   const fetchedAt = nowIso();
@@ -93,6 +98,9 @@ export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringMode
   // transient API error) fall back to the mutable ref so a refresh is never blocked purely on the SHA lookup.
   const upstreamSourceSha = await fetchUpstreamRefSha(upstream, env.GITHUB_PUBLIC_TOKEN);
   const fetchRef = upstreamSourceSha ?? upstream.ref;
+  // Surface the unpinned fall-back: when the SHA can't be resolved we fetch from the MUTABLE ref, so a later
+  // upstream force-push could change what every repo scores against with no other signal. (#audit-3.6/drift)
+  if (!upstreamSourceSha) warnings.push(`Could not resolve upstream ${upstream.repo}@${upstream.ref} to an immutable commit SHA; fetched from the mutable ref (scoring is unpinned until the next successful resolve).`);
   const constantsUrl = upstreamRawUrl({ repo: upstream.repo, ref: fetchRef }, "gittensor/constants.py");
   const programmingLanguagesUrl = upstreamRawUrl({ repo: upstream.repo, ref: fetchRef }, "gittensor/validator/weights/programming_languages.json");
   const [registrySnapshot, constantsResult, languagesResult] = await Promise.all([
@@ -101,14 +109,23 @@ export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringMode
     fetchJson(programmingLanguagesUrl, env.GITHUB_PUBLIC_TOKEN),
   ]);
 
-  // FAIL-CLOSED (#scoring-fail-closed): a failed constants fetch must NEVER silently overwrite the last verified
-  // upstream constants with hardcoded DEFAULT_SCORING_CONSTANTS — that would move live scoring with no one
-  // noticing. Freeze the last-good snapshot instead (its age is surfaced by scoringSnapshotStalenessWarning), and
-  // only bootstrap to defaults when there is no verified last-good to fall back to.
-  if (!constantsResult.ok) {
+  // Parse once. `recognizedCount` tells us whether a 200 body is a REAL constants.py or semantically garbage —
+  // an HTML interstitial, a Git-LFS pointer, or a truncated body — which parses to ~0 known scoring constants.
+  const parsedConstants = constantsResult.ok ? parsePythonNumberConstants(constantsResult.value) : {};
+  const recognizedCount = Object.keys(parsedConstants).filter((name) => SCORING_CONSTANT_NAMES.has(name)).length;
+  const constantsUsable = constantsResult.ok && recognizedCount >= MIN_RECOGNIZED_SCORING_CONSTANTS;
+
+  // FAIL-CLOSED (#scoring-fail-closed, #audit-3.6): a failed OR semantically-garbage constants fetch must NEVER
+  // silently overwrite the last verified upstream constants with hardcoded DEFAULT_SCORING_CONSTANTS — that would
+  // move live scoring with no one noticing. Freeze the last-good snapshot instead (its age is surfaced by
+  // scoringSnapshotStalenessWarning), and only bootstrap to defaults when there is no verified last-good.
+  if (!constantsUsable) {
     const lastGood = await getLatestScoringModelSnapshot(env);
     if (lastGood && lastGood.sourceKind !== "fallback") {
-      const frozenNote = `Upstream scoring constants refresh failed (${constantsResult.error}); froze the last-good snapshot rather than reverting to default constants.`;
+      const reason = constantsResult.ok
+        ? `parsed only ${recognizedCount} recognized constant(s) (expected ≥ ${MIN_RECOGNIZED_SCORING_CONSTANTS}) — body looks truncated or non-source`
+        : constantsResult.error;
+      const frozenNote = `Upstream scoring constants refresh failed (${reason}); froze the last-good snapshot rather than reverting to default constants.`;
       return { ...lastGood, warnings: [...lastGood.warnings, frozenNote] };
     }
   }
@@ -118,8 +135,8 @@ export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringMode
   let activeModelConstants: Record<string, number> = {};
   let constantsPayload: Record<string, JsonValue> = {};
 
-  if (constantsResult.ok) {
-    const parsed = parsePythonNumberConstants(constantsResult.value);
+  if (constantsResult.ok && constantsUsable) {
+    const parsed = parsedConstants;
     constants = { ...constants, ...parsed };
     activeModelConstants = parsed;
     const unmodeled = findUnmodeledUpstreamConstants(constantsResult.value);
@@ -133,7 +150,11 @@ export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringMode
     }
   } else {
     sourceKind = "fallback";
-    warnings.push(`Scoring constants fetch failed: ${constantsResult.error}`);
+    warnings.push(
+      constantsResult.ok
+        ? `Scoring constants body parsed only ${recognizedCount} recognized constant(s) (expected ≥ ${MIN_RECOGNIZED_SCORING_CONSTANTS}); using default constants.`
+        : `Scoring constants fetch failed: ${constantsResult.error}`,
+    );
   }
 
   const programmingLanguages = languagesResult.ok ? languagesResult.value : {};
@@ -160,7 +181,7 @@ export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringMode
   if (constantsResult.ok) {
     await syncUnmodeledScoringConstantDrift(env, {
       unmodeledConstants: findUnmodeledUpstreamConstants(constantsResult.value),
-      source: { repo: upstream.repo, ref: upstream.ref, commitSha: upstreamSourceSha },
+      source: { repo: upstream.repo, ref: fetchRef, commitSha: upstreamSourceSha },
     });
   }
   return snapshot;
diff --git a/test/unit/scoring.test.ts b/test/unit/scoring.test.ts
index e532b42aae..7592bdbde7 100644
--- a/test/unit/scoring.test.ts
+++ b/test/unit/scoring.test.ts
@@ -7,6 +7,12 @@ import type { ScorePreviewInput } from "../../src/scoring/preview";
 import type { RepositoryRecord, ScoringModelSnapshotRecord } from "../../src/types";
 import { createTestEnv } from "../helpers/d1";
 
+// A realistic constants.py body — at least MIN_RECOGNIZED_SCORING_CONSTANTS (8) recognized constants — so a
+// refresh is treated as a genuine raw-github fetch rather than tripping the semantic-garbage sanity floor.
+// None of these are active-model indicators or values the tests below override.
+const VALID_CONSTANTS_PY =
+  "ISSUES_TREASURY_EMISSION_SHARE = 0.1\nPR_LOOKBACK_DAYS = 30\nCONTRIBUTION_SCORE_FOR_FULL_BONUS = 1500\nMIN_VALID_MERGED_PRS = 3\nMIN_CREDIBILITY = 0.8\nMIN_VALID_SOLVED_ISSUES = 3\nMIN_ISSUE_CREDIBILITY = 0.8\nMIN_TOKEN_SCORE_FOR_VALID_ISSUE = 5\n";
+
 const snapshot: ScoringModelSnapshotRecord = {
   id: "score-model-fixture",
   sourceKind: "test",
@@ -122,7 +128,7 @@ BARE = .5_0
     const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "token" });
     vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
       const url = input.toString();
-      if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
+      if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
       if (url.includes("programming_languages.json")) return Response.json({});
       return new Response("not found", { status: 404 });
     });
@@ -314,7 +320,7 @@ NOVELTY_BONUS_SCALAR = 3
       const url = input.toString();
       fetchedUrls.push(url);
       if (url.includes("constants.py")) {
-        return new Response("MIN_TOKEN_SCORE_FOR_BASE_SCORE = 5\nMAX_CODE_DENSITY_MULTIPLIER = 1.15\n");
+        return new Response(VALID_CONSTANTS_PY + "MIN_TOKEN_SCORE_FOR_BASE_SCORE = 5\nMAX_CODE_DENSITY_MULTIPLIER = 1.15\n");
       }
       if (url.includes("programming_languages.json")) return Response.json({ TypeScript: 1 });
       return new Response("not found", { status: 404 });
@@ -335,7 +341,7 @@ NOVELTY_BONUS_SCALAR = 3
     const env = createTestEnv();
     vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
       const url = input.toString();
-      if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
+      if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
       if (url.includes("programming_languages.json")) return Response.json({});
       return new Response("not found", { status: 404 });
     });
@@ -389,7 +395,7 @@ NOVELTY_BONUS_SCALAR = 3
     const manyUnmodeled = Array.from({ length: 15 }, (_, index) => `UNMODELED_CONST_${String(index).padStart(2, "0")} = ${index + 1}`).join("\n");
     vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
       const url = input.toString();
-      if (url.includes("constants.py")) return new Response(manyUnmodeled);
+      if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + manyUnmodeled);
       if (url.includes("programming_languages.json")) return Response.json({});
       return new Response("not found", { status: 404 });
     });
@@ -414,7 +420,7 @@ NOVELTY_BONUS_SCALAR = 3
     vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
       const url = input.toString();
       fetchedUrls.push(url);
-      if (url.includes("constants.py")) return new Response("SRC_TOK_SATURATION_SCALE = 58.0\nNOVELTY_BONUS_SCALAR = 3\n");
+      if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "SRC_TOK_SATURATION_SCALE = 58.0\nNOVELTY_BONUS_SCALAR = 3\n");
       if (url.includes("programming_languages.json")) return Response.json({ TypeScript: 1 });
       return new Response("not found", { status: 404 });
     });
@@ -442,7 +448,7 @@ NOVELTY_BONUS_SCALAR = 3
     const EXPECTED_SHA = "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2";
     vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
       const url = input.toString();
-      if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
+      if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
       if (url.includes("programming_languages.json")) return Response.json({});
       // Upstream HEAD SHA endpoint: api.github.com/repos/{owner}/{repo}/commits/{ref}
       if (url.includes("api.github.com") && url.includes("/commits/main")) return Response.json({ sha: EXPECTED_SHA });
@@ -461,7 +467,7 @@ NOVELTY_BONUS_SCALAR = 3
     const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "token" });
     vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
       const url = input.toString();
-      if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
+      if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
       if (url.includes("programming_languages.json")) return Response.json({});
       // SHA endpoint fails — network error
       if (url.includes("api.github.com") && url.includes("/commits/")) throw new Error("network error");
@@ -484,7 +490,7 @@ NOVELTY_BONUS_SCALAR = 3
       const url = input.toString();
       fetchedUrls.push(url);
       if (url.includes("api.github.com") && url.includes("/commits/test")) return Response.json({ sha: SHA });
-      if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
+      if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
       if (url.includes("programming_languages.json")) return Response.json({});
       return new Response("not found", { status: 404 });
     });
@@ -504,7 +510,7 @@ NOVELTY_BONUS_SCALAR = 3
     // 1) A good refresh persists a verified raw-github snapshot.
     vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
       const url = input.toString();
-      if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\nOSS_EMISSION_SHARE = 0.5\n");
+      if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\nOSS_EMISSION_SHARE = 0.5\n");
       if (url.includes("programming_languages.json")) return Response.json({ TypeScript: 1 });
       return new Response("not found", { status: 404 });
     });
@@ -523,6 +529,45 @@ NOVELTY_BONUS_SCALAR = 3
     await expect(getLatestScoringModelSnapshot(env)).resolves.toMatchObject({ id: good.id, sourceKind: "raw-github" });
   });
 
+  it("freezes the last-good snapshot when a 200 constants body is semantically garbage (LFS/HTML/truncated)", async () => {
+    const env = createTestEnv();
+    vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
+      const url = input.toString();
+      if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "OSS_EMISSION_SHARE = 0.42\n");
+      if (url.includes("programming_languages.json")) return Response.json({});
+      return new Response("not found", { status: 404 });
+    });
+    const good = await refreshScoringModelSnapshot(env);
+    expect(good.sourceKind).toBe("raw-github");
+
+    // Upstream now returns a 200 Git-LFS pointer — 0 recognized scoring constants. Fail-closed: freeze last-good.
+    vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
+      const url = input.toString();
+      if (url.includes("constants.py")) return new Response("version https://git-lfs.github.com/spec/v1\noid sha256:abc123\nsize 1234\n");
+      if (url.includes("programming_languages.json")) return Response.json({});
+      return new Response("not found", { status: 404 });
+    });
+    const frozen = await refreshScoringModelSnapshot(env);
+    expect(frozen.id).toBe(good.id);
+    expect(frozen.sourceKind).toBe("raw-github"); // NOT reverted to defaults
+    expect(frozen.constants.OSS_EMISSION_SHARE).toBe(0.42);
+    expect(frozen.warnings.join(" ")).toMatch(/parsed only \d+ recognized constant/i);
+  });
+
+  it("bootstraps to fallback (not raw-github) when a 200 constants body is garbage and there is no last-good", async () => {
+    const env = createTestEnv();
+    vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
+      const url = input.toString();
+      if (url.includes("constants.py")) return new Response("<!DOCTYPE html><html><body>rate limited</body></html>");
+      if (url.includes("programming_languages.json")) return Response.json({});
+      return new Response("not found", { status: 404 });
+    });
+    const refreshed = await refreshScoringModelSnapshot(env);
+    expect(refreshed.sourceKind).toBe("fallback"); // labeled fallback, NOT a deceptive raw-github
+    expect(refreshed.warnings.join(" ")).toMatch(/parsed only \d+ recognized constant/i);
+    expect(refreshed.constants.MERGED_PR_BASE_SCORE).toBe(25); // the hardcoded default
+  });
+
   it("bootstraps to defaults (fallback) on a failed fetch ONLY when there is no verified last-good", async () => {
     const env = createTestEnv();
     vi.stubGlobal("fetch", async () => new Response("missing", { status: 404 }));
@@ -550,7 +595,7 @@ NOVELTY_BONUS_SCALAR = 3
     vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
       const url = input.toString();
       if (url.includes("api.github.com") && url.includes("/commits/")) throw new Error("network failure");
-      if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
+      if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
       if (url.includes("programming_languages.json")) return Response.json({});
       return new Response("not found", { status: 404 });
     });
@@ -565,7 +610,7 @@ NOVELTY_BONUS_SCALAR = 3
     vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
       const url = input.toString();
       if (url.includes("api.github.com") && url.includes("/commits/")) return Response.json({ sha: 42 });
-      if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
+      if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
       if (url.includes("programming_languages.json")) return Response.json({});
       return new Response("not found", { status: 404 });
     });
@@ -579,7 +624,7 @@ NOVELTY_BONUS_SCALAR = 3
     vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
       const url = input.toString();
       if (url.includes("api.github.com") && url.includes("/commits/")) return Response.json({ sha: "" });
-      if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
+      if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
       if (url.includes("programming_languages.json")) return Response.json({});
       return new Response("not found", { status: 404 });
     });
@@ -1173,7 +1218,7 @@ NOVELTY_BONUS_SCALAR = 3
     vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
       const url = input.toString();
       if (url.includes("constants.py")) {
-        return new Response("OSS_EMISSION_SHARE = 0.90\nMERGED_PR_BASE_SCORE = 25\nSRC_TOK_SATURATION_SCALE = 58\nMIN_TOKEN_SCORE_FOR_BASE_SCORE = 5\nMAX_CODE_DENSITY_MULTIPLIER = 1.15\n");
+        return new Response(VALID_CONSTANTS_PY + "OSS_EMISSION_SHARE = 0.90\nMERGED_PR_BASE_SCORE = 25\nSRC_TOK_SATURATION_SCALE = 58\nMIN_TOKEN_SCORE_FOR_BASE_SCORE = 5\nMAX_CODE_DENSITY_MULTIPLIER = 1.15\n");
       }
       if (url.includes("programming_languages.json")) return Response.json({ TypeScript: 1, Python: 0.8 });
       return new Response("not found", { status: 404 });