Skip to content

fix(scoring): fail closed on a semantically-garbage 200 constants body #1235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JSONbored merged 1 commit into from
Jun 24, 2026
Merged

fix(scoring): fail closed on a semantically-garbage 200 constants body #1235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 31 additions & 10 deletions src/scoring/model.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,11 @@ async function fetchUpstreamRefSha(upstream: { repo: string; ref: string }, toke

const SCORING_CONSTANT_NAMES = new Set([...Object.keys(DEFAULT_SCORING_CONSTANTS), "MIN_TOKEN_SCORE_FOR_BASE_SCORE", "MAX_CODE_DENSITY_MULTIPLIER"]);

// Sanity floor for a 200 constants.py body. A real upstream file defines ~30 recognized constants; an HTML
// interstitial, a Git-LFS pointer, or a truncated body parses to ~0. Below this, treat the body as non-source
// and fail closed rather than reverting live scoring to defaults under a "raw-github" label. (#audit-3.6)
const MIN_RECOGNIZED_SCORING_CONSTANTS = 8;

export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringModelSnapshotRecord> {
const warnings: string[] = [];
const fetchedAt = nowIso();
Expand All @@ -93,6 +98,9 @@ export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringMode
// transient API error) fall back to the mutable ref so a refresh is never blocked purely on the SHA lookup.
const upstreamSourceSha = await fetchUpstreamRefSha(upstream, env.GITHUB_PUBLIC_TOKEN);
const fetchRef = upstreamSourceSha ?? upstream.ref;
// Surface the unpinned fall-back: when the SHA can't be resolved we fetch from the MUTABLE ref, so a later
// upstream force-push could change what every repo scores against with no other signal. (#audit-3.6/drift)
if (!upstreamSourceSha) warnings.push(`Could not resolve upstream ${upstream.repo}@${upstream.ref} to an immutable commit SHA; fetched from the mutable ref (scoring is unpinned until the next successful resolve).`);
const constantsUrl = upstreamRawUrl({ repo: upstream.repo, ref: fetchRef }, "gittensor/constants.py");
const programmingLanguagesUrl = upstreamRawUrl({ repo: upstream.repo, ref: fetchRef }, "gittensor/validator/weights/programming_languages.json");
const [registrySnapshot, constantsResult, languagesResult] = await Promise.all([
Expand All @@ -101,14 +109,23 @@ export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringMode
fetchJson(programmingLanguagesUrl, env.GITHUB_PUBLIC_TOKEN),
]);

// FAIL-CLOSED (#scoring-fail-closed): a failed constants fetch must NEVER silently overwrite the last verified
// upstream constants with hardcoded DEFAULT_SCORING_CONSTANTS — that would move live scoring with no one
// noticing. Freeze the last-good snapshot instead (its age is surfaced by scoringSnapshotStalenessWarning), and
// only bootstrap to defaults when there is no verified last-good to fall back to.
if (!constantsResult.ok) {
// Parse once. `recognizedCount` tells us whether a 200 body is a REAL constants.py or semantically garbage —
// an HTML interstitial, a Git-LFS pointer, or a truncated body — which parses to ~0 known scoring constants.
const parsedConstants = constantsResult.ok ? parsePythonNumberConstants(constantsResult.value) : {};
const recognizedCount = Object.keys(parsedConstants).filter((name) => SCORING_CONSTANT_NAMES.has(name)).length;
const constantsUsable = constantsResult.ok && recognizedCount >= MIN_RECOGNIZED_SCORING_CONSTANTS;

// FAIL-CLOSED (#scoring-fail-closed, #audit-3.6): a failed OR semantically-garbage constants fetch must NEVER
// silently overwrite the last verified upstream constants with hardcoded DEFAULT_SCORING_CONSTANTS — that would
// move live scoring with no one noticing. Freeze the last-good snapshot instead (its age is surfaced by
// scoringSnapshotStalenessWarning), and only bootstrap to defaults when there is no verified last-good.
if (!constantsUsable) {
const lastGood = await getLatestScoringModelSnapshot(env);
if (lastGood && lastGood.sourceKind !== "fallback") {
const frozenNote = `Upstream scoring constants refresh failed (${constantsResult.error}); froze the last-good snapshot rather than reverting to default constants.`;
const reason = constantsResult.ok
? `parsed only ${recognizedCount} recognized constant(s) (expected ≥ ${MIN_RECOGNIZED_SCORING_CONSTANTS}) — body looks truncated or non-source`
: constantsResult.error;
const frozenNote = `Upstream scoring constants refresh failed (${reason}); froze the last-good snapshot rather than reverting to default constants.`;
return { ...lastGood, warnings: [...lastGood.warnings, frozenNote] };
}
}
Expand All @@ -118,8 +135,8 @@ export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringMode
let activeModelConstants: Record<string, number> = {};
let constantsPayload: Record<string, JsonValue> = {};

if (constantsResult.ok) {
const parsed = parsePythonNumberConstants(constantsResult.value);
if (constantsResult.ok && constantsUsable) {
const parsed = parsedConstants;
constants = { ...constants, ...parsed };
activeModelConstants = parsed;
const unmodeled = findUnmodeledUpstreamConstants(constantsResult.value);
Expand All @@ -133,7 +150,11 @@ export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringMode
}
} else {
sourceKind = "fallback";
warnings.push(`Scoring constants fetch failed: ${constantsResult.error}`);
warnings.push(
constantsResult.ok
? `Scoring constants body parsed only ${recognizedCount} recognized constant(s) (expected ≥ ${MIN_RECOGNIZED_SCORING_CONSTANTS}); using default constants.`
: `Scoring constants fetch failed: ${constantsResult.error}`,
);
}

const programmingLanguages = languagesResult.ok ? languagesResult.value : {};
Expand All @@ -160,7 +181,7 @@ export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringMode
if (constantsResult.ok) {
await syncUnmodeledScoringConstantDrift(env, {
unmodeledConstants: findUnmodeledUpstreamConstants(constantsResult.value),
source: { repo: upstream.repo, ref: upstream.ref, commitSha: upstreamSourceSha },
source: { repo: upstream.repo, ref: fetchRef, commitSha: upstreamSourceSha },
});
}
return snapshot;
Expand Down
71 changes: 58 additions & 13 deletions test/unit/scoring.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ import type { ScorePreviewInput } from "../../src/scoring/preview";
import type { RepositoryRecord, ScoringModelSnapshotRecord } from "../../src/types";
import { createTestEnv } from "../helpers/d1";

// A realistic constants.py body — at least MIN_RECOGNIZED_SCORING_CONSTANTS (8) recognized constants — so a
// refresh is treated as a genuine raw-github fetch rather than tripping the semantic-garbage sanity floor.
// None of these are active-model indicators or values the tests below override.
const VALID_CONSTANTS_PY =
"ISSUES_TREASURY_EMISSION_SHARE = 0.1\nPR_LOOKBACK_DAYS = 30\nCONTRIBUTION_SCORE_FOR_FULL_BONUS = 1500\nMIN_VALID_MERGED_PRS = 3\nMIN_CREDIBILITY = 0.8\nMIN_VALID_SOLVED_ISSUES = 3\nMIN_ISSUE_CREDIBILITY = 0.8\nMIN_TOKEN_SCORE_FOR_VALID_ISSUE = 5\n";

const snapshot: ScoringModelSnapshotRecord = {
id: "score-model-fixture",
sourceKind: "test",
Expand Down Expand Up @@ -122,7 +128,7 @@ BARE = .5_0
const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "token" });
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("programming_languages.json")) return Response.json({});
return new Response("not found", { status: 404 });
});
Expand Down Expand Up @@ -314,7 +320,7 @@ NOVELTY_BONUS_SCALAR = 3
const url = input.toString();
fetchedUrls.push(url);
if (url.includes("constants.py")) {
return new Response("MIN_TOKEN_SCORE_FOR_BASE_SCORE = 5\nMAX_CODE_DENSITY_MULTIPLIER = 1.15\n");
return new Response(VALID_CONSTANTS_PY + "MIN_TOKEN_SCORE_FOR_BASE_SCORE = 5\nMAX_CODE_DENSITY_MULTIPLIER = 1.15\n");
}
if (url.includes("programming_languages.json")) return Response.json({ TypeScript: 1 });
return new Response("not found", { status: 404 });
Expand All @@ -335,7 +341,7 @@ NOVELTY_BONUS_SCALAR = 3
const env = createTestEnv();
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("programming_languages.json")) return Response.json({});
return new Response("not found", { status: 404 });
});
Expand Down Expand Up @@ -389,7 +395,7 @@ NOVELTY_BONUS_SCALAR = 3
const manyUnmodeled = Array.from({ length: 15 }, (_, index) => `UNMODELED_CONST_${String(index).padStart(2, "0")} = ${index + 1}`).join("\n");
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("constants.py")) return new Response(manyUnmodeled);
if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + manyUnmodeled);
if (url.includes("programming_languages.json")) return Response.json({});
return new Response("not found", { status: 404 });
});
Expand All @@ -414,7 +420,7 @@ NOVELTY_BONUS_SCALAR = 3
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
fetchedUrls.push(url);
if (url.includes("constants.py")) return new Response("SRC_TOK_SATURATION_SCALE = 58.0\nNOVELTY_BONUS_SCALAR = 3\n");
if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "SRC_TOK_SATURATION_SCALE = 58.0\nNOVELTY_BONUS_SCALAR = 3\n");
if (url.includes("programming_languages.json")) return Response.json({ TypeScript: 1 });
return new Response("not found", { status: 404 });
});
Expand Down Expand Up @@ -442,7 +448,7 @@ NOVELTY_BONUS_SCALAR = 3
const EXPECTED_SHA = "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2";
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("programming_languages.json")) return Response.json({});
// Upstream HEAD SHA endpoint: github.com/ghapi/repos/{owner}/{repo}/commits/{ref}
if (url.includes("github.com/ghapi") && url.includes("/commits/main")) return Response.json({ sha: EXPECTED_SHA });
Expand All @@ -461,7 +467,7 @@ NOVELTY_BONUS_SCALAR = 3
const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "token" });
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("programming_languages.json")) return Response.json({});
// SHA endpoint fails — network error
if (url.includes("github.com/ghapi") && url.includes("/commits/")) throw new Error("network error");
Expand All @@ -484,7 +490,7 @@ NOVELTY_BONUS_SCALAR = 3
const url = input.toString();
fetchedUrls.push(url);
if (url.includes("github.com/ghapi") && url.includes("/commits/test")) return Response.json({ sha: SHA });
if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("programming_languages.json")) return Response.json({});
return new Response("not found", { status: 404 });
});
Expand All @@ -504,7 +510,7 @@ NOVELTY_BONUS_SCALAR = 3
// 1) A good refresh persists a verified raw-github snapshot.
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\nOSS_EMISSION_SHARE = 0.5\n");
if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\nOSS_EMISSION_SHARE = 0.5\n");
if (url.includes("programming_languages.json")) return Response.json({ TypeScript: 1 });
return new Response("not found", { status: 404 });
});
Expand All @@ -523,6 +529,45 @@ NOVELTY_BONUS_SCALAR = 3
await expect(getLatestScoringModelSnapshot(env)).resolves.toMatchObject({ id: good.id, sourceKind: "raw-github" });
});

it("freezes the last-good snapshot when a 200 constants body is semantically garbage (LFS/HTML/truncated)", async () => {
const env = createTestEnv();
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "OSS_EMISSION_SHARE = 0.42\n");
if (url.includes("programming_languages.json")) return Response.json({});
return new Response("not found", { status: 404 });
});
const good = await refreshScoringModelSnapshot(env);
expect(good.sourceKind).toBe("raw-github");

// Upstream now returns a 200 Git-LFS pointer — 0 recognized scoring constants. Fail-closed: freeze last-good.
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("constants.py")) return new Response("version https://git-lfs.github.com/spec/v1\noid sha256:abc123\nsize 1234\n");
if (url.includes("programming_languages.json")) return Response.json({});
return new Response("not found", { status: 404 });
});
const frozen = await refreshScoringModelSnapshot(env);
expect(frozen.id).toBe(good.id);
expect(frozen.sourceKind).toBe("raw-github"); // NOT reverted to defaults
expect(frozen.constants.OSS_EMISSION_SHARE).toBe(0.42);
expect(frozen.warnings.join(" ")).toMatch(/parsed only \d+ recognized constant/i);
});

it("bootstraps to fallback (not raw-github) when a 200 constants body is garbage and there is no last-good", async () => {
const env = createTestEnv();
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("constants.py")) return new Response("<!DOCTYPE html><html><body>rate limited</body></html>");
if (url.includes("programming_languages.json")) return Response.json({});
return new Response("not found", { status: 404 });
});
const refreshed = await refreshScoringModelSnapshot(env);
expect(refreshed.sourceKind).toBe("fallback"); // labeled fallback, NOT a deceptive raw-github
expect(refreshed.warnings.join(" ")).toMatch(/parsed only \d+ recognized constant/i);
expect(refreshed.constants.MERGED_PR_BASE_SCORE).toBe(25); // the hardcoded default
});

it("bootstraps to defaults (fallback) on a failed fetch ONLY when there is no verified last-good", async () => {
const env = createTestEnv();
vi.stubGlobal("fetch", async () => new Response("missing", { status: 404 }));
Expand Down Expand Up @@ -550,7 +595,7 @@ NOVELTY_BONUS_SCALAR = 3
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("github.com/ghapi") && url.includes("/commits/")) throw new Error("network failure");
if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("programming_languages.json")) return Response.json({});
return new Response("not found", { status: 404 });
});
Expand All @@ -565,7 +610,7 @@ NOVELTY_BONUS_SCALAR = 3
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("github.com/ghapi") && url.includes("/commits/")) return Response.json({ sha: 42 });
if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("programming_languages.json")) return Response.json({});
return new Response("not found", { status: 404 });
});
Expand All @@ -579,7 +624,7 @@ NOVELTY_BONUS_SCALAR = 3
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("github.com/ghapi") && url.includes("/commits/")) return Response.json({ sha: "" });
if (url.includes("constants.py")) return new Response("MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("constants.py")) return new Response(VALID_CONSTANTS_PY + "MERGED_PR_BASE_SCORE = 25\n");
if (url.includes("programming_languages.json")) return Response.json({});
return new Response("not found", { status: 404 });
});
Expand Down Expand Up @@ -1173,7 +1218,7 @@ NOVELTY_BONUS_SCALAR = 3
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("constants.py")) {
return new Response("OSS_EMISSION_SHARE = 0.90\nMERGED_PR_BASE_SCORE = 25\nSRC_TOK_SATURATION_SCALE = 58\nMIN_TOKEN_SCORE_FOR_BASE_SCORE = 5\nMAX_CODE_DENSITY_MULTIPLIER = 1.15\n");
return new Response(VALID_CONSTANTS_PY + "OSS_EMISSION_SHARE = 0.90\nMERGED_PR_BASE_SCORE = 25\nSRC_TOK_SATURATION_SCALE = 58\nMIN_TOKEN_SCORE_FOR_BASE_SCORE = 5\nMAX_CODE_DENSITY_MULTIPLIER = 1.15\n");
}
if (url.includes("programming_languages.json")) return Response.json({ TypeScript: 1, Python: 0.8 });
return new Response("not found", { status: 404 });
Expand Down
Loading