diff --git a/src/scoring/model.ts b/src/scoring/model.ts index 87ef55c83c..5db3c7b238 100644 --- a/src/scoring/model.ts +++ b/src/scoring/model.ts @@ -57,8 +57,15 @@ export async function refreshScoringModelSnapshot(env: Env): Promise 0) { + warnings.push( + `Upstream gittensor defines ${unmodeled.length} scoring constant(s) gittensory does not yet model: ${unmodeled.slice(0, 12).join(", ")}${unmodeled.length > 12 ? ", …" : ""}. Scoring may be behind upstream.`, + ); + } } else { sourceKind = "fallback"; warnings.push(`Scoring constants fetch failed: ${constantsResult.error}`); @@ -104,6 +111,20 @@ export function parsePythonNumberConstants(source: string, options: { knownOnly? return constants; } +/** + * Numeric constant names upstream gittensor defines that gittensory's scoring engine does NOT model. + * The normal parse is `knownOnly` (it keeps only constants we already encode), which silently hides + * upstream ADDITIONS — e.g. a newly-introduced time-decay constant. Surfacing these makes scoring + * staleness visible: if upstream adds a scoring dimension, an operator sees it instead of the gate + * silently drifting behind. Detection only — it does not change any score. + */ +export function findUnmodeledUpstreamConstants(source: string): string[] { + const all = parsePythonNumberConstants(source, { knownOnly: false }); + return Object.keys(all) + .filter((name) => !SCORING_CONSTANT_NAMES.has(name)) + .sort(); +} + export function detectActiveModel(constants: Record): ScoringModelSnapshotRecord["activeModel"] { if (hasSaturationConstants(constants)) return "pending_saturation_model"; if (hasDensityConstants(constants)) { diff --git a/test/unit/scoring.test.ts b/test/unit/scoring.test.ts index 47bd0a924a..914450165e 100644 --- a/test/unit/scoring.test.ts +++ b/test/unit/scoring.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { getLatestScoringModelSnapshot } from "../../src/db/repositories"; -import { detectActiveModel, parsePythonNumberConstants, refreshScoringModelSnapshot } from "../../src/scoring/model"; +import { detectActiveModel, findUnmodeledUpstreamConstants, parsePythonNumberConstants, refreshScoringModelSnapshot } from "../../src/scoring/model"; import { buildScorePreview, makeScorePreviewRecord } from "../../src/scoring/preview"; import type { ScorePreviewInput } from "../../src/scoring/preview"; import type { RepositoryRecord, ScoringModelSnapshotRecord } from "../../src/types"; @@ -118,6 +118,30 @@ MAX_CODE_DENSITY_MULTIPLIER = 1.15 expect(refreshed.warnings.join(" ")).toMatch(/recognized active-model indicator/i); }); + it("flags upstream scoring constants gittensory does not model (staleness visibility)", () => { + // SRC_TOK_SATURATION_SCALE is modeled; the TIME_DECAY_* constants are NOT — so they surface as unmodeled. + const unmodeled = findUnmodeledUpstreamConstants( + "SRC_TOK_SATURATION_SCALE = 58.0\nTIME_DECAY_GRACE_PERIOD_HOURS = 12\nTIME_DECAY_SIGMOID_MIDPOINT = 10\n", + ); + expect(unmodeled).toEqual(["TIME_DECAY_GRACE_PERIOD_HOURS", "TIME_DECAY_SIGMOID_MIDPOINT"]); + expect(unmodeled).not.toContain("SRC_TOK_SATURATION_SCALE"); + }); + + it("warns on the snapshot when upstream defines an unmodeled scoring dimension", async () => { + const env = createTestEnv(); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("constants.py")) return new Response("SRC_TOK_SATURATION_SCALE = 58.0\nTIME_DECAY_GRACE_PERIOD_HOURS = 12\n"); + if (url.includes("programming_languages.json")) return Response.json({ TypeScript: 1 }); + return new Response("not found", { status: 404 }); + }); + + const refreshed = await refreshScoringModelSnapshot(env); + + expect(refreshed.warnings.join(" ")).toMatch(/does not yet model.*TIME_DECAY_GRACE_PERIOD_HOURS/); + expect(refreshed.payload.constants).toMatchObject({ unmodeledUpstreamConstants: ["TIME_DECAY_GRACE_PERIOD_HOURS"] }); + }); + it("uses saturation math as the active private preview model", () => { const saturationSnapshot: ScoringModelSnapshotRecord = { ...snapshot,