From 84de0b9641239bc302caa3be2805de6bf422e956 Mon Sep 17 00:00:00 2001 From: kiannidev <156195510+kiannidev@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:21:31 +0200 Subject: [PATCH 1/3] feat(scoring): route unmodeled upstream constants into drift pipeline Persist a stable upstream drift report when gittensor defines scoring constants gittensory does not model, so scheduled issue filing and operator status can act on the signal instead of burying it in snapshot warnings alone. Fixes #811 Co-authored-by: Cursor --- src/scoring/model.ts | 16 ++++- src/upstream/ruleset.ts | 11 ++- src/upstream/unmodeled-scoring-drift.ts | 81 +++++++++++++++++++++++ test/unit/scoring.test.ts | 9 ++- test/unit/unmodeled-scoring-drift.test.ts | 60 +++++++++++++++++ test/unit/upstream-ruleset.test.ts | 33 +++++++++ 6 files changed, 205 insertions(+), 5 deletions(-) create mode 100644 src/upstream/unmodeled-scoring-drift.ts create mode 100644 test/unit/unmodeled-scoring-drift.test.ts diff --git a/src/scoring/model.ts b/src/scoring/model.ts index 86f028c6ac..737a71b25c 100644 --- a/src/scoring/model.ts +++ b/src/scoring/model.ts @@ -3,6 +3,7 @@ import { persistScoringModelSnapshot, } from "../db/repositories"; import { getLatestRegistrySnapshot } from "../registry/sync"; +import { syncUnmodeledScoringConstantDrift } from "../upstream/unmodeled-scoring-drift"; import type { JsonValue, ScoringModelSnapshotRecord } from "../types"; import { errorMessage, nowIso } from "../utils/json"; @@ -101,6 +102,12 @@ export async function refreshScoringModelSnapshot(env: Env): Promise): string[] { + return Object.keys(allConstants) .filter((name) => !SCORING_CONSTANT_NAMES.has(name)) .sort(); } +export function findUnmodeledUpstreamConstants(source: string): string[] { + return findUnmodeledConstantKeys(parsePythonNumberConstants(source, { knownOnly: false })); +} + /** * Owner-controlled global gate for applying upstream time-decay to score previews (#703). Default OFF: the * roadmap deferral requires the owner to review a before/after ranking diff before enabling. Even when on, diff --git a/src/upstream/ruleset.ts b/src/upstream/ruleset.ts index 6be4a99f65..1ab48170ee 100644 --- a/src/upstream/ruleset.ts +++ b/src/upstream/ruleset.ts @@ -10,7 +10,8 @@ import { upsertUpstreamDriftReport, } from "../db/repositories"; import { normalizeRegistryPayload } from "../registry/normalize"; -import { detectActiveModel, parsePythonNumberConstants } from "../scoring/model"; +import { detectActiveModel, findUnmodeledConstantKeys, parsePythonNumberConstants } from "../scoring/model"; +import { syncUnmodeledScoringConstantDrift } from "./unmodeled-scoring-drift"; import type { JsonValue, RegistryDriftSurface, @@ -201,6 +202,14 @@ export async function refreshUpstreamDrift(env: Env): Promise<{ sources: Upstrea const ruleset = await buildUpstreamRulesetSnapshot(env, sources); const drift = await buildUpstreamDriftReport(ruleset, (await listLatestUpstreamRulesetSnapshots(env, 2))[1] ?? null); if (drift) await upsertUpstreamDriftReport(env, drift); + const constantsSource = sources.find((source) => source.sourceKey === "constants"); + if (constantsSource && constantsSource.status !== "error") { + await syncUnmodeledScoringConstantDrift(env, { + unmodeledConstants: findUnmodeledConstantKeys(numericRecord(constantsSource.parsed.constants)), + currentRulesetId: ruleset.id, + source: { repo: ruleset.sourceRepo, ref: ruleset.sourceRef, commitSha: ruleset.commitSha ?? null }, + }); + } await recordAuditEvent(env, { eventType: "upstream.drift_detected", outcome: drift ? "completed" : "success", diff --git a/src/upstream/unmodeled-scoring-drift.ts b/src/upstream/unmodeled-scoring-drift.ts new file mode 100644 index 0000000000..71005c8fac --- /dev/null +++ b/src/upstream/unmodeled-scoring-drift.ts @@ -0,0 +1,81 @@ +import { + getLatestUpstreamRulesetSnapshot, + listUpstreamDriftReports, + upsertUpstreamDriftReport, +} from "../db/repositories"; +import type { UpstreamDriftArea, UpstreamDriftReportRecord, UpstreamDriftSeverity } from "../types"; +import { sha256Hex } from "../utils/crypto"; +import { nowIso } from "../utils/json"; + +const UNMODELED_SCORING_CONSTANTS_FINGERPRINT_SEED = "gittensory:upstream:unmodeled_scoring_constants:v1"; +const SCORING_MODEL_FOLLOW_UP = ["src/scoring/model.ts", "src/upstream/ruleset.ts", "test/unit/upstream-ruleset.test.ts"]; + +export async function unmodeledScoringConstantsFingerprint(): Promise { + return sha256Hex(UNMODELED_SCORING_CONSTANTS_FINGERPRINT_SEED); +} + +export async function syncUnmodeledScoringConstantDrift( + env: Env, + args: { + unmodeledConstants: string[]; + currentRulesetId?: string | null; + source?: { repo: string; ref: string; commitSha?: string | null }; + }, +): Promise { + const fingerprint = await unmodeledScoringConstantsFingerprint(); + const existing = (await listUpstreamDriftReports(env, 50)).find((report) => report.fingerprint === fingerprint) ?? null; + const now = nowIso(); + + if (args.unmodeledConstants.length === 0) { + if (!existing || existing.status === "resolved") return existing; + const resolved: UpstreamDriftReportRecord = { + ...existing, + status: "resolved", + severity: "low", + summary: "All upstream scoring constants are modeled in gittensory.", + updatedAt: now, + payload: { + ...existing.payload, + kind: "unmodeled_scoring_constants", + unmodeledUpstreamConstants: [], + resolvedAt: now, + }, + }; + await upsertUpstreamDriftReport(env, resolved); + return resolved; + } + + const rulesetId = args.currentRulesetId ?? (await getLatestUpstreamRulesetSnapshot(env))?.id ?? null; + const source = args.source ?? { + repo: env.GITTENSOR_UPSTREAM_REPO || "entrius/gittensor", + ref: env.GITTENSOR_UPSTREAM_REF || "test", + commitSha: null, + }; + const unmodeled = [...args.unmodeledConstants].sort(); + const summary = `Upstream defines ${unmodeled.length} scoring constant(s) gittensory does not model: ${unmodeled.slice(0, 12).join(", ")}${unmodeled.length > 12 ? ", …" : ""}`; + const severity: UpstreamDriftSeverity = unmodeled.length >= 3 ? "high" : "medium"; + const affectedAreas: UpstreamDriftArea[] = ["scoring_model"]; + const report: UpstreamDriftReportRecord = { + id: existing?.id ?? crypto.randomUUID(), + fingerprint, + severity, + status: "open", + summary, + affectedAreas, + previousRulesetId: existing?.previousRulesetId ?? null, + currentRulesetId: rulesetId, + issueNumber: existing?.issueNumber ?? null, + issueUrl: existing?.issueUrl ?? null, + payload: { + kind: "unmodeled_scoring_constants", + unmodeledUpstreamConstants: unmodeled, + changes: [`${unmodeled.length} upstream scoring constant(s) are not modeled in gittensory`], + source, + recommendedFollowUp: SCORING_MODEL_FOLLOW_UP, + }, + generatedAt: existing?.generatedAt ?? now, + updatedAt: now, + }; + await upsertUpstreamDriftReport(env, report); + return report; +} diff --git a/test/unit/scoring.test.ts b/test/unit/scoring.test.ts index fc05c09756..9fb5ff0a27 100644 --- a/test/unit/scoring.test.ts +++ b/test/unit/scoring.test.ts @@ -1,7 +1,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { getLatestScoringModelSnapshot } from "../../src/db/repositories"; +import { getLatestScoringModelSnapshot, listUpstreamDriftReports } from "../../src/db/repositories"; import { DEFAULT_SCORING_CONSTANTS, detectActiveModel, findUnmodeledUpstreamConstants, isTimeDecayEnabled, parsePythonNumberConstants, refreshScoringModelSnapshot } from "../../src/scoring/model"; import { buildScorePreview, calculateTimeDecay, makeScorePreviewRecord, resolveTimeDecay } from "../../src/scoring/preview"; +import { unmodeledScoringConstantsFingerprint } from "../../src/upstream/unmodeled-scoring-drift"; import type { ScorePreviewInput } from "../../src/scoring/preview"; import type { RepositoryRecord, ScoringModelSnapshotRecord } from "../../src/types"; import { createTestEnv } from "../helpers/d1"; @@ -158,6 +159,12 @@ MAX_CODE_DENSITY_MULTIPLIER = 1.15 expect(refreshed.warnings.join(" ")).toMatch(/does not yet model.*NOVELTY_BONUS_SCALAR/); expect(refreshed.payload.constants).toMatchObject({ unmodeledUpstreamConstants: ["NOVELTY_BONUS_SCALAR"] }); + const fingerprint = await unmodeledScoringConstantsFingerprint(); + expect((await listUpstreamDriftReports(env, 10)).find((report) => report.fingerprint === fingerprint)).toMatchObject({ + status: "open", + affectedAreas: ["scoring_model"], + payload: expect.objectContaining({ unmodeledUpstreamConstants: ["NOVELTY_BONUS_SCALAR"] }), + }); }); it("uses saturation math as the active private preview model", () => { diff --git a/test/unit/unmodeled-scoring-drift.test.ts b/test/unit/unmodeled-scoring-drift.test.ts new file mode 100644 index 0000000000..5d21f57148 --- /dev/null +++ b/test/unit/unmodeled-scoring-drift.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { listUpstreamDriftReports, updateUpstreamDriftReportIssue } from "../../src/db/repositories"; +import { syncUnmodeledScoringConstantDrift, unmodeledScoringConstantsFingerprint } from "../../src/upstream/unmodeled-scoring-drift"; +import { createTestEnv } from "../helpers/d1"; + +describe("unmodeled scoring constant drift", () => { + it("opens a stable-fingerprint drift report for unmodeled upstream constants", async () => { + const env = createTestEnv(); + const fingerprint = await unmodeledScoringConstantsFingerprint(); + const report = await syncUnmodeledScoringConstantDrift(env, { + unmodeledConstants: ["NOVELTY_BONUS_SCALAR", "EXTRA_WEIGHT"], + source: { repo: "entrius/gittensor", ref: "test", commitSha: "abc123" }, + }); + + expect(report).toMatchObject({ + fingerprint, + status: "open", + severity: "medium", + affectedAreas: ["scoring_model"], + summary: expect.stringContaining("NOVELTY_BONUS_SCALAR"), + payload: expect.objectContaining({ + kind: "unmodeled_scoring_constants", + unmodeledUpstreamConstants: ["EXTRA_WEIGHT", "NOVELTY_BONUS_SCALAR"], + }), + }); + expect(await listUpstreamDriftReports(env, 5)).toContainEqual(expect.objectContaining({ fingerprint, status: "open" })); + }); + + it("escalates severity when many constants are unmodeled", async () => { + const env = createTestEnv(); + const report = await syncUnmodeledScoringConstantDrift(env, { + unmodeledConstants: ["A", "B", "C"], + }); + expect(report?.severity).toBe("high"); + }); + + it("resolves the drift report when all constants are modeled", async () => { + const env = createTestEnv(); + const fingerprint = await unmodeledScoringConstantsFingerprint(); + await syncUnmodeledScoringConstantDrift(env, { unmodeledConstants: ["NOVELTY_BONUS_SCALAR"] }); + const resolved = await syncUnmodeledScoringConstantDrift(env, { unmodeledConstants: [] }); + expect(resolved).toMatchObject({ fingerprint, status: "resolved" }); + }); + + it("preserves linked issue metadata across unmodeled updates", async () => { + const env = createTestEnv(); + const fingerprint = await unmodeledScoringConstantsFingerprint(); + await syncUnmodeledScoringConstantDrift(env, { unmodeledConstants: ["ALPHA"] }); + await updateUpstreamDriftReportIssue(env, fingerprint, { + number: 811, + url: "https://github.com/JSONbored/gittensory/issues/811", + }); + const updated = await syncUnmodeledScoringConstantDrift(env, { unmodeledConstants: ["ALPHA", "BETA"] }); + expect(updated).toMatchObject({ + issueNumber: 811, + issueUrl: "https://github.com/JSONbored/gittensory/issues/811", + payload: expect.objectContaining({ unmodeledUpstreamConstants: ["ALPHA", "BETA"] }), + }); + }); +}); diff --git a/test/unit/upstream-ruleset.test.ts b/test/unit/upstream-ruleset.test.ts index ac6951f0fc..4e4f7fb032 100644 --- a/test/unit/upstream-ruleset.test.ts +++ b/test/unit/upstream-ruleset.test.ts @@ -52,6 +52,39 @@ describe("upstream ruleset drift tracking", () => { }); }); + it("opens an upstream drift report when upstream defines scoring constants gittensory does not model", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2026-05-30T00:00:00.000Z")); + const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "token" }); + const files = fixtures("58", 0.01); + files["gittensor/constants.py"] += "\nNOVELTY_BONUS_SCALAR = 3\n"; + vi.stubGlobal("fetch", upstreamFetch(files)); + + await refreshUpstreamDrift(env); + const reports = await listUpstreamDriftReports(env, 10); + const status = await loadUpstreamStatus(env); + + expect(reports).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + status: "open", + severity: "medium", + affectedAreas: ["scoring_model"], + summary: expect.stringContaining("NOVELTY_BONUS_SCALAR"), + payload: expect.objectContaining({ + kind: "unmodeled_scoring_constants", + unmodeledUpstreamConstants: ["NOVELTY_BONUS_SCALAR"], + }), + }), + ]), + ); + expect(status).toMatchObject({ + status: "drift_detected", + openReportCount: 1, + affectedAreas: ["scoring_model"], + }); + }); + it("detects high-severity scoring and registry drift between semantic rulesets", async () => { vi.useFakeTimers({ toFake: ["Date"] }); const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "token" }); From 2356936089fa6fb79903bc4db619d34b20068c5d Mon Sep 17 00:00:00 2001 From: kiannidev <156195510+kiannidev@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:29:41 +0200 Subject: [PATCH 2/3] test(scoring): cover unmodeled drift branches for codecov patch gate Exercise env fallbacks, resolve/no-op paths, error-source skip, and long constant summaries so changed lines meet the 97% patch coverage requirement. Co-authored-by: Cursor --- test/unit/unmodeled-scoring-drift.test.ts | 45 +++++++++++++++++++++++ test/unit/upstream-ruleset.test.ts | 12 ++++++ 2 files changed, 57 insertions(+) diff --git a/test/unit/unmodeled-scoring-drift.test.ts b/test/unit/unmodeled-scoring-drift.test.ts index 5d21f57148..8d24e5977e 100644 --- a/test/unit/unmodeled-scoring-drift.test.ts +++ b/test/unit/unmodeled-scoring-drift.test.ts @@ -57,4 +57,49 @@ describe("unmodeled scoring constant drift", () => { payload: expect.objectContaining({ unmodeledUpstreamConstants: ["ALPHA", "BETA"] }), }); }); + + it("uses upstream env defaults when source metadata is omitted", async () => { + const env = createTestEnv({ + GITTENSOR_UPSTREAM_REPO: "entrius/gittensor", + GITTENSOR_UPSTREAM_REF: "staging", + }); + const report = await syncUnmodeledScoringConstantDrift(env, { unmodeledConstants: ["ALPHA"] }); + expect(report?.payload.source).toEqual({ + repo: "entrius/gittensor", + ref: "staging", + commitSha: null, + }); + }); + + it("falls back to baked-in upstream repo/ref when env overrides are empty", async () => { + const env = createTestEnv({ GITTENSOR_UPSTREAM_REPO: "", GITTENSOR_UPSTREAM_REF: "" }); + const report = await syncUnmodeledScoringConstantDrift(env, { unmodeledConstants: ["ALPHA"] }); + expect(report?.payload.source).toEqual({ + repo: "entrius/gittensor", + ref: "test", + commitSha: null, + }); + }); + + it("returns null when resolving with no prior drift report", async () => { + const env = createTestEnv(); + expect(await syncUnmodeledScoringConstantDrift(env, { unmodeledConstants: [] })).toBeNull(); + }); + + it("returns an already-resolved report without rewriting it", async () => { + const env = createTestEnv(); + const fingerprint = await unmodeledScoringConstantsFingerprint(); + await syncUnmodeledScoringConstantDrift(env, { unmodeledConstants: ["ALPHA"] }); + await syncUnmodeledScoringConstantDrift(env, { unmodeledConstants: [] }); + const again = await syncUnmodeledScoringConstantDrift(env, { unmodeledConstants: [] }); + expect(again).toMatchObject({ fingerprint, status: "resolved" }); + }); + + it("truncates long unmodeled-constant lists in the summary", async () => { + const env = createTestEnv(); + const names = Array.from({ length: 13 }, (_, index) => `CONST_${index}`); + const report = await syncUnmodeledScoringConstantDrift(env, { unmodeledConstants: names }); + expect(report?.summary).toMatch(/, …$/); + expect(report?.severity).toBe("high"); + }); }); diff --git a/test/unit/upstream-ruleset.test.ts b/test/unit/upstream-ruleset.test.ts index 4e4f7fb032..52b6a28d2b 100644 --- a/test/unit/upstream-ruleset.test.ts +++ b/test/unit/upstream-ruleset.test.ts @@ -85,6 +85,18 @@ describe("upstream ruleset drift tracking", () => { }); }); + it("skips unmodeled-constant drift sync when the constants source fetch failed", async () => { + const env = createTestEnv(); + vi.stubGlobal("fetch", upstreamFetch(fixtures("58", 0.01))); + await refreshUpstreamSourceSnapshots(env); + + vi.stubGlobal("fetch", upstreamFailedFetch()); + const result = await refreshUpstreamDrift(env); + + expect(result.sources.find((source) => source.sourceKey === "constants")?.status).toBe("error"); + expect((await listUpstreamDriftReports(env, 10)).some((report) => report.payload.kind === "unmodeled_scoring_constants")).toBe(false); + }); + it("detects high-severity scoring and registry drift between semantic rulesets", async () => { vi.useFakeTimers({ toFake: ["Date"] }); const env = createTestEnv({ GITHUB_PUBLIC_TOKEN: "token" }); From 3988491ff1d8dd781fa65b9291b3e856e090fd93 Mon Sep 17 00:00:00 2001 From: kiannidev <156195510+kiannidev@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:14:01 +0200 Subject: [PATCH 3/3] fix(scoring): use env upstream metadata for unmodeled drift sync Stop hardcoding entrius/gittensor:test in refreshScoringModelSnapshot; let syncUnmodeledScoringConstantDrift resolve repo/ref from env defaults. Addresses review on #832. Co-authored-by: Cursor --- src/scoring/model.ts | 1 - test/unit/scoring.test.ts | 10 ++++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/scoring/model.ts b/src/scoring/model.ts index 737a71b25c..c427b2305b 100644 --- a/src/scoring/model.ts +++ b/src/scoring/model.ts @@ -105,7 +105,6 @@ export async function refreshScoringModelSnapshot(env: Env): Promise { - const env = createTestEnv(); + const env = createTestEnv({ + GITTENSOR_UPSTREAM_REPO: "custom/upstream", + GITTENSOR_UPSTREAM_REF: "staging", + }); vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { const url = input.toString(); if (url.includes("constants.py")) return new Response("SRC_TOK_SATURATION_SCALE = 58.0\nNOVELTY_BONUS_SCALAR = 3\n"); @@ -163,7 +166,10 @@ MAX_CODE_DENSITY_MULTIPLIER = 1.15 expect((await listUpstreamDriftReports(env, 10)).find((report) => report.fingerprint === fingerprint)).toMatchObject({ status: "open", affectedAreas: ["scoring_model"], - payload: expect.objectContaining({ unmodeledUpstreamConstants: ["NOVELTY_BONUS_SCALAR"] }), + payload: expect.objectContaining({ + unmodeledUpstreamConstants: ["NOVELTY_BONUS_SCALAR"], + source: { repo: "custom/upstream", ref: "staging", commitSha: null }, + }), }); });