Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions migrations/0012_official_miner_detection_cache.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
CREATE TABLE IF NOT EXISTS official_miner_detections (login TEXT PRIMARY KEY NOT NULL, status TEXT NOT NULL CHECK (status IN ('confirmed', 'not_found', 'unavailable')), snapshot_json TEXT NOT NULL DEFAULT '{}', error TEXT, fetched_at TEXT NOT NULL, expires_at TEXT NOT NULL, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
107 changes: 107 additions & 0 deletions src/db/repositories.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
issueQualityReports,
issues,
githubRateLimitObservations,
officialMinerDetections,
pullRequestFiles,
pullRequestDetailSyncState,
pullRequestReviews,
Expand Down Expand Up @@ -90,6 +91,7 @@ import type {
ScoringModelSnapshotRecord,
SignalSnapshotRecord,
} from "../types";
import type { GittensorContributorSnapshot, OfficialGittensorMinerDetection } from "../gittensor/api";
import { jsonString, nowIso, parseJson, repoParts } from "../utils/json";

const MAX_STORED_BODY_CHARS = 4000;
Expand Down Expand Up @@ -745,6 +747,101 @@ export async function hasRecentAuditEvent(env: Env, actor: string, eventType: st
return rows.length > 0;
}

export async function getFreshOfficialMinerDetection(env: Env, login: string, now = nowIso()): Promise<OfficialGittensorMinerDetection | null> {
const [row] = await getDb(env.DB).select().from(officialMinerDetections).where(and(eq(officialMinerDetections.login, login.toLowerCase()), gte(officialMinerDetections.expiresAt, now))).limit(1);
return row ? toOfficialMinerDetection(row) : null;
}

export async function upsertOfficialMinerDetection(env: Env, login: string, detection: OfficialGittensorMinerDetection, ttlMs: number, fetchedAtMs = Date.now()): Promise<void> {
const fetchedAt = new Date(fetchedAtMs).toISOString();
const cacheableDetection = toCacheableOfficialMinerDetection(detection);
const values = {
login: login.toLowerCase(), status: cacheableDetection.status,
snapshotJson: cacheableDetection.status === "confirmed" ? jsonString(cacheableDetection.snapshot) : "{}",
error: cacheableDetection.status === "unavailable" ? cacheableDetection.error : null, fetchedAt,
expiresAt: new Date(fetchedAtMs + ttlMs).toISOString(), updatedAt: fetchedAt,
};
await getDb(env.DB).insert(officialMinerDetections).values(values).onConflictDoUpdate({ target: officialMinerDetections.login, set: values });
}

function toCacheableOfficialMinerDetection(detection: OfficialGittensorMinerDetection): OfficialGittensorMinerDetection {
return detection.status === "confirmed" ? { status: "confirmed", snapshot: toCacheableGittensorSnapshot(detection.snapshot) } : detection;
}

function toCacheableGittensorSnapshot(snapshot: Partial<GittensorContributorSnapshot>): GittensorContributorSnapshot {
return {
source: "gittensor_api",
githubId: String(snapshot.githubId ?? ""),
githubUsername: String(snapshot.githubUsername ?? ""),
uid: optionalNumber(snapshot.uid),
failedReason: typeof snapshot.failedReason === "string" ? snapshot.failedReason : snapshot.failedReason === null ? null : undefined,
evaluatedAt: typeof snapshot.evaluatedAt === "string" ? snapshot.evaluatedAt : undefined,
updatedAt: typeof snapshot.updatedAt === "string" ? snapshot.updatedAt : undefined,
isEligible: Boolean(snapshot.isEligible),
credibility: finiteNumber(snapshot.credibility),
eligibleRepoCount: finiteNumber(snapshot.eligibleRepoCount),
issueDiscoveryScore: finiteNumber(snapshot.issueDiscoveryScore),
issueTokenScore: finiteNumber(snapshot.issueTokenScore),
issueCredibility: finiteNumber(snapshot.issueCredibility),
isIssueEligible: Boolean(snapshot.isIssueEligible),
issueEligibleRepoCount: finiteNumber(snapshot.issueEligibleRepoCount),
alphaPerDay: finiteNumber(snapshot.alphaPerDay),
taoPerDay: finiteNumber(snapshot.taoPerDay),
usdPerDay: finiteNumber(snapshot.usdPerDay),
totals: {
pullRequests: finiteNumber(snapshot.totals?.pullRequests),
mergedPullRequests: finiteNumber(snapshot.totals?.mergedPullRequests),
openPullRequests: finiteNumber(snapshot.totals?.openPullRequests),
closedPullRequests: finiteNumber(snapshot.totals?.closedPullRequests),
openIssues: finiteNumber(snapshot.totals?.openIssues),
closedIssues: finiteNumber(snapshot.totals?.closedIssues),
solvedIssues: finiteNumber(snapshot.totals?.solvedIssues),
validSolvedIssues: finiteNumber(snapshot.totals?.validSolvedIssues),
},
repositories: Array.isArray(snapshot.repositories)
? snapshot.repositories.map((repo) => ({
repoFullName: String(repo.repoFullName ?? ""),
pullRequests: finiteNumber(repo.pullRequests),
mergedPullRequests: finiteNumber(repo.mergedPullRequests),
openPullRequests: finiteNumber(repo.openPullRequests),
closedPullRequests: finiteNumber(repo.closedPullRequests),
openIssues: finiteNumber(repo.openIssues),
closedIssues: finiteNumber(repo.closedIssues),
solvedIssues: finiteNumber(repo.solvedIssues),
validSolvedIssues: finiteNumber(repo.validSolvedIssues),
isEligible: Boolean(repo.isEligible),
isIssueEligible: Boolean(repo.isIssueEligible),
credibility: finiteNumber(repo.credibility),
issueCredibility: finiteNumber(repo.issueCredibility),
totalScore: finiteNumber(repo.totalScore),
baseTotalScore: finiteNumber(repo.baseTotalScore),
}))
: [],
pullRequests: Array.isArray(snapshot.pullRequests)
? snapshot.pullRequests.map((pr) => ({
repoFullName: String(pr.repoFullName ?? ""),
number: finiteNumber(pr.number),
title: String(pr.title ?? ""),
state: String(pr.state ?? ""),
mergedAt: typeof pr.mergedAt === "string" ? pr.mergedAt : pr.mergedAt === null ? null : undefined,
label: typeof pr.label === "string" ? pr.label : pr.label === null ? null : undefined,
score: finiteNumber(pr.score),
baseScore: finiteNumber(pr.baseScore),
tokenScore: finiteNumber(pr.tokenScore),
}))
: [],
issueLabels: Array.isArray(snapshot.issueLabels) ? snapshot.issueLabels.filter((label): label is string => typeof label === "string") : [],
};
}

function finiteNumber(value: unknown): number {
return typeof value === "number" && Number.isFinite(value) ? value : 0;
}

function optionalNumber(value: unknown): number | undefined {
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
}

export async function recordAiUsageEvent(
env: Env,
event: {
Expand Down Expand Up @@ -2118,6 +2215,16 @@ function toInstallationHealthRecord(row: typeof installationHealth.$inferSelect)
};
}

function toOfficialMinerDetection(row: typeof officialMinerDetections.$inferSelect): OfficialGittensorMinerDetection {
if (row.status === "confirmed") {
const snapshot = parseJson<Partial<GittensorContributorSnapshot> | null>(row.snapshotJson, null);
return snapshot?.githubId && snapshot.githubUsername
? { status: "confirmed", snapshot: toCacheableGittensorSnapshot(snapshot) }
: { status: "unavailable", error: "cached Gittensor miner snapshot is invalid" };
}
return row.status === "unavailable" ? { status: "unavailable", error: row.error ?? "cached Gittensor API unavailable" } : { status: "not_found" };
}

function toAuthSessionRecord(row: typeof authSessions.$inferSelect): AuthSessionRecord {
return {
id: row.id,
Expand Down
7 changes: 7 additions & 0 deletions src/db/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,13 @@ export const contributorScoringProfiles = sqliteTable("contributor_scoring_profi
generatedAt: text("generated_at").notNull().default("CURRENT_TIMESTAMP"),
});

export const officialMinerDetections = sqliteTable("official_miner_detections", {
login: text("login").primaryKey(), status: text("status").notNull(),
snapshotJson: text("snapshot_json").notNull().default("{}"), error: text("error"),
fetchedAt: text("fetched_at").notNull(), expiresAt: text("expires_at").notNull(),
updatedAt: text("updated_at").notNull().default("CURRENT_TIMESTAMP"),
});

export const issueQualityReports = sqliteTable(
"issue_quality_reports",
{
Expand Down
47 changes: 36 additions & 11 deletions src/queue/processors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {
countOpenIssues,
countOpenPullRequests,
getLatestRepoGithubTotalsSnapshot,
getFreshOfficialMinerDetection,
getPullRequest,
getRepository,
getRepositorySettings,
Expand All @@ -27,6 +28,7 @@ import {
persistSignalSnapshot,
recordWebhookEvent,
replaceCollisionEdges,
upsertOfficialMinerDetection,
upsertBurdenForecast,
upsertContributorEvidence,
upsertContributorScoringProfile,
Expand All @@ -43,7 +45,7 @@ import {
refreshContributorActivity,
refreshInstallationHealth,
} from "../github/backfill";
import { contributorRepoStatsFromGittensor, fetchGittensorContributorSnapshot, fetchOfficialGittensorMiner, type GittensorContributorSnapshot } from "../gittensor/api";
import { contributorRepoStatsFromGittensor, fetchGittensorContributorSnapshot, fetchOfficialGittensorMiner, type GittensorContributorSnapshot, type OfficialGittensorMinerDetection } from "../gittensor/api";
import { createOrUpdateCheckRun, getInstallationId } from "../github/app";
import { createOrUpdateAgentCommandComment, createOrUpdatePrIntelligenceComment } from "../github/comments";
import {
Expand Down Expand Up @@ -85,6 +87,9 @@ import { decidePublicSurface } from "../signals/settings-preview";
import type { ContributorEvidenceRecord, GitHubWebhookPayload, JobMessage, JsonValue } from "../types";
import { errorMessage } from "../utils/json";

const OFFICIAL_MINER_DETECTION_TTL_MS = 5 * 60 * 1000;
const OFFICIAL_MINER_DETECTION_UNAVAILABLE_TTL_MS = 60 * 1000;

export async function processJob(env: Env, message: JobMessage): Promise<void> {
switch (message.type) {
case "refresh-registry":
Expand Down Expand Up @@ -549,16 +554,12 @@ async function maybePublishPrPublicSurface(
}
if (!author) return;

const official = await fetchOfficialGittensorMiner(author);
const official = await getCachedOfficialMinerDetection(env, author, {
targetKey: `${repoFullName}#${pr.number}`,
deliveryId: webhook.deliveryId,
});
if (official.status === "unavailable") {
await recordAuditEvent(env, {
eventType: "github_app.miner_detection_unavailable",
actor: author,
targetKey: `${repoFullName}#${pr.number}`,
outcome: "error",
detail: official.error,
metadata: { deliveryId: webhook.deliveryId },
});
await auditPrVisibilitySkip(env, repoFullName, pr.number, author, "miner_detection_unavailable", webhook.deliveryId);
return;
}
if (official.status !== "confirmed") {
Expand Down Expand Up @@ -686,7 +687,9 @@ async function maybeProcessGittensoryMentionCommand(env: Env, deliveryId: string

const [repo, cachedPullRequest] = await Promise.all([getRepository(env, repoFullName), getPullRequest(env, repoFullName, issue.number)]);
const pullRequestAuthor = cachedPullRequest?.authorLogin ?? issue.user?.login ?? null;
const official = pullRequestAuthor ? await fetchOfficialGittensorMiner(pullRequestAuthor) : undefined;
const official = pullRequestAuthor
? await getCachedOfficialMinerDetection(env, pullRequestAuthor, { targetKey: `${repoFullName}#${issue.number}`, deliveryId })
: undefined;
const authorization = isAuthorizedCommandActor({
commenterLogin: commenter,
commenterAssociation: payload.comment?.author_association ?? issue.author_association,
Expand Down Expand Up @@ -755,6 +758,28 @@ async function auditPrVisibilitySkip(
});
}

async function getCachedOfficialMinerDetection(env: Env, login: string, context: { targetKey: string; deliveryId: string }): Promise<OfficialGittensorMinerDetection> {
const cached = await getFreshOfficialMinerDetection(env, login);
if (cached) {
await auditMinerDetectionCache(env, "github_app.miner_detection_cache_hit", login, context, cached.status);
if (cached.status === "unavailable") await auditMinerDetectionUnavailable(env, login, context, cached.error);
return cached;
}
await auditMinerDetectionCache(env, "github_app.miner_detection_cache_miss", login, context, "miss");
const detection = await fetchOfficialGittensorMiner(login);
await upsertOfficialMinerDetection(env, login, detection, detection.status === "unavailable" ? OFFICIAL_MINER_DETECTION_UNAVAILABLE_TTL_MS : OFFICIAL_MINER_DETECTION_TTL_MS);
if (detection.status === "unavailable") await auditMinerDetectionUnavailable(env, login, context, detection.error);
return detection;
}

async function auditMinerDetectionUnavailable(env: Env, actor: string, context: { targetKey: string; deliveryId: string }, detail: string): Promise<void> {
await recordAuditEvent(env, { eventType: "github_app.miner_detection_unavailable", actor, targetKey: context.targetKey, outcome: "error", detail, metadata: { deliveryId: context.deliveryId } });
}

async function auditMinerDetectionCache(env: Env, eventType: "github_app.miner_detection_cache_hit" | "github_app.miner_detection_cache_miss", actor: string, context: { targetKey: string; deliveryId: string }, detail: string): Promise<void> {
await recordAuditEvent(env, { eventType, actor, targetKey: context.targetKey, outcome: "completed", detail, metadata: { deliveryId: context.deliveryId } });
}

function officialGittensorContributorDetection(
snapshot: GittensorContributorSnapshot,
currentPr: Awaited<ReturnType<typeof upsertPullRequestFromGitHub>>,
Expand Down
Loading