diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 5393df9f21..cc5322bd15 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -129,6 +129,7 @@ import type { ProductUsageSurface, ProductUsageSurfaceActivationFunnel, ProductUsageSurfaceRetention, + PullRequestFilePathRecord, PullRequestFileRecord, PullRequestDetailSyncStateRecord, PullRequestRecord, @@ -2657,6 +2658,28 @@ export async function listPullRequestFiles(env: Env, fullName: string, pullNumbe return rows.map(toPullRequestFileRecord); } +export async function listRepoPullRequestFilePaths( + env: Env, + fullName: string, + options: { pullNumbers?: number[] | undefined; limit?: number | undefined } = {}, +): Promise { + const db = getDb(env.DB); + const pullNumbers = [...new Set(options.pullNumbers ?? [])].filter((number) => Number.isInteger(number) && number > 0); + if (options.pullNumbers && pullNumbers.length === 0) return []; + const where = pullNumbers.length > 0 + ? and(eq(pullRequestFiles.repoFullName, fullName), inArray(pullRequestFiles.pullNumber, pullNumbers)) + : eq(pullRequestFiles.repoFullName, fullName); + return db + .select({ + repoFullName: pullRequestFiles.repoFullName, + pullNumber: pullRequestFiles.pullNumber, + path: pullRequestFiles.path, + }) + .from(pullRequestFiles) + .where(where) + .limit(Math.max(0, Math.min(options.limit ?? 500, 500))); +} + export async function listRepoPullRequestFiles(env: Env, fullName: string): Promise { const db = getDb(env.DB); const rows = await db.select().from(pullRequestFiles).where(eq(pullRequestFiles.repoFullName, fullName)).limit(2000); diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 0abc37e874..66581eefe6 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -30,7 +30,7 @@ import { listRecentMergedPullRequests, updatePullRequestSlopAssessment, listRepoLabels, - listRepoPullRequestFiles, + listRepoPullRequestFilePaths, listRepoSyncStates, listRepoSyncSegments, listRepositories, @@ -140,6 +140,7 @@ import { detectGittensorContributor, PR_PANEL_RETRIGGER_MARKER, unionScopedOverlapClusters, + type ContributorProfile, } from "../signals/engine"; import { buildIssueSlopAssessment, buildSlopAssessment, type SlopBand } from "../signals/slop"; import { runGittensoryAiSlopAdvisory } from "../services/ai-slop"; @@ -148,7 +149,7 @@ import { loadRepoFocusManifest } from "../signals/focus-manifest-loader"; import { buildFocusManifestGuidance, resolveEffectiveSettings } from "../signals/focus-manifest"; import type { LocalBranchAnalysisInput } from "../signals/local-branch"; import { runGittensoryAiReview } from "../services/ai-review"; -import type { AdvisoryFinding, ContributorEvidenceRecord, DetectedNotificationEvent, GitHubWebhookPayload, JobMessage, JsonValue, PullRequestRecord, RepositorySettings } from "../types"; +import type { AdvisoryFinding, ContributorEvidenceRecord, ContributorRepoStatRecord, DetectedNotificationEvent, GitHubWebhookPayload, IssueRecord, JobMessage, JsonValue, PullRequestFilePathRecord, PullRequestRecord, RepositoryRecord, RepositorySettings } from "../types"; import { sha256Hex } from "../utils/crypto"; import { errorMessage, nowIso } from "../utils/json"; @@ -545,6 +546,41 @@ async function discoverContributorLogins(env: Env): Promise { return [...new Set([...pullRequests, ...issues].flatMap((record) => (record.authorLogin ? [record.authorLogin] : [])))].slice(0, 200); } +const CONTRIBUTOR_EVIDENCE_MAX_PR_FILE_PATHS = 2000; +const CONTRIBUTOR_EVIDENCE_PR_FILE_PATHS_PER_REPO = 200; + +async function loadContributorPullRequestFilePaths( + env: Env, + args: { + login: string; + profile: ContributorProfile; + pullRequests: PullRequestRecord[]; + issues: IssueRecord[]; + repoStats: ContributorRepoStatRecord[]; + repositories: RepositoryRecord[]; + }, +): Promise { + const pullNumbersByRepo = new Map>(); + for (const pr of args.pullRequests) { + if (pr.authorLogin?.toLowerCase() !== args.login.toLowerCase()) continue; + const key = pr.repoFullName.toLowerCase(); + const current = pullNumbersByRepo.get(key) ?? new Set(); + current.add(pr.number); + pullNumbersByRepo.set(key, current); + } + const files: PullRequestFilePathRecord[] = []; + for (const repoFullName of evidenceGraphTouchedRepoFullNames(args)) { + if (files.length >= CONTRIBUTOR_EVIDENCE_MAX_PR_FILE_PATHS) break; + const remaining = CONTRIBUTOR_EVIDENCE_MAX_PR_FILE_PATHS - files.length; + const repoFiles = await listRepoPullRequestFilePaths(env, repoFullName, { + pullNumbers: [...(pullNumbersByRepo.get(repoFullName.toLowerCase()) ?? [])], + limit: Math.min(CONTRIBUTOR_EVIDENCE_PR_FILE_PATHS_PER_REPO, remaining), + }); + files.push(...repoFiles); + } + return files; +} + async function buildContributorEvidence(env: Env, login?: string): Promise { const [allPullRequests, allIssues, repositories, syncStates, allBounties, snapshot] = await Promise.all([ listAllPullRequests(env), @@ -569,18 +605,14 @@ async function buildContributorEvidence(env: Env, login?: string): Promise ]); const repoStats = authoritativeContributorRepoStats(gittensorSnapshot, cachedRepoStats); const profile = buildContributorProfile(contributorLogin, github, contributorPullRequests, contributorIssues, repoStats, gittensorSnapshot); - const pullRequestFiles = ( - await Promise.all( - evidenceGraphTouchedRepoFullNames({ - login: contributorLogin, - profile, - pullRequests: contributorPullRequests, - issues: contributorIssues, - repoStats, - repositories, - }).map((repoFullName) => listRepoPullRequestFiles(env, repoFullName)), - ) - ).flat(); + const pullRequestFiles = await loadContributorPullRequestFilePaths(env, { + login: contributorLogin, + profile, + pullRequests: contributorPullRequests, + issues: contributorIssues, + repoStats, + repositories, + }); const fit = buildContributorFit(profile, repositories, allIssues, allPullRequests, syncStates, repoStats, allBounties, issueQualityByRepo); const scoringProfile = buildContributorScoringProfile({ login: contributorLogin, fit, scoringSnapshot: snapshot }); const outcomeHistory = buildContributorOutcomeHistory({ login: contributorLogin, profile, repositories, pullRequests: allPullRequests, issues: allIssues, repoStats, cachedRepoStats }); diff --git a/src/services/contributor-evidence-graph.ts b/src/services/contributor-evidence-graph.ts index 75346f992b..44db6b209f 100644 --- a/src/services/contributor-evidence-graph.ts +++ b/src/services/contributor-evidence-graph.ts @@ -3,7 +3,7 @@ import type { ContributorOutcomeHistory, ContributorProfile, RoleContext } from import type { ContributorRepoStatRecord, IssueRecord, - PullRequestFileRecord, + PullRequestFilePathRecord, PullRequestRecord, RepositoryRecord, RepoSyncStateRecord, @@ -153,7 +153,7 @@ export type ContributorEvidenceGraphInput = { issues?: IssueRecord[] | undefined; repoStats?: ContributorRepoStatRecord[] | undefined; syncStates?: RepoSyncStateRecord[] | undefined; - pullRequestFiles?: PullRequestFileRecord[] | undefined; + pullRequestFiles?: PullRequestFilePathRecord[] | undefined; gittensorSnapshot?: GittensorContributorSnapshot | null | undefined; }; @@ -374,7 +374,7 @@ function preferredLabelEdges(buckets: LabelBucket[], generatedAt: string): Contr ); } -function buildPathEdges(login: string, contributorPullRequests: PullRequestRecord[], files: PullRequestFileRecord[], generatedAt: string): ContributorEvidenceGraphPath[] { +function buildPathEdges(login: string, contributorPullRequests: PullRequestRecord[], files: PullRequestFilePathRecord[], generatedAt: string): ContributorEvidenceGraphPath[] { const prByKey = new Map(contributorPullRequests.filter((pr) => sameLogin(pr.authorLogin, login)).map((pr) => [`${pr.repoFullName.toLowerCase()}#${pr.number}`, pr])); const buckets = new Map(); for (const file of files) { diff --git a/src/services/decision-pack.ts b/src/services/decision-pack.ts index 9a6ef50d41..56c651efeb 100644 --- a/src/services/decision-pack.ts +++ b/src/services/decision-pack.ts @@ -10,7 +10,7 @@ import { listContributorPullRequests, listContributorRepoStats, listLatestRepoGithubTotalsSnapshots, - listRepoPullRequestFiles, + listRepoPullRequestFilePaths, listRepositories, listRepoSyncSegments, listRepoSyncStates, @@ -57,7 +57,7 @@ import type { ContributorRepoStatRecord, IssueRecord, JsonValue, - PullRequestFileRecord, + PullRequestFilePathRecord, PullRequestRecord, RepositoryRecord, RepoGithubTotalsSnapshotRecord, @@ -71,6 +71,8 @@ import { nowIso } from "../utils/json"; export const CONTRIBUTOR_DECISION_PACK_SIGNAL = "contributor-decision-pack"; export const DECISION_PACK_MAX_AGE_MS = 6 * 60 * 60 * 1000; const DEFAULT_OSS_EMISSION_SHARE = DEFAULT_SCORING_CONSTANTS.OSS_EMISSION_SHARE ?? 0.9; +const DECISION_PACK_MAX_PR_FILE_PATHS = 2000; +const DECISION_PACK_PR_FILE_PATHS_PER_REPO = 200; function resolveOssEmissionShare(constants: Record | undefined): number { const value = constants?.OSS_EMISSION_SHARE; @@ -79,6 +81,38 @@ function resolveOssEmissionShare(constants: Record | undefined): export const DECISION_PACK_REBUILD_DEBOUNCE_MS = 15 * 1000; const pendingDecisionPackRebuilds = new Map>(); +async function loadContributorPullRequestFilePaths( + env: Env, + args: { + login: string; + profile: ContributorProfile; + pullRequests: PullRequestRecord[]; + issues: IssueRecord[]; + repoStats: ContributorRepoStatRecord[]; + repositories: RepositoryRecord[]; + }, +): Promise { + const pullNumbersByRepo = new Map>(); + for (const pr of args.pullRequests) { + if (pr.authorLogin?.toLowerCase() !== args.login.toLowerCase()) continue; + const key = pr.repoFullName.toLowerCase(); + const current = pullNumbersByRepo.get(key) ?? new Set(); + current.add(pr.number); + pullNumbersByRepo.set(key, current); + } + const files: PullRequestFilePathRecord[] = []; + for (const repoFullName of evidenceGraphTouchedRepoFullNames(args)) { + if (files.length >= DECISION_PACK_MAX_PR_FILE_PATHS) break; + const remaining = DECISION_PACK_MAX_PR_FILE_PATHS - files.length; + const repoFiles = await listRepoPullRequestFilePaths(env, repoFullName, { + pullNumbers: [...(pullNumbersByRepo.get(repoFullName.toLowerCase()) ?? [])], + limit: Math.min(DECISION_PACK_PR_FILE_PATHS_PER_REPO, remaining), + }); + files.push(...repoFiles); + } + return files; +} + export type DecisionRecommendation = "pursue" | "cleanup_first" | "maintainer_lane" | "avoid_for_now" | "watch"; export type DecisionActionKind = "cleanup_existing_prs" | "land_existing_prs" | "open_new_direct_pr" | "file_issue_discovery" | "maintainer_lane_improve_repo" | "maintainer_cut_readiness"; export type DecisionPackFreshness = "fresh" | "stale" | "rebuilding" | "missing"; @@ -442,18 +476,14 @@ export async function buildAndPersistContributorDecisionPack(env: Env, login: st repositories.filter((repo) => repo.isRegistered).map((repo) => repo.fullName), ); const profile = buildContributorProfile(login, github, contributorPullRequests, contributorIssues, repoStats, gittensorSnapshot); - const pullRequestFiles = ( - await Promise.all( - evidenceGraphTouchedRepoFullNames({ - login, - profile, - pullRequests: contributorPullRequests, - issues: contributorIssues, - repoStats, - repositories, - }).map((repoFullName) => listRepoPullRequestFiles(env, repoFullName)), - ) - ).flat(); + const pullRequestFiles = await loadContributorPullRequestFilePaths(env, { + login, + profile, + pullRequests: contributorPullRequests, + issues: contributorIssues, + repoStats, + repositories, + }); const outcomeHistory = buildContributorOutcomeHistory({ login, profile, @@ -547,7 +577,7 @@ function buildContributorDecisionPack(args: { contributorPullRequests: Parameters[0]["pullRequests"]; contributorIssues: Parameters[0]["issues"]; repoStats?: ContributorRepoStatRecord[] | undefined; - pullRequestFiles?: PullRequestFileRecord[] | undefined; + pullRequestFiles?: PullRequestFilePathRecord[] | undefined; gittensorSnapshot?: Awaited> | undefined; issueQualityByRepo?: Map | undefined; openPrMonitor: ContributorOpenPrMonitor; diff --git a/src/types.ts b/src/types.ts index a27bac5832..4d251e6e22 100644 --- a/src/types.ts +++ b/src/types.ts @@ -703,6 +703,8 @@ export type PullRequestFileRecord = { payload: Record; }; +export type PullRequestFilePathRecord = Pick; + export type PullRequestReviewRecord = { id: string; repoFullName: string; diff --git a/test/unit/db-persistence.test.ts b/test/unit/db-persistence.test.ts index 16169db12a..d3b999c590 100644 --- a/test/unit/db-persistence.test.ts +++ b/test/unit/db-persistence.test.ts @@ -4,6 +4,7 @@ import { getOpenUpstreamDriftReportByFingerprint, listContributorRepoStats, listLatestRepoGithubTotalsSnapshots, + listRepoPullRequestFilePaths, persistBountyLifecycleEvent, persistRegistryDriftEvents, persistRepoGithubTotalsSnapshot, @@ -11,8 +12,11 @@ import { upsertContributorRepoStat, upsertContributorScoringProfile, upsertIssueQualityReport, + upsertPullRequestFile, upsertUpstreamDriftReport, } from "../../src/db/repositories"; +import { buildContributorEvidenceGraph } from "../../src/services/contributor-evidence-graph"; +import type { PullRequestFileRecord, PullRequestRecord, RepositoryRecord } from "../../src/types"; import { createTestEnv } from "../helpers/d1"; describe("database persistence helpers", () => { @@ -121,8 +125,113 @@ describe("database persistence helpers", () => { }), ]); }); + + it("caps contributor-graph file-path loading and still builds path edges from the capped set", async () => { + const env = createTestEnv(); + const repoFullName = "owner/big-repo"; + // Seed more than the hard cap (500) of distinct file paths across several authored PRs. + const seededPaths = 600; + const pullNumbers = [1, 2, 3, 4, 5, 6]; + for (let index = 0; index < seededPaths; index += 1) { + const pullNumber = pullNumbers[index % pullNumbers.length]!; + await upsertPullRequestFile(env, pullRequestFile(repoFullName, pullNumber, `src/path-${String(index).padStart(4, "0")}.ts`)); + } + + // Hard cap: the path-only query never returns more than 500 rows even when more exist. + const allPaths = await listRepoPullRequestFilePaths(env, repoFullName, { pullNumbers }); + expect(allPaths).toHaveLength(500); + expect(allPaths.every((entry) => entry.repoFullName === repoFullName && pullNumbers.includes(entry.pullNumber) && entry.path.length > 0)).toBe(true); + + // A smaller requested limit is honored; a too-large limit is clamped down to the cap. + const smallLimit = await listRepoPullRequestFilePaths(env, repoFullName, { pullNumbers, limit: 50 }); + expect(smallLimit).toHaveLength(50); + const oversizedLimit = await listRepoPullRequestFilePaths(env, repoFullName, { pullNumbers, limit: 5000 }); + expect(oversizedLimit).toHaveLength(500); + + // Filtering by a subset of pull numbers still respects the cap and only returns matching PRs. + const subset = await listRepoPullRequestFilePaths(env, repoFullName, { pullNumbers: [1, 2], limit: 500 }); + expect(subset.length).toBeGreaterThan(0); + expect(subset.every((entry) => entry.pullNumber === 1 || entry.pullNumber === 2)).toBe(true); + + // The capped, path-only set still feeds buildPathEdges correctly via the evidence graph. + const cappedPaths = await listRepoPullRequestFilePaths(env, repoFullName, { pullNumbers, limit: 500 }); + const graph = buildContributorEvidenceGraph({ + login: "dev", + generatedAt: "2026-05-30T00:00:00.000Z", + profile: graphProfile(repoFullName), + outcomeHistory: graphHistory(), + roleContexts: [], + repositories: [graphRepo(repoFullName)], + pullRequests: pullNumbers.map((number) => graphPr(repoFullName, number)), + pullRequestFiles: cappedPaths, + }); + + expect(graph.paths.length).toBeGreaterThan(0); + expect(graph.paths.every((entry) => entry.repoFullName === repoFullName)).toBe(true); + // Every emitted path edge traces back to a path that survived the cap. + const cappedPathSet = new Set(cappedPaths.map((entry) => entry.path)); + expect(graph.paths.every((entry) => cappedPathSet.has(entry.path))).toBe(true); + }); }); +function pullRequestFile(repoFullName: string, pullNumber: number, path: string): PullRequestFileRecord { + return { repoFullName, pullNumber, path, status: "modified", additions: 5, deletions: 1, changes: 6, payload: {} }; +} + +function graphProfile(repoFullName: string) { + return { + login: "dev", + generatedAt: "2026-05-30T00:00:00.000Z", + github: { login: "dev", topLanguages: ["TypeScript"], source: "github" }, + source: "github_cache", + registeredRepoActivity: { pullRequests: 6, mergedPullRequests: 6, issues: 0, reposTouched: [repoFullName], dominantLabels: [] }, + trustSignals: { evidenceScore: 0, level: "new", unlinkedOpenPullRequests: 0, maintainerAssociatedPullRequests: 0 }, + } as unknown as Parameters[0]["profile"]; +} + +function graphHistory() { + return { + login: "dev", + generatedAt: "2026-05-30T00:00:00.000Z", + source: "github_cache", + totals: {}, + repoOutcomes: [], + successPatterns: [], + failurePatterns: [], + summary: "fixture", + } as unknown as Parameters[0]["outcomeHistory"]; +} + +function graphRepo(fullName: string): RepositoryRecord { + const [owner, name] = fullName.split("/") as [string, string]; + return { + fullName, + owner, + name, + isInstalled: true, + isRegistered: true, + isPrivate: false, + defaultBranch: "main", + registryConfig: { repo: fullName, emissionShare: 0.02, issueDiscoveryShare: 0, labelMultipliers: {}, trustedLabelPipeline: false, maintainerCut: 0, raw: {} }, + }; +} + +function graphPr(repoFullName: string, number: number): PullRequestRecord { + return { + repoFullName, + number, + title: `PR ${number}`, + state: "merged", + authorLogin: "dev", + authorAssociation: "CONTRIBUTOR", + labels: [], + linkedIssues: [], + createdAt: "2026-05-27T00:00:00.000Z", + updatedAt: "2026-05-27T00:00:00.000Z", + mergedAt: "2026-05-27T00:00:00.000Z", + }; +} + function totalsSnapshot(id: string, repoFullName: string, fetchedAt: string, openIssuesTotal: number) { return { id,