diff --git a/src/services/agent-orchestrator.ts b/src/services/agent-orchestrator.ts index 5e1796cc3f..7886490053 100644 --- a/src/services/agent-orchestrator.ts +++ b/src/services/agent-orchestrator.ts @@ -939,6 +939,7 @@ function contextSnapshotFromPack(runId: string, pack: ContributorDecisionPack, d source: pack.source, selectedRepos: decisions.map((decision) => decision.repoFullName), actionPortfolio: scopedActionPortfolio(pack.actionPortfolio, decisions) as unknown as JsonValue, + counterfactualReasons: scopedCounterfactualReasons(decisions) as unknown as JsonValue, evidenceGraph: (pack.evidenceGraph ? { version: pack.evidenceGraph.version, @@ -954,6 +955,16 @@ function contextSnapshotFromPack(runId: string, pack: ContributorDecisionPack, d }; } +function scopedCounterfactualReasons(decisions: RepoDecision[]): Array<{ repoFullName: string; recommendation: RepoDecision["recommendation"]; rejectedAlternatives: NonNullable }> { + return decisions + .map((decision) => ({ + repoFullName: decision.repoFullName, + recommendation: decision.recommendation, + rejectedAlternatives: (decision.counterfactualReasons ?? []).slice(0, 5), + })) + .filter((entry) => entry.rejectedAlternatives.length > 0); +} + function scopedActionPortfolio(portfolio: ActionPortfolio | undefined, decisions: RepoDecision[]): ActionPortfolio | null { if (!portfolio) return null; const repoKeys = new Set(decisions.map((decision) => decision.repoFullName.toLowerCase())); @@ -1060,6 +1071,7 @@ export const __agentOrchestratorInternals = { actionFromRepoDecision, actionRecord, contextSnapshotFromPack, + scopedCounterfactualReasons, scopedActionPortfolio, buildRunRecord, mapDecisionAction, diff --git a/src/services/decision-pack.ts b/src/services/decision-pack.ts index ad1c3def44..0001ae19a4 100644 --- a/src/services/decision-pack.ts +++ b/src/services/decision-pack.ts @@ -77,6 +77,17 @@ export type DecisionRecommendation = "pursue" | "cleanup_first" | "maintainer_la export type DecisionActionKind = "cleanup_existing_prs" | "land_existing_prs" | "open_new_direct_pr" | "file_issue_discovery" | "maintainer_lane_improve_repo" | "maintainer_cut_readiness"; export type DecisionPackFreshness = "fresh" | "stale" | "rebuilding" | "missing"; export type ActionPortfolioBucketName = "cleanup" | "wait" | "direct_pr" | "issue_discovery" | "avoid" | "maintainer_lane"; +export type CounterfactualAlternativeKind = "wait" | "close" | "replace" | "cleanup_first" | "choose_another_issue" | "open_direct_pr" | "file_issue_discovery"; + +export type CounterfactualReason = { + alternative: CounterfactualAlternativeKind; + group: ActionPortfolioBucketName | "close" | "replace" | "choose_another_issue"; + rank: number; + reason: string; + facts: string[]; + assumptions: string[]; + publicSummary: string; +}; export type ContributorDecisionPack = { status: "ready"; @@ -185,6 +196,7 @@ export type RepoDecision = { issueQuality?: IssueQualitySummary | undefined; manifestSummary?: RepoDecisionManifestSummary | undefined; tradeoffSummary?: RepoDecisionTradeoffSummary | undefined; + counterfactualReasons?: CounterfactualReason[] | undefined; }; export type RepoDecisionManifestSummary = { @@ -228,6 +240,7 @@ export type DecisionAction = { whyThisHelps: string[]; nextActions: string[]; publicNextActions: string[]; + counterfactualReasons?: CounterfactualReason[] | undefined; }; export type ActionPortfolioScenarioProjection = { @@ -720,6 +733,20 @@ function buildRepoDecision(args: { manifestSummary, blockers, }); + const finalRiskReasons = [...new Set([...riskReasons, ...manifestReasons.riskReasons, ...outcomeRiskLines, ...recommendationFeedbackRiskLines])]; + const finalWhyThisHelps = [...new Set([...whyThisHelpsFor(recommendation, copyContext), ...manifestReasons.whyThisHelps, ...outcomeSuccessLines, ...recommendationFeedbackSuccessLines])]; + const finalNextActions = [...new Set([...nextActionsFor(recommendation, copyContext), ...manifestReasons.nextActions])]; + const finalPublicNextActions = [...new Set([...publicNextActionsFor(recommendation, copyContext), ...manifestReasons.publicNextActions])]; + const counterfactualReasons = buildRepoDecisionCounterfactualReasons({ + repoFullName: args.repo.fullName, + recommendation, + lane: lane.lane, + queue, + roleContext: args.roleContext, + outcome: args.outcome, + issueQuality, + blockers, + }); return { repoFullName: args.repo.fullName, recommendation, @@ -734,13 +761,14 @@ function buildRepoDecision(args: { scoreBlockers: blockers, repoOutcomePatterns, recommendationOutcomeFeedback: recommendationFeedback, - riskReasons: [...new Set([...riskReasons, ...manifestReasons.riskReasons, ...outcomeRiskLines, ...recommendationFeedbackRiskLines])], - whyThisHelps: [...new Set([...whyThisHelpsFor(recommendation, copyContext), ...manifestReasons.whyThisHelps, ...outcomeSuccessLines, ...recommendationFeedbackSuccessLines])], - nextActions: [...new Set([...nextActionsFor(recommendation, copyContext), ...manifestReasons.nextActions])], - publicNextActions: [...new Set([...publicNextActionsFor(recommendation, copyContext), ...manifestReasons.publicNextActions])], + riskReasons: finalRiskReasons, + whyThisHelps: finalWhyThisHelps, + nextActions: finalNextActions, + publicNextActions: finalPublicNextActions, issueQuality, manifestSummary, tradeoffSummary, + counterfactualReasons, }; } @@ -1105,6 +1133,7 @@ function action(kind: DecisionActionKind, decision: RepoDecision, priorityScore: whyThisHelps: decision.whyThisHelps, nextActions: decision.nextActions, publicNextActions: decision.publicNextActions, + counterfactualReasons: decision.counterfactualReasons, }; } @@ -1347,6 +1376,192 @@ function sanitizeTradeoffPublicText(value: string): string { .trim(); } +type CounterfactualReasonDraft = Omit; + +function buildRepoDecisionCounterfactualReasons(args: { + repoFullName: string; + recommendation: DecisionRecommendation; + lane: string; + queue: RepoDecision["queue"]; + roleContext: RoleContext; + outcome: ContributorOutcomeHistory["repoOutcomes"][number] | undefined; + issueQuality: IssueQualitySummary | undefined; + blockers: ScoreBlocker[]; +}): CounterfactualReason[] { + const drafts: CounterfactualReasonDraft[] = []; + const baseFacts = counterfactualBaseFacts(args); + const baseAssumptions = counterfactualBaseAssumptions(args); + const add = (draft: Omit, facts: string[] = [], assumptions: string[] = []) => { + drafts.push(counterfactualReason(draft, [...baseFacts, ...facts], [...baseAssumptions, ...assumptions])); + }; + + if (args.recommendation === "cleanup_first") { + add({ + alternative: "wait", + group: "wait", + reason: "Passive waiting was rejected because visible cleanup work can reduce queue pressure sooner.", + publicSummary: `${args.repoFullName}: active cleanup is preferred over passive waiting while open-work pressure is visible.`, + }); + add( + { + alternative: "choose_another_issue", + group: "choose_another_issue", + reason: "Choosing another issue was rejected because it would add review load before current open work is resolved.", + publicSummary: `${args.repoFullName}: choose-another-issue is deferred until existing open work is cleaned up.`, + }, + counterfactualIssueFacts(args), + ["Assumes the contributor should avoid adding parallel review load while cleanup pressure is visible."], + ); + add({ + alternative: "replace", + group: "replace", + reason: "Replacing the repo was rejected because the cleanup obligation is attached to existing work in this repo.", + publicSummary: `${args.repoFullName}: replacing the repo does not remove the need to address existing open work here.`, + }); + } else { + add({ + alternative: "cleanup_first", + group: "cleanup", + reason: "Cleanup-first was rejected because the decision does not show enough open-work pressure to make cleanup the primary action.", + publicSummary: `${args.repoFullName}: cleanup-first is not the primary recommendation from the current queue signals.`, + }); + add({ + alternative: "close", + group: "close", + reason: "Closing or withdrawing work was rejected because no cleanup-priority blocker is present for this recommendation.", + publicSummary: `${args.repoFullName}: closing work is not the primary recommendation from the current blocker signals.`, + }); + } + + if (args.recommendation === "pursue") { + add({ + alternative: "wait", + group: "wait", + reason: "Waiting was rejected because the lane and blocker facts support a narrow next step now.", + publicSummary: `${args.repoFullName}: waiting is not the clearest path while the current lane remains actionable.`, + }); + add( + { + alternative: "choose_another_issue", + group: "choose_another_issue", + reason: "Choosing another issue was rejected because the selected repo path is currently actionable; switch only if preflight or duplicate checks fail.", + publicSummary: `${args.repoFullName}: choose-another-issue is a fallback if preflight or duplicate checks fail.`, + }, + counterfactualIssueFacts(args), + ["Assumes the chosen change stays narrow and passes local preflight before public posting."], + ); + add({ + alternative: "replace", + group: "replace", + reason: "Replacing the repo was rejected because this repo still has an actionable lane and no critical blocker in the decision.", + publicSummary: `${args.repoFullName}: replacing the repo is not favored while this repo has an actionable lane.`, + }); + } else if (args.recommendation === "watch") { + add({ + alternative: "wait", + group: "wait", + reason: "Pure waiting was rejected because an actionable, non-duplicate issue report can still fit this lane.", + publicSummary: `${args.repoFullName}: pure waiting is not required if an actionable, non-duplicate report is available.`, + }); + add( + { + alternative: "choose_another_issue", + group: "choose_another_issue", + reason: "Choosing another issue was rejected unless duplicate checks or issue-quality evidence make the current candidate unsuitable.", + publicSummary: `${args.repoFullName}: choose-another-issue is a fallback after duplicate and issue-quality checks.`, + }, + counterfactualIssueFacts(args), + ["Assumes any issue report remains evidence-backed and non-duplicate."], + ); + } else if (args.recommendation === "maintainer_lane") { + add({ + alternative: "wait", + group: "wait", + reason: "Waiting was rejected because owner-side intake or queue-health work can improve the repo without adding outside-contributor work.", + publicSummary: `${args.repoFullName}: owner-side repo-health work is preferred over passive waiting.`, + }); + add({ + alternative: "choose_another_issue", + group: "choose_another_issue", + reason: "Choosing another contributor issue was rejected because the current lane is maintainer-owned repo health work.", + publicSummary: `${args.repoFullName}: contributor issue selection is separated from maintainer-lane repo-health work.`, + }); + } else if (args.recommendation === "avoid_for_now") { + add({ + alternative: "wait", + group: "wait", + reason: "Only waiting was rejected because a cleaner repo or refreshed lane signal is a more concrete next planning step.", + publicSummary: `${args.repoFullName}: waiting alone is less useful than rerunning after signals improve or choosing a cleaner target.`, + }); + add({ + alternative: "open_direct_pr", + group: "direct_pr", + reason: "Opening a direct PR was rejected because current blocker or lane facts make new implementation work a poor fit.", + publicSummary: `${args.repoFullName}: direct PR work is not advised until blocker or lane signals improve.`, + }); + } + + return drafts.slice(0, 5).map((draft, index) => ({ ...draft, rank: index + 1 })); +} + +function counterfactualReason(draft: Omit, facts: string[], assumptions: string[]): CounterfactualReasonDraft { + return { + ...draft, + reason: sanitizeCounterfactualPublicText(draft.reason), + facts: uniqueCounterfactualStrings(facts.map(sanitizeCounterfactualPublicText)).slice(0, 5), + assumptions: uniqueCounterfactualStrings(assumptions.map(sanitizeCounterfactualPublicText)).slice(0, 4), + publicSummary: sanitizeCounterfactualPublicText(draft.publicSummary), + }; +} + +function counterfactualBaseFacts(args: { + repoFullName: string; + recommendation: DecisionRecommendation; + lane: string; + queue: RepoDecision["queue"]; + outcome: ContributorOutcomeHistory["repoOutcomes"][number] | undefined; + issueQuality: IssueQualitySummary | undefined; + blockers: ScoreBlocker[]; +}): string[] { + const blockerCodes = args.blockers.map((blocker) => blocker.code); + return [ + `${args.repoFullName}: current recommendation is ${args.recommendation}.`, + `${args.repoFullName}: configured lane is ${args.lane}.`, + `${args.repoFullName}: cached repo queue has ${args.queue.openPullRequests} open PR(s) and ${args.queue.openIssues} open issue(s).`, + ...(args.outcome ? [`${args.repoFullName}: contributor has ${args.outcome.openPullRequests ?? 0} open PR(s) in this repo.`] : []), + blockerCodes.length > 0 ? `${args.repoFullName}: visible blocker code(s): ${blockerCodes.join(", ")}.` : `${args.repoFullName}: no visible blocker code is present.`, + ...(args.issueQuality ? counterfactualIssueFacts(args) : []), + ]; +} + +function counterfactualBaseAssumptions(args: { + lane: string; + roleContext: RoleContext; + outcome: ContributorOutcomeHistory["repoOutcomes"][number] | undefined; + issueQuality: IssueQualitySummary | undefined; +}): string[] { + return [ + ...(!args.outcome && !args.roleContext.maintainerLane ? ["Repo-specific contributor outcome data is missing; lane and queue signals carry more weight."] : []), + ...(!args.issueQuality ? ["No issue-quality cache is available for this repo decision."] : []), + ...(args.lane === "split" ? ["Split-lane planning assumes the final work choice still passes duplicate and scope checks."] : []), + ]; +} + +function counterfactualIssueFacts(args: { repoFullName: string; issueQuality: IssueQualitySummary | undefined }): string[] { + if (!args.issueQuality) return []; + return [ + `${args.repoFullName}: issue-quality cache has ${args.issueQuality.readyCount} ready candidate(s), ${args.issueQuality.needsProofCount} needing proof, ${args.issueQuality.holdCount} on hold, and ${args.issueQuality.doNotUseCount} do-not-use candidate(s).`, + ]; +} + +function sanitizeCounterfactualPublicText(value: string): string { + return sanitizeTradeoffPublicText(value); +} + +function uniqueCounterfactualStrings(values: string[]): string[] { + return [...new Set(values.map((value) => value.trim()).filter(Boolean))]; +} + type RepoCopyContext = { repoFullName: string; lane: string; @@ -1555,4 +1770,6 @@ export const __decisionPackInternals = { round, buildRepoDecisionTradeoffSummary, sanitizeTradeoffPublicText, + buildRepoDecisionCounterfactualReasons, + sanitizeCounterfactualPublicText, }; diff --git a/test/unit/agent-orchestrator.test.ts b/test/unit/agent-orchestrator.test.ts index aac9f8de58..0acdb44ffb 100644 --- a/test/unit/agent-orchestrator.test.ts +++ b/test/unit/agent-orchestrator.test.ts @@ -150,6 +150,46 @@ describe("agent orchestrator", () => { }); }); + it("threads scoped counterfactual reasons into decision context snapshots", () => { + const generatedAt = nowIso(); + const rejectedAlternative = { + alternative: "wait", + group: "wait", + rank: 1, + reason: "Waiting was rejected because current facts support active cleanup.", + facts: ["owner/repo: current recommendation is cleanup_first."], + assumptions: ["No issue-quality cache is available for this repo decision."], + publicSummary: "owner/repo: active cleanup is preferred over passive waiting.", + } satisfies NonNullable[number]; + const decision = repoDecision({ + repoFullName: "owner/repo", + recommendation: "cleanup_first", + counterfactualReasons: [rejectedAlternative], + }); + const pack = decisionPackFixture({ + generatedAt, + repoDecisions: [decision], + topActions: [], + }); + + const snapshot = __agentOrchestratorInternals.contextSnapshotFromPack("run-counterfactual", pack, [decision]); + const counterfactuals = snapshot.payload.counterfactualReasons as unknown as Array<{ + repoFullName: string; + recommendation: string; + rejectedAlternatives: Array; + }>; + + expect(counterfactuals).toEqual([ + { + repoFullName: "owner/repo", + recommendation: "cleanup_first", + rejectedAlternatives: [rejectedAlternative], + }, + ]); + expect(__agentOrchestratorInternals.scopedCounterfactualReasons([{ ...decision, counterfactualReasons: [] }])).toEqual([]); + expect(JSON.stringify(counterfactuals)).not.toMatch(/wallet|hotkey|raw trust score|payout|reward estimate|farming|private reviewability|public score estimate/i); + }); + it("does not fall back to cross-repo private rankings for public GitHub comments", async () => { const env = createTestEnv(); const secretDecision = repoDecision({ diff --git a/test/unit/decision-pack.test.ts b/test/unit/decision-pack.test.ts index 15f6f9578f..9ac4e5d07e 100644 --- a/test/unit/decision-pack.test.ts +++ b/test/unit/decision-pack.test.ts @@ -1260,6 +1260,69 @@ describe("decision-pack service", () => { ).not.toMatch(FORBIDDEN_PUBLIC_TRADEOFF_LANGUAGE); }); + it("adds deterministic counterfactual reasons for wait, cleanup-first, and choose-another-issue alternatives", () => { + const pursue = __decisionPackInternals.buildRepoDecision({ + repo: repoWithLabels("owner/direct-counterfactual", 0.04, 0, { bug: 1.1 }), + roleContext: { maintainerLane: false } as any, + outcome: { openPullRequests: 0, mergedPullRequests: 2, closedPullRequestRate: 0, credibility: 1 } as any, + syncState: { primaryLanguage: "TypeScript", openPullRequestsCount: 1, openIssuesCount: 6 } as any, + languageSet: new Set(["typescript"]), + labelHistory: new Set(["bug"]), + }); + expect(pursue.recommendation).toBe("pursue"); + expect(pursue.counterfactualReasons?.map((reason) => reason.alternative)).toEqual( + expect.arrayContaining(["wait", "cleanup_first", "choose_another_issue"]), + ); + expect(pursue.counterfactualReasons?.find((reason) => reason.alternative === "wait")).toMatchObject({ + rank: expect.any(Number), + facts: expect.arrayContaining([expect.stringContaining("current recommendation is pursue")]), + assumptions: expect.arrayContaining([expect.stringContaining("No issue-quality cache")]), + }); + + const cleanupFirst = __decisionPackInternals.buildRepoDecision({ + repo: repoWithLabels("owner/cleanup-counterfactual", 0.04, 0, { bug: 1.1 }), + roleContext: { maintainerLane: false } as any, + outcome: { openPullRequests: 7, mergedPullRequests: 1, closedPullRequestRate: 0.1, credibility: 1 } as any, + syncState: { primaryLanguage: "TypeScript", openPullRequestsCount: 8, openIssuesCount: 4 } as any, + languageSet: new Set(["typescript"]), + labelHistory: new Set(["bug"]), + }); + expect(cleanupFirst.recommendation).toBe("cleanup_first"); + expect(cleanupFirst.counterfactualReasons?.map((reason) => reason.alternative)).toEqual( + expect.arrayContaining(["wait", "choose_another_issue", "replace"]), + ); + expect(cleanupFirst.counterfactualReasons?.map((reason) => reason.alternative)).not.toContain("cleanup_first"); + + const issueDiscovery = __decisionPackInternals.buildRepoDecision({ + repo: repoWithLabels("owner/issue-counterfactual", 0.02, 1, { bug: 1.1 }), + roleContext: { maintainerLane: false } as any, + issueQuality: { + repoFullName: "owner/issue-counterfactual", + generatedAt: "2026-06-02T00:00:00.000Z", + lane: { repoFullName: "owner/issue-counterfactual", lane: "issue_discovery", issueDiscoveryShare: 1, directPrShare: 0, summary: "", contributorGuidance: "", maintainerGuidance: "" }, + issues: [ + { number: 11, title: "Ready issue", status: "ready", score: 90, reasons: [], warnings: [] }, + { number: 12, title: "Covered issue", status: "do_not_use", score: 0, reasons: [], warnings: [] }, + ], + summary: "2 open issues evaluated.", + }, + }); + const chooseAnother = issueDiscovery.counterfactualReasons?.find((reason) => reason.alternative === "choose_another_issue"); + expect(issueDiscovery.recommendation).toBe("watch"); + expect(chooseAnother?.facts.join(" ")).toMatch(/issue-quality cache has 1 ready candidate/); + + const summaries = [pursue, cleanupFirst, issueDiscovery].flatMap((decision) => decision.counterfactualReasons?.map((reason) => reason.publicSummary) ?? []); + expect(summaries.join(" ")).not.toMatch(FORBIDDEN_PUBLIC_TRADEOFF_LANGUAGE); + }); + + it("sanitizes counterfactual public summaries", () => { + expect( + __decisionPackInternals.sanitizeCounterfactualPublicText( + "wallet hotkey reward-estimate payout scoreability public-score-prediction trust-score private-reviewability private-scoreability farming-language", + ), + ).not.toMatch(FORBIDDEN_PUBLIC_TRADEOFF_LANGUAGE); + }); + it("covers languageMatch true/false and labelFit empty/non-empty paths", () => { const ctx = (overrides: Record = {}) => __decisionPackInternals.buildRepoDecision({