From 3274073007ad1b0785a3729f378ac92795c8a29d Mon Sep 17 00:00:00 2001 From: mkdev11 Date: Tue, 2 Jun 2026 03:21:53 +0200 Subject: [PATCH] feat(agent): add recommendation confidence provenance --- src/services/agent-orchestrator.ts | 253 ++++++++++++++++++++++++++- test/integration/api.test.ts | 15 +- test/unit/agent-orchestrator.test.ts | 169 ++++++++++++++++++ 3 files changed, 428 insertions(+), 9 deletions(-) diff --git a/src/services/agent-orchestrator.ts b/src/services/agent-orchestrator.ts index c74bd6171b..39e4273e83 100644 --- a/src/services/agent-orchestrator.ts +++ b/src/services/agent-orchestrator.ts @@ -67,6 +67,32 @@ export type AgentRunBundle = { summary: string; }; +type RecommendationConfidence = "high" | "medium" | "low"; +type RecommendationFreshness = "fresh" | "stale" | "rebuilding" | "missing" | "degraded" | "possibly_stale" | "unknown"; + +type RecommendationEvidenceSource = { + name: string; + source: string | null; + generatedAt: string | null; + freshness: RecommendationFreshness; + summary: string; +}; + +type RecommendationEvidence = { + confidence: RecommendationConfidence; + sourceSummary: string; + freshness: RecommendationFreshness; + sources: RecommendationEvidenceSource[]; + assumptions: string[]; + warnings: string[]; + userSuppliedScenarios: boolean; + userSuppliedScenarioCount: number; +}; + +type LocalBranchActionAnalysis = LocalBranchAnalysis & { + dataQuality?: { status: "complete" | "degraded" | "blocked" | "unknown"; warnings: string[] } | undefined; +}; + export async function startAgentRun(env: Env, input: AgentRunCreateRequest): Promise { const run = buildRunRecord({ objective: input.objective, @@ -350,9 +376,9 @@ function buildDecisionActions(run: AgentRunRecord, pack: ContributorDecisionPack const candidateActions = pack.topActions .filter((action) => decisionByRepo.has(action.repoFullName)) .slice(0, 8) - .map((action, index) => actionFromDecisionAction(run, action, decisionByRepo.get(action.repoFullName)!, monitorActions.length + index)); + .map((action, index) => actionFromDecisionAction(run, action, decisionByRepo.get(action.repoFullName)!, monitorActions.length + index, pack)); if (candidateActions.length > 0) return [...monitorActions, ...candidateActions].slice(0, 8); - const fallback = decisions.slice(0, 5).map((decision, index) => actionFromRepoDecision(run, decision, monitorActions.length + index)); + const fallback = decisions.slice(0, 5).map((decision, index) => actionFromRepoDecision(run, decision, monitorActions.length + index, pack)); return [...monitorActions, ...fallback].slice(0, 8); } @@ -394,6 +420,7 @@ function buildOpenPrMonitorActions(run: AgentRunRecord, pack: ContributorDecisio openPrPacket: packet as unknown as JsonValue, decision: decision as unknown as JsonValue, }, + evidence: decisionPackEvidence(pack, decision, "Open PR monitor recommendation from cached GitHub queue state."), safetyClass: "public_safe", approvalRequired: false, }); @@ -423,11 +450,12 @@ function buildBlockerActions( rerunWhen: "Rerun after open PRs merge/close, credibility updates, linked issue context changes, or validation changes.", publicSafeSummary: `${decision.repoFullName}: blocker context is available privately; public output should stay focused on review hygiene.`, payload: { decision: decision as unknown as JsonValue }, + evidence: decisionPackEvidence(pack, decision, "Scoreability blocker explanation from the contributor decision pack."), }), ); } -function buildLocalBranchActions(run: AgentRunRecord, analysis: LocalBranchAnalysis): AgentActionRecord[] { +function buildLocalBranchActions(run: AgentRunRecord, analysis: LocalBranchActionAnalysis): AgentActionRecord[] { const actions: AgentActionRecord[] = [ actionRecord({ run, @@ -448,6 +476,7 @@ function buildLocalBranchActions(run: AgentRunRecord, analysis: LocalBranchAnaly rerunWhen: analysis.recommendedRerunCondition, publicSafeSummary: sanitizePublicSummary(`${analysis.repoFullName}: preflight found ${analysis.preflight.findings.length} finding(s); use the public-safe PR packet before posting.`), payload: { analysis: analysis as unknown as JsonValue }, + evidence: localBranchEvidence(analysis, "Local branch preflight recommendation from structured metadata."), }), localPrPacketAction(run, analysis, 1), ]; @@ -455,7 +484,7 @@ function buildLocalBranchActions(run: AgentRunRecord, analysis: LocalBranchAnaly return actions.slice(0, 8); } -function buildLocalBlockerActions(run: AgentRunRecord, analysis: LocalBranchAnalysis, startIndex = 0): AgentActionRecord[] { +function buildLocalBlockerActions(run: AgentRunRecord, analysis: LocalBranchActionAnalysis, startIndex = 0): AgentActionRecord[] { return [ actionRecord({ run, @@ -475,11 +504,12 @@ function buildLocalBlockerActions(run: AgentRunRecord, analysis: LocalBranchAnal scenarioScorePreview: analysis.scenarioScorePreview as unknown as JsonValue, baseFreshness: analysis.baseFreshness as unknown as JsonValue, }, + evidence: localBranchEvidence(analysis, "Private scoreability blocker explanation from local metadata."), }), ]; } -function localPrPacketAction(run: AgentRunRecord, analysis: LocalBranchAnalysis, index = 0): AgentActionRecord { +function localPrPacketAction(run: AgentRunRecord, analysis: LocalBranchActionAnalysis, index = 0): AgentActionRecord { return actionRecord({ run, actionType: "prepare_pr_packet", @@ -493,12 +523,13 @@ function localPrPacketAction(run: AgentRunRecord, analysis: LocalBranchAnalysis, rerunWhen: analysis.recommendedRerunCondition, publicSafeSummary: sanitizePublicSummary(`${analysis.repoFullName}: public-safe PR packet prepared from metadata only.`), payload: { prPacket: analysis.prPacket as unknown as JsonValue }, + evidence: localBranchEvidence(analysis, "Public-safe PR packet recommendation from local metadata."), safetyClass: "public_safe", approvalRequired: false, }); } -function actionFromDecisionAction(run: AgentRunRecord, action: DecisionAction, decision: RepoDecision, index: number): AgentActionRecord { +function actionFromDecisionAction(run: AgentRunRecord, action: DecisionAction, decision: RepoDecision, index: number, pack?: ContributorDecisionPack | undefined): AgentActionRecord { return actionRecord({ run, actionType: mapDecisionAction(action.actionKind), @@ -517,10 +548,11 @@ function actionFromDecisionAction(run: AgentRunRecord, action: DecisionAction, d action: action as unknown as JsonValue, decision: decision as unknown as JsonValue, }, + evidence: pack ? decisionPackEvidence(pack, decision, "Ranked next-action recommendation from the contributor decision pack.") : repoDecisionEvidence(decision), }); } -function actionFromRepoDecision(run: AgentRunRecord, decision: RepoDecision, index: number): AgentActionRecord { +function actionFromRepoDecision(run: AgentRunRecord, decision: RepoDecision, index: number, pack?: ContributorDecisionPack | undefined): AgentActionRecord { return actionRecord({ run, actionType: "explain_repo_fit", @@ -536,6 +568,7 @@ function actionFromRepoDecision(run: AgentRunRecord, decision: RepoDecision, ind rerunWhen: rerunWhenForDecision(decision), publicSafeSummary: sanitizePublicSummary(decision.publicNextActions?.[0] ?? `${decision.repoFullName}: Use local branch preflight before posting.`), payload: { decision: decision as unknown as JsonValue }, + evidence: pack ? decisionPackEvidence(pack, decision, "Repo-fit fallback recommendation from the contributor decision pack.") : repoDecisionEvidence(decision), }); } @@ -558,7 +591,9 @@ function actionRecord(args: { approvalRequired?: boolean | undefined; safetyClass?: AgentSafetyClass | undefined; payload: Record; + evidence?: RecommendationEvidence | undefined; }): AgentActionRecord { + const evidence = args.evidence ?? defaultRecommendationEvidence(args.actionType); return { id: `${args.run.id}:${String(args.index).padStart(2, "0")}:${args.actionType}`, runId: args.run.id, @@ -577,11 +612,213 @@ function actionRecord(args: { publicSafeSummary: sanitizePublicSummary(args.publicSafeSummary), approvalRequired: args.approvalRequired ?? true, safetyClass: args.safetyClass ?? "private", - payload: args.payload, + payload: { + ...args.payload, + recommendationEvidence: evidence as unknown as JsonValue, + }, createdAt: nowIso(), }; } +function decisionPackEvidence(pack: ContributorDecisionPack, decision: RepoDecision, sourceSummary: string): RecommendationEvidence { + const repoQuality = repoSignalQuality(pack, decision.repoFullName); + const userSuppliedScenarioCount = userSuppliedScenarioCountForRepo(pack, decision.repoFullName); + const missingOfficialStats = !pack.profile.officialStats || pack.profile.source !== "gittensor_api"; + const missingRepoOutcome = !decision.outcome && !decision.roleContext.maintainerLane; + const freshness = pack.freshness !== "fresh" ? pack.freshness : repoQuality.freshness; + const warnings = uniqueStrings([ + ...(pack.freshness === "rebuilding" ? ["Decision pack is stale; a background rebuild was enqueued."] : []), + ...(pack.freshness === "stale" ? ["Decision pack is stale and no rebuild was enqueued."] : []), + ...(pack.dataQuality.signalFidelity.status === "blocked" ? ["Signal fidelity is blocked for this decision pack."] : []), + ...repoQuality.warnings, + ...(missingOfficialStats ? ["Official Gittensor contributor stats were unavailable; confidence is reduced."] : []), + ...(missingRepoOutcome ? ["No repo-specific official outcome row was available; confidence is reduced."] : []), + ]); + const assumptions = uniqueStrings([ + ...(missingOfficialStats ? ["Contributor-level official stats are missing, so cached GitHub and registry data carry more weight."] : []), + ...(missingRepoOutcome ? ["Repo-specific prior outcomes are missing, so queue, lane, and role heuristics carry more weight."] : []), + ...(userSuppliedScenarioCount > 0 ? ["Pending-PR scenario projections include user-supplied assumptions."] : []), + ]); + return { + confidence: confidenceForDecisionPack(pack, decision, repoQuality, userSuppliedScenarioCount), + sourceSummary, + freshness, + sources: [ + evidenceSource("contributor_decision_pack", pack.source, pack.generatedAt, pack.freshness, `${pack.login} decision pack with ${pack.dataQuality.signalFidelity.status} signal fidelity.`), + evidenceSource("repo_decision", decision.roleContext.source, pack.generatedAt, repoQuality.freshness, `${decision.repoFullName} ranked ${decision.recommendation} at priority ${decision.priorityScore}.`), + evidenceSource( + "official_contributor_stats", + pack.profile.source, + pack.generatedAt, + missingOfficialStats ? "missing" : "fresh", + missingOfficialStats ? "Official contributor stats missing for this snapshot." : "Official contributor stats present in this snapshot.", + ), + evidenceSource( + "repo_outcome_history", + decision.outcome ? pack.outcomeHistory.source : null, + pack.generatedAt, + decision.outcome ? "fresh" : "missing", + decision.outcome ? "Repo-specific contributor outcomes present." : "Repo-specific contributor outcomes missing.", + ), + ...(pack.openPrMonitor + ? [evidenceSource("open_pr_monitor", "cached_github_data", pack.openPrMonitor.generatedAt, pack.freshness === "fresh" ? "fresh" : pack.freshness, pack.openPrMonitor.summary)] + : []), + ], + assumptions, + warnings, + userSuppliedScenarios: userSuppliedScenarioCount > 0, + userSuppliedScenarioCount, + }; +} + +function repoDecisionEvidence(decision: RepoDecision): RecommendationEvidence { + const missingRepoOutcome = !decision.outcome && !decision.roleContext.maintainerLane; + return { + confidence: missingRepoOutcome ? "medium" : "high", + sourceSummary: "Repo decision recommendation without serving-pack freshness metadata.", + freshness: "unknown", + sources: [ + evidenceSource("repo_decision", decision.roleContext.source, null, "unknown", `${decision.repoFullName} ranked ${decision.recommendation} at priority ${decision.priorityScore}.`), + evidenceSource("repo_outcome_history", decision.outcome ? "gittensor_api" : null, null, decision.outcome ? "fresh" : "missing", decision.outcome ? "Repo-specific contributor outcomes present." : "Repo-specific contributor outcomes missing."), + ], + assumptions: missingRepoOutcome ? ["Repo-specific prior outcomes are missing, so queue, lane, and role heuristics carry more weight."] : [], + warnings: missingRepoOutcome ? ["No repo-specific official outcome row was available; confidence is reduced."] : [], + userSuppliedScenarios: false, + userSuppliedScenarioCount: 0, + }; +} + +function localBranchEvidence(analysis: LocalBranchActionAnalysis, sourceSummary: string): RecommendationEvidence { + const freshness = localEvidenceFreshness(analysis); + const userSuppliedScenarioCount = analysis.scorePreview.scenarioPreviews.filter((scenario) => scenario.source === "user_supplied").length; + const userSuppliedLinkedIssue = analysis.scorePreview.linkedIssueMultiplier.source === "user_supplied"; + const userSuppliedBranchEligibility = analysis.branchEligibility.source === "user_supplied"; + const userSuppliedScenarios = userSuppliedScenarioCount > 0 || userSuppliedLinkedIssue || userSuppliedBranchEligibility; + const warnings = uniqueStrings([ + ...analysis.baseFreshness.warnings, + ...(analysis.dataQuality?.warnings ?? []), + ...analysis.scorePreview.warnings, + ...analysis.branchEligibility.warnings, + ...(analysis.githubBranchStatus.status === "unknown" ? analysis.githubBranchStatus.notes : []), + ]); + const assumptions = uniqueStrings([ + "Local agent analysis used structured git and GitHub metadata only; source contents were not uploaded.", + ...analysis.scorePreview.assumptions.filter((assumption) => /scenario|linked issue|advisory|metadata|branch/i.test(assumption)).slice(0, 8), + ...(userSuppliedScenarios ? ["One or more scenario, linked-issue, or branch-eligibility inputs were supplied by the caller."] : []), + ]); + return { + confidence: confidenceForLocalBranch(analysis, userSuppliedScenarios), + sourceSummary, + freshness, + sources: [ + evidenceSource("local_branch_metadata", "metadata_only", analysis.generatedAt, freshness, "Structured local branch metadata; source upload disabled."), + evidenceSource("base_branch_freshness", "local_git_metadata", analysis.generatedAt, localFreshnessStatus(analysis.baseFreshness.status), `${analysis.baseFreshness.status} base/head metadata.`), + evidenceSource("score_preview", analysis.scorePreview.activeModel, analysis.scorePreview.generatedAt, "fresh", `${analysis.scorePreview.scoreabilityStatus} private score preview.`), + evidenceSource("github_branch_status", analysis.githubBranchStatus.source, analysis.generatedAt, analysis.githubBranchStatus.status === "unknown" ? "unknown" : "fresh", `${analysis.githubBranchStatus.status} cached GitHub branch status.`), + evidenceSource("linked_issue_multiplier", analysis.scorePreview.linkedIssueMultiplier.source, analysis.scorePreview.generatedAt, analysis.scorePreview.linkedIssueMultiplier.status === "unavailable" ? "missing" : "fresh", analysis.scorePreview.linkedIssueMultiplier.reason), + ], + assumptions, + warnings, + userSuppliedScenarios, + userSuppliedScenarioCount, + }; +} + +function defaultRecommendationEvidence(actionType: AgentActionType): RecommendationEvidence { + return { + confidence: "medium", + sourceSummary: "Generated from Gittensory agent metadata.", + freshness: "unknown", + sources: [evidenceSource("agent_action", null, null, "unknown", `${actionType} action generated without source-specific evidence.`)], + assumptions: [], + warnings: ["Source-specific evidence was not attached; treat this recommendation as medium confidence."], + userSuppliedScenarios: false, + userSuppliedScenarioCount: 0, + }; +} + +function confidenceForDecisionPack( + pack: ContributorDecisionPack, + decision: RepoDecision, + repoQuality: { freshness: RecommendationFreshness; warnings: string[] }, + userSuppliedScenarioCount: number, +): RecommendationConfidence { + let confidence: RecommendationConfidence = "high"; + const fidelity = pack.dataQuality.signalFidelity; + if (pack.freshness !== "fresh" || fidelity.status === "blocked" || repoQuality.freshness === "stale" || repoQuality.warnings.some((warning) => /rate limited/i.test(warning))) { + confidence = lowerConfidence(confidence, "low"); + } else if (fidelity.status !== "complete" || repoQuality.freshness === "degraded") { + confidence = lowerConfidence(confidence, "medium"); + } + if (!pack.profile.officialStats || pack.profile.source !== "gittensor_api") confidence = lowerConfidence(confidence, "medium"); + if (!decision.outcome && !decision.roleContext.maintainerLane) confidence = lowerConfidence(confidence, "medium"); + if (userSuppliedScenarioCount > 0) confidence = lowerConfidence(confidence, "medium"); + return confidence; +} + +function confidenceForLocalBranch(analysis: LocalBranchActionAnalysis, userSuppliedScenarios: boolean): RecommendationConfidence { + let confidence: RecommendationConfidence = "high"; + if (analysis.baseFreshness.status === "stale" || analysis.dataQuality?.status === "blocked") confidence = lowerConfidence(confidence, "low"); + if (analysis.baseFreshness.status === "possibly_stale" || analysis.baseFreshness.status === "unknown") confidence = lowerConfidence(confidence, "medium"); + if (analysis.dataQuality && analysis.dataQuality.status !== "complete") confidence = lowerConfidence(confidence, "medium"); + if (analysis.githubBranchStatus.status === "unknown") confidence = lowerConfidence(confidence, "medium"); + if (analysis.branchEligibility.stale || analysis.branchEligibility.evidence === "missing") confidence = lowerConfidence(confidence, "medium"); + if (userSuppliedScenarios) confidence = lowerConfidence(confidence, "medium"); + if (analysis.scorePreview.warnings.some((warning) => /unavailable|missing|stale/i.test(warning))) confidence = lowerConfidence(confidence, "medium"); + return confidence; +} + +function repoSignalQuality(pack: ContributorDecisionPack, repoFullName: string): { freshness: RecommendationFreshness; warnings: string[] } { + const fidelity = pack.dataQuality.signalFidelity; + const repo = repoFullName.toLowerCase(); + const has = (repos: string[]) => repos.some((entry) => entry.toLowerCase() === repo); + const warnings = [ + ...(has(fidelity.partialRepos) ? [`${repoFullName}: partial signal coverage.`] : []), + ...(has(fidelity.cappedRepos) ? [`${repoFullName}: capped signal coverage.`] : []), + ...(has(fidelity.staleRepos) ? [`${repoFullName}: stale signal coverage.`] : []), + ...(has(fidelity.rateLimitedRepos) ? [`${repoFullName}: rate limited signal coverage.`] : []), + ]; + if (has(fidelity.staleRepos)) return { freshness: "stale", warnings }; + if (warnings.length > 0 || fidelity.status !== "complete") return { freshness: "degraded", warnings }; + return { freshness: "fresh", warnings }; +} + +function localEvidenceFreshness(analysis: LocalBranchActionAnalysis): RecommendationFreshness { + if (analysis.baseFreshness.status === "stale") return "stale"; + if (analysis.baseFreshness.status === "possibly_stale") return "possibly_stale"; + if (analysis.baseFreshness.status === "unknown") return "unknown"; + if (analysis.dataQuality && analysis.dataQuality.status !== "complete") return "degraded"; + return "fresh"; +} + +function localFreshnessStatus(status: LocalBranchAnalysis["baseFreshness"]["status"]): RecommendationFreshness { + if (status === "possibly_stale") return "possibly_stale"; + return status; +} + +function userSuppliedScenarioCountForRepo(pack: ContributorDecisionPack, repoFullName: string): number { + return (pack.openPrMonitor?.pendingScenarios ?? []).filter((scenario) => sameRepo(scenario.repoFullName, repoFullName) && scenario.detection.source === "user_supplied").length; +} + +function lowerConfidence(current: RecommendationConfidence, target: RecommendationConfidence): RecommendationConfidence { + const rank: Record = { low: 0, medium: 1, high: 2 }; + return rank[target] < rank[current] ? target : current; +} + +function evidenceSource(name: string, source: string | null | undefined, generatedAt: string | null | undefined, freshness: RecommendationFreshness, summary: string): RecommendationEvidenceSource { + return { + name, + source: source ?? null, + generatedAt: generatedAt ?? null, + freshness, + summary, + }; +} + +function uniqueStrings(values: string[]): string[] { + return [...new Set(values.map((value) => value.trim()).filter(Boolean))]; +} + function contextSnapshotFromPack(runId: string, pack: ContributorDecisionPack, decisions: RepoDecision[]): AgentContextSnapshotRecord { const fidelity = pack.dataQuality.signalFidelity; const ageSeconds = pack.snapshotAgeSeconds ?? null; diff --git a/test/integration/api.test.ts b/test/integration/api.test.ts index e5cc9c0499..620ee8864c 100644 --- a/test/integration/api.test.ts +++ b/test/integration/api.test.ts @@ -633,6 +633,12 @@ describe("api routes", () => { expect(agentPlanPayload.actions.length).toBeGreaterThan(0); expect(agentPlanPayload.actions[0]?.publicSafeSummary).not.toMatch(/wallet|hotkey|reward estimate|payout|farming|raw trust score/i); expect(agentPlanPayload.actions[0]?.payload).toHaveProperty("decision"); + expect(agentPlanPayload.actions[0]?.payload.recommendationEvidence).toMatchObject({ + confidence: expect.stringMatching(/^(high|medium|low)$/), + sourceSummary: expect.any(String), + freshness: expect.any(String), + sources: expect.arrayContaining([expect.objectContaining({ name: "contributor_decision_pack" })]), + }); const fetchedAgentRun = await app.request(`/v1/agent/runs/${agentPlanPayload.run.id}`, { headers: apiHeaders(env) }, env); expect(fetchedAgentRun.status).toBe(200); @@ -3215,8 +3221,15 @@ describe("api routes", () => { env, ); expect(response.status).toBe(200); - const payload = (await mcpJson(response)) as { result?: { content?: Array<{ text: string }> } }; + const payload = (await mcpJson(response)) as { result?: { content?: Array<{ text: string }>; structuredContent?: { actions?: Array<{ payload?: Record }> } } }; const text = payload.result?.content?.[0]?.text ?? ""; + if (name === "gittensory_agent_plan_next_work") { + expect(payload.result?.structuredContent?.actions?.[0]?.payload?.recommendationEvidence).toMatchObject({ + confidence: expect.stringMatching(/^(high|medium|low)$/), + sourceSummary: expect.any(String), + sources: expect.arrayContaining([expect.objectContaining({ name: "contributor_decision_pack" })]), + }); + } const privateRewardTools = new Set([ "gittensory_get_decision_pack", "gittensory_explain_repo_decision", diff --git a/test/unit/agent-orchestrator.test.ts b/test/unit/agent-orchestrator.test.ts index 4fada4f048..51f394ab8e 100644 --- a/test/unit/agent-orchestrator.test.ts +++ b/test/unit/agent-orchestrator.test.ts @@ -120,7 +120,20 @@ describe("agent orchestrator", () => { approvalRequired: true, safetyClass: "private", }); + expect(bundle.actions[0]?.payload.recommendationEvidence).toMatchObject({ + confidence: "low", + sourceSummary: "Ranked next-action recommendation from the contributor decision pack.", + freshness: "stale", + userSuppliedScenarios: false, + sources: expect.arrayContaining([ + expect.objectContaining({ name: "contributor_decision_pack", freshness: "fresh" }), + expect.objectContaining({ name: "repo_decision", freshness: "stale" }), + expect.objectContaining({ name: "official_contributor_stats", freshness: "fresh" }), + ]), + warnings: expect.arrayContaining(["we-promise/sure: partial signal coverage.", "we-promise/sure: stale signal coverage.", "No repo-specific official outcome row was available; confidence is reduced."]), + }); expect(bundle.actions[0]?.publicSafeSummary).not.toMatch(/reward|wallet|hotkey|raw trust score|estimated score/i); + expect(JSON.stringify(bundle.actions[0]?.payload.recommendationEvidence)).not.toMatch(/wallet|hotkey|raw trust score/i); expect(bundle.contextSnapshots[0]).toMatchObject({ scoringModelId: "scoring-1", freshnessWarnings: expect.arrayContaining(["we-promise/sure: partial signal coverage", "we-promise/sure: stale signal coverage"]), @@ -238,6 +251,11 @@ describe("agent orchestrator", () => { payload: expect.objectContaining({ freshness: "rebuilding", rebuildEnqueued: true, refreshReason: "stale_decision_pack" }), }); expect(bundle.actions.length).toBeGreaterThan(0); + expect(bundle.actions[0]?.payload.recommendationEvidence).toMatchObject({ + confidence: "low", + freshness: "rebuilding", + warnings: expect.arrayContaining(["Decision pack is stale; a background rebuild was enqueued."]), + }); expect(bundle.contextSnapshots[0]?.freshnessWarnings ?? []).toEqual( expect.arrayContaining([expect.stringMatching(/^decision pack is stale.*background rebuild enqueued$/)]), ); @@ -344,7 +362,47 @@ describe("agent orchestrator", () => { const readyAction = __agentOrchestratorInternals.actionFromDecisionAction(run, action("open_new_direct_pr", "owner/ready", "pursue", 80), readyDecision, 2); const emptyNextAction = __agentOrchestratorInternals.actionFromDecisionAction(run, { ...action("open_new_direct_pr", "owner/ready", "pursue", 80), nextActions: [] }, readyDecision, 4); const repoFit = __agentOrchestratorInternals.actionFromRepoDecision(run, { ...readyDecision, nextActions: [] }, 3); + const outcomeRepoFit = __agentOrchestratorInternals.actionFromRepoDecision(run, { ...readyDecision, outcome: { repoFullName: "owner/ready" } as any }, 5); + const defaultEvidenceAction = __agentOrchestratorInternals.actionRecord({ + run, + actionType: "choose_next_work", + index: 6, + targetRepoFullName: "owner/default", + status: "recommended", + recommendation: "Use the default evidence fallback.", + why: [], + blockedBy: [], + publicSafeSummary: "owner/default: fallback.", + payload: {}, + }); const noDecisionActions = __agentOrchestratorInternals.buildDecisionActions(run, decisionPackFixture({ generatedAt, topActions: [], repoDecisions: [readyDecision] }), [readyDecision]); + const staleEvidenceActions = __agentOrchestratorInternals.buildDecisionActions( + run, + decisionPackFixture({ generatedAt, freshness: "stale", rebuildEnqueued: false, topActions: [action("open_new_direct_pr", "owner/ready", "pursue", 80)], repoDecisions: [readyDecision] }), + [readyDecision], + ); + const blockedFidelityActions = __agentOrchestratorInternals.buildDecisionActions( + run, + decisionPackFixture({ + generatedAt, + topActions: [action("open_new_direct_pr", "owner/ready", "pursue", 80)], + repoDecisions: [readyDecision], + dataQuality: { + signalFidelity: { + status: "blocked", + repoCount: 1, + completeRepos: 0, + degradedRepos: 0, + blockedRepos: 1, + partialRepos: [], + cappedRepos: [], + staleRepos: [], + rateLimitedRepos: [], + }, + }, + }), + [readyDecision], + ); const blockerFallback = __agentOrchestratorInternals.buildBlockerActions( run, decisionPackFixture({ generatedAt, repoDecisions: [criticalDecision], topActions: [] }), @@ -356,6 +414,29 @@ describe("agent orchestrator", () => { expect(repoFit.recommendation).toMatch(/repo fit/); expect(noDecisionActions[0]).toMatchObject({ actionType: "explain_repo_fit", status: "recommended" }); expect(blockerFallback[0]).toMatchObject({ actionType: "explain_score_blockers", status: "blocked" }); + expect(watchAction.payload.recommendationEvidence).toMatchObject({ + confidence: "medium", + sourceSummary: "Repo decision recommendation without serving-pack freshness metadata.", + freshness: "unknown", + }); + expect(outcomeRepoFit.payload.recommendationEvidence).toMatchObject({ + confidence: "high", + sources: expect.arrayContaining([expect.objectContaining({ name: "repo_outcome_history", freshness: "fresh" })]), + }); + expect(defaultEvidenceAction.payload.recommendationEvidence).toMatchObject({ + confidence: "medium", + sourceSummary: "Generated from Gittensory agent metadata.", + warnings: expect.arrayContaining(["Source-specific evidence was not attached; treat this recommendation as medium confidence."]), + }); + expect(staleEvidenceActions[0]?.payload.recommendationEvidence).toMatchObject({ + confidence: "low", + freshness: "stale", + warnings: expect.arrayContaining(["Decision pack is stale and no rebuild was enqueued."]), + }); + expect(blockedFidelityActions[0]?.payload.recommendationEvidence).toMatchObject({ + confidence: "low", + warnings: expect.arrayContaining(["Signal fidelity is blocked for this decision pack."]), + }); expect(__agentOrchestratorInternals.summarizeRun({ ...run, status: "failed", errorSummary: undefined }, [])).toContain("unknown"); const monitorRun = __agentOrchestratorInternals.buildRunRecord({ @@ -542,8 +623,16 @@ describe("agent orchestrator", () => { scoreabilityStatus: "scoreable", underlyingPotentialScore: 20, scoringModelSnapshotId: "scoring-ready", + generatedAt: nowIso(), + activeModel: "pending_saturation_model", + warnings: [], + assumptions: [], + scenarioPreviews: [{ name: "current", source: "current_data" }], + linkedIssueMultiplier: { status: "not_required", source: "none", reason: "No linked issue multiplier applies." }, }, scenarioScorePreview: { blockedBy: [] }, + branchEligibility: { required: false, status: "not_required", evidence: "provided", source: "missing", stale: false, warnings: [] }, + githubBranchStatus: { source: "cached_github_data", status: "no_pr", notes: [] }, rewardRisk: { summary: "Risk is acceptable." }, maintainerFit: { risks: [] }, recommendedRerunCondition: "Rerun before opening a PR.", @@ -563,12 +652,43 @@ describe("agent orchestrator", () => { scorePreview: { ...analysis.scorePreview, scoreabilityStatus: "blocked" }, scenarioScorePreview: { blockedBy: [{ detail: "openPrMultiplier is 0." }] }, }); + const assumptionHeavyActions = __agentOrchestratorInternals.buildLocalBranchActions(run, { + ...analysis, + baseFreshness: { ...analysis.baseFreshness, status: "possibly_stale", warnings: ["Base branch may be stale."] }, + dataQuality: { status: "degraded", warnings: ["Official mirror data is unavailable."] }, + githubBranchStatus: { source: "cached_github_data", status: "unknown", notes: ["GitHub branch status is incomplete."] }, + branchEligibility: { required: true, status: "unknown", evidence: "missing", source: "user_supplied", stale: true, warnings: ["Branch eligibility is stale."] }, + scorePreview: { + ...analysis.scorePreview, + warnings: ["Linked issue data is missing."], + assumptions: ["User scenario note: approved PRs may land.", "Private API/MCP output only; public comments intentionally omit these details."], + scenarioPreviews: [{ name: "afterPendingMerges", source: "user_supplied" }], + linkedIssueMultiplier: { status: "unavailable", source: "user_supplied", reason: "Linked issue mirror is unavailable." }, + }, + }); expect(actions.map((entry) => entry.actionType)).toEqual(["preflight_branch", "prepare_pr_packet"]); expect(actions[0]).toMatchObject({ status: "ready", scoreabilityImpact: "Current scoreability is not hard-blocked by branch metadata." }); + expect(actions[0]?.payload.recommendationEvidence).toMatchObject({ + confidence: "high", + sourceSummary: "Local branch preflight recommendation from structured metadata.", + freshness: "fresh", + sources: expect.arrayContaining([expect.objectContaining({ name: "local_branch_metadata", source: "metadata_only" })]), + }); expect(blockers[0]).toMatchObject({ status: "ready", recommendation: "No hard scoreability blocker is visible from local metadata." }); expect(blockedActions.map((entry) => entry.actionType)).toEqual(["preflight_branch", "prepare_pr_packet", "explain_score_blockers"]); expect(blockedActions[0]?.scoreabilityImpact).toContain("scenario projections"); + expect(assumptionHeavyActions[0]?.payload.recommendationEvidence).toMatchObject({ + confidence: "medium", + freshness: "possibly_stale", + userSuppliedScenarios: true, + sources: expect.arrayContaining([ + expect.objectContaining({ name: "github_branch_status", freshness: "unknown" }), + expect.objectContaining({ name: "linked_issue_multiplier", freshness: "missing" }), + ]), + warnings: expect.arrayContaining(["Base branch may be stale.", "GitHub branch status is incomplete.", "Branch eligibility is stale."]), + assumptions: expect.arrayContaining(["One or more scenario, linked-issue, or branch-eligibility inputs were supplied by the caller."]), + }); }); it("covers watch, pursue, and no-blocker decision branches", async () => { @@ -648,6 +768,55 @@ describe("agent orchestrator", () => { expect(missingRepoPlan.actions.length).toBeGreaterThan(0); }); + it("marks user-supplied pending scenarios and missing official data in action evidence", async () => { + const env = createTestEnv(); + const userScenarioPack = decisionPackFixture({ + profile: { + ...decisionPackFixture().profile, + source: "github_cache", + officialStats: null, + }, + openPrMonitor: { + login: "oktofeesh1", + generatedAt: nowIso(), + openPrCount: 0, + registeredRepoCount: 1, + cleanupFirst: false, + summary: "User supplied a pending-PR scenario.", + guidance: [], + pendingScenarios: [ + { + repoFullName: "we-promise/sure", + detection: { + source: "user_supplied", + pendingMergedPrCount: 1, + pendingClosedPrCount: 0, + approvedPrCount: 0, + expectedOpenPrCountAfterMerge: 1, + scenarioNotes: ["manual assumption"], + classified: [], + }, + }, + ], + pullRequests: [], + }, + }); + await persistDecisionPack(env, userScenarioPack); + + const bundle = await planNextWork(env, { login: "oktofeesh1", repoFullName: "we-promise/sure" }); + + expect(bundle.actions[0]?.payload.recommendationEvidence).toMatchObject({ + confidence: "low", + userSuppliedScenarios: true, + userSuppliedScenarioCount: 1, + assumptions: expect.arrayContaining([ + "Contributor-level official stats are missing, so cached GitHub and registry data carry more weight.", + "Pending-PR scenario projections include user-supplied assumptions.", + ]), + warnings: expect.arrayContaining(["Official Gittensor contributor stats were unavailable; confidence is reduced."]), + }); + }); + it("marks failed runs without throwing when local branch input is malformed", async () => { const env = createTestEnv(); const run: AgentRunRecord = {