diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 14ed5f94f..b3e995303 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -85,6 +85,14 @@ export type RagQualityResult = { unverifiedNumericTokenCount: number; hasFaithfulnessWarning: boolean; routingReason?: string; + /** Opening sentence of the answer — the span the text-shape gates inspect. */ + answerOpeningSentence?: string | null; + /** + * Full answer text, recorded only when a text-shape gate rejected it. This is the + * pre-fallback candidate text the gate actually judged (RagAnswer.rejectedCandidateText), + * not the generic fallback text ultimately shown to the user. + */ + answerText?: string | null; timings?: { retrievalMs: number; routingMs: number; @@ -208,7 +216,7 @@ export function ragAnswerTimingDiagnostics( }; } -const qualityThresholds = { +export const qualityThresholds = { retrievalTopKHitRate: 0.8, retrievalDocumentRecallAt5: 0.8, retrievalContentRecallAt5: 0.8, @@ -234,10 +242,16 @@ const qualityThresholds = { // // Do not add an entry to clear a red run. A block here names the offending case and reason: // investigate that case, exactly as canary 32589154243 did for these two. - ragSourceBackedReviewFallbackAllowance: [ - { id: "quality-antipsychotic-metabolic-monitoring", reason: "guidance_wrapper_fragment" }, - { id: "quality-discharge-documentation", reason: "guidance_wrapper_fragment" }, - ] as ReadonlyArray<{ id: string; reason: string }>, + // + // EMPTIED 2026-09-02, and this is a no-op rather than a policy change. Both entries were + // written as `quality-` while every id in ragEvalCases is bare, and the match is + // `allowed.id === result.id`, so neither has excused a single case since PR #2301 landed. + // The canary has been red continuously since 2026-08-22 on cases nobody accepted, while the + // source read as though two degradations had been signed off. Removing dead entries changes + // no runtime behaviour and stops the file asserting a waiver that does not exist; the + // accompanying test in tests/eval-quality.test.ts fails if an id is ever added that names no + // case. Re-accepting a degradation now requires naming a real case id, with evidence. + ragSourceBackedReviewFallbackAllowance: [] as ReadonlyArray<{ id: string; reason: string }>, numericGroundingFailureRate: 0, staleTopResultRate: 0.25, reviewRequiredTopResultRate: 0.25, @@ -428,6 +442,27 @@ function rate(numerator: number, denominator: number) { return denominator === 0 ? 0 : Number((numerator / denominator).toFixed(4)); } +const TEXT_SHAPE_GATE_REASONS = [ + "guidance_wrapper_fragment", + "bare_document_title_list", + "provider_source_gap", + "empty_after_sanitize", +] as const; + +/** True when a gate that judges the answer PROSE rejected this answer. */ +function textShapeGateRejected(routingReason: string | undefined) { + const reason = (routingReason ?? "").toLowerCase(); + return TEXT_SHAPE_GATE_REASONS.some((token) => reason.includes(token)); +} + +/** The opening sentence, which is the span the wrapper predicate actually inspects. */ +function openingSentenceOf(answer: string | undefined | null) { + const text = (answer ?? "").replace(/\s+/g, " ").trim(); + if (!text) return null; + const stop = text.search(/[.!?](\s|$)/); + return (stop === -1 ? text : text.slice(0, stop + 1)).slice(0, 400); +} + function isSourceBackedReviewFallback(routingReason: string | undefined) { return (routingReason ?? "") .split(";") @@ -568,7 +603,12 @@ function topResultGovernanceCounts(results: GoldenRetrievalResult[]) { }; } -function summarizeRagQualityResults(results: RagQualityResult[], providerMode: EvalQualityProviderMode) { +function summarizeRagQualityResults( + results: RagQualityResult[], + providerMode: EvalQualityProviderMode, + allowanceOverride?: ReadonlyArray<{ id: string; reason: string }>, +) { + const fallbackAllowance = allowanceOverride ?? qualityThresholds.ragSourceBackedReviewFallbackAllowance; const supported = results.filter((result) => result.supported); const unsupported = results.filter((result) => !result.supported); // A supported case counts as grounded-supported when it grounds, OR — for @@ -614,7 +654,7 @@ function summarizeRagQualityResults(results: RagQualityResult[], providerMode: E const sourceBackedReviewFallbackUnaccounted = sourceBackedReviewFallbackResults .filter( (result) => - !qualityThresholds.ragSourceBackedReviewFallbackAllowance.some( + !fallbackAllowance.some( (allowed) => allowed.id === result.id && (result.routingReason ?? "").toLowerCase().includes(allowed.reason), ), ) @@ -678,10 +718,18 @@ export function buildEvalQualityReport(args: { ragResults: RagQualityResult[]; sourceMetadataDebtAcceptance?: SourceMetadataDebtAcceptance; providerMode?: EvalQualityProviderMode; + // Tests supply their own allowance so the mechanism can be exercised without depending on + // whatever the production list happens to contain. The production list and the test fixtures + // previously shared the same typo'd ids, so both agreed and neither could fail. + sourceBackedReviewFallbackAllowance?: ReadonlyArray<{ id: string; reason: string }>; }) { const providerMode = args.providerMode ?? "openai"; const retrievalSummary = summarizeGoldenRetrievalResults(args.retrievalResults); - const ragSummary = summarizeRagQualityResults(args.ragResults, providerMode); + const ragSummary = summarizeRagQualityResults( + args.ragResults, + providerMode, + args.sourceBackedReviewFallbackAllowance, + ); // `--rag-only` intentionally leaves retrieval metrics and gates empty. The // canary can still supply the preceding golden-retrieval artifact so this // report renders its source-governance metadata without rerunning retrieval @@ -1254,6 +1302,24 @@ async function runRagQualityCases(args: { unverifiedNumericTokenCount: answer.unverifiedNumericTokens?.length ?? 0, hasFaithfulnessWarning: Boolean(answer.faithfulnessWarning), routingReason: answer.routingReason, + // The text-shape gates (guidance_wrapper_fragment, bare_document_title_list, + // provider_source_gap) judge the ANSWER PROSE and then the harness discarded it, so a + // blocked canary could not be investigated without paying for another live run. That is + // why #NPQJKP sat red for eleven days. Record what the gate actually read: the opening + // sentence always (it is what isLaunderedGuidanceWrapperAnswer inspects), and the full + // text only for the cases a text-shape gate rejected, so reports stay small. + // + // The rejected candidate's text is not `answer.answer` by the time we get here — for a + // gate rejection, `finalizeRagAnswerQualityCore` has already replaced `answer.answer` + // with a generic evidence-gap response, and for a source-backed-review rejection, rag.ts + // has additionally replaced it again with a different generic fallback answer. Prefer + // `answer.rejectedCandidateText`, the pre-replacement text the gate actually judged + // (see RagAnswer.rejectedCandidateText); fall back to `answer.answer` only for the rare + // case a text-shape reason fired without that field being populated. + answerOpeningSentence: openingSentenceOf(answer.answer), + answerText: textShapeGateRejected(answer.routingReason) + ? (answer.rejectedCandidateText ?? answer.answer ?? null) + : undefined, timings, routeCeilingExceeded, executionType: diff --git a/src/lib/rag/rag-abort-signal.ts b/src/lib/rag/rag-abort-signal.ts new file mode 100644 index 000000000..5d2255f9b --- /dev/null +++ b/src/lib/rag/rag-abort-signal.ts @@ -0,0 +1,19 @@ +export function awaitWithCallerSignal(pending: Promise, signal?: AbortSignal): Promise { + if (!signal) return pending; + if (signal.aborted) throw signal.reason ?? new DOMException("The operation was aborted.", "AbortError"); + + return new Promise((resolve, reject) => { + const onAbort = () => reject(signal.reason ?? new DOMException("The operation was aborted.", "AbortError")); + signal.addEventListener("abort", onAbort, { once: true }); + pending.then( + (value) => { + signal.removeEventListener("abort", onAbort); + resolve(value); + }, + (error) => { + signal.removeEventListener("abort", onAbort); + reject(error); + }, + ); + }); +} diff --git a/src/lib/rag/rag-classifier-fallback.ts b/src/lib/rag/rag-classifier-fallback.ts new file mode 100644 index 000000000..37246e600 --- /dev/null +++ b/src/lib/rag/rag-classifier-fallback.ts @@ -0,0 +1,284 @@ +import { createAdminClient } from "@/lib/supabase/admin"; +import { classifyCorpusGrounding } from "@/lib/corpus-grounding"; +import { generateParsedTextResult, openAISafetyIdentifier } from "@/lib/openai"; +import { env } from "@/lib/env"; +import { ragQueryClassifierPromptVersion } from "@/lib/rag/rag-versioning"; +import { hasAdversarialManipulationIntent } from "@/lib/rag/rag-routing"; +import { + clearlyOutsideCorpusMedicalPattern, + isUnsupportedSoftTailAnalysis, + unavailableDocumentNoisePattern, +} from "@/lib/rag/rag-query-guard"; +import { awaitWithCallerSignal } from "@/lib/rag/rag-abort-signal"; +import type { ClinicalQueryAnalysis } from "@/lib/types"; +import { z } from "zod"; + +const queryClassifierParseSchema = z + .object({ + queryClass: z.enum([ + "document_lookup", + "table_threshold", + "medication_dose_risk", + "comparison", + "broad_summary", + "unsupported_or_general", + ]), + confidence: z.number(), + reasons: z.array(z.string()), + expandedTerms: z.array(z.string()), + }) + .strict(); + +const queryClassifierVerdictSchema = queryClassifierParseSchema.extend({ + confidence: z.number().min(0).max(1), + reasons: z.array(z.string().max(80)).max(4), + expandedTerms: z.array(z.string().max(60)).max(10), +}); + +/** Unique text values. */ +export function uniqueTextValues(values: Array, limit = 32) { + const seen = new Set(); + const output: string[] = []; + for (const value of values) { + const normalized = value?.replace(/\s+/g, " ").trim(); + if (!normalized) continue; + const key = normalized.toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + output.push(normalized); + if (output.length >= limit) break; + } + return output; +} + +type ClassifierVerdict = z.infer; + +// Finding #11 interim fix (docs/process-hardening.md): the LLM classifier verdict flips +// run-to-run for the same query, so the unsupported short-circuit downstream intermittently +// returned 0 results for valid in-corpus topics. Memoizing successful verdicts makes the +// verdict — and therefore retrieval behaviour — deterministic per query for the TTL window. +// Only *successful* classifier calls are memoized (accepted and rejected verdicts alike); +// transport errors and timeouts stay retryable, otherwise one transient 6s timeout would pin +// a query's classification for the whole TTL. The full corpus-grounded relevance fix remains +// scoped to RAG optimisation Phase 2. +const classifierVerdictMemoTtlMs = 15 * 60 * 1000; +// Finding #11 follow-up: bounds retries for a rejected soft-tail verdict (isUnsupportedSoftTailAnalysis). +const rejectedSoftTailMemoTtlMs = 60 * 1000; +const classifierVerdictMemoMaxEntries = 500; +const classifierVerdictMemo = new Map(); +const classifierVerdictInflight = new Map>(); + +/** Classifier verdict memo key. */ +function classifierVerdictMemoKey(query: string, analysis: ClinicalQueryAnalysis) { + const normalizedQuery = query.normalize("NFKC").toLowerCase().replace(/\s+/g, " ").trim(); + // The deterministic class + confidence bucket are part of the key so a deterministic-analyzer + // change invalidates stale verdicts instead of replaying them against a different baseline. + return [ + env.OPENAI_QUERY_CLASSIFIER_MODEL, + ragQueryClassifierPromptVersion, + normalizedQuery, + analysis.queryClass, + analysis.confidence.toFixed(2), + ].join("::"); +} + +/** Store classifier verdict memo. */ +function storeClassifierVerdictMemo(key: string, verdict: ClassifierVerdict, ttlMs = classifierVerdictMemoTtlMs) { + if (classifierVerdictMemo.size >= classifierVerdictMemoMaxEntries) { + const oldestKey = classifierVerdictMemo.keys().next().value; + if (oldestKey !== undefined) classifierVerdictMemo.delete(oldestKey); + } + classifierVerdictMemo.set(key, { expiresAt: Date.now() + ttlMs, verdict }); +} + +/** Reset classifier verdict memo for tests. */ +export function resetClassifierVerdictMemoForTests() { + classifierVerdictMemo.clear(); + classifierVerdictInflight.clear(); +} + +/** Request classifier verdict. */ +async function requestClassifierVerdict( + query: string, + analysis: ClinicalQueryAnalysis, + ownerId?: string | null, +): Promise { + const result = await generateParsedTextResult( + [ + { + role: "user", + content: [ + { + type: "input_text", + text: [ + `Query: ${query}`, + `Deterministic query class: ${analysis.queryClass}`, + `Deterministic confidence: ${analysis.confidence}`, + `Known expanded terms: ${analysis.expandedTerms.join(", ") || "none"}`, + ].join("\n"), + }, + ], + }, + ], + queryClassifierParseSchema, + { + model: env.OPENAI_QUERY_CLASSIFIER_MODEL, + maxOutputTokens: 220, + operation: "text_generation", + instructions: + "Classify this query for retrieval routing only. Do not answer the clinical question. Prefer unsupported when the query is not about indexed clinical document retrieval.", + reasoningEffort: "low", + textVerbosity: "low", + schemaName: "clinical_rag_query_classifier", + promptCacheKey: ragQueryClassifierPromptVersion, + timeoutMs: 6000, + safetyIdentifier: env.OPENAI_SAFETY_IDENTIFIER_SECRET ? openAISafetyIdentifier(ownerId) : undefined, + }, + ); + return queryClassifierVerdictSchema.parse(result.parsed); +} + +/** Apply classifier verdict. */ +function applyClassifierVerdict(analysis: ClinicalQueryAnalysis, parsed: ClassifierVerdict): ClinicalQueryAnalysis { + if (parsed.confidence < 0.58 || parsed.queryClass === "unsupported_or_general") return analysis; + return { + ...analysis, + queryClass: parsed.queryClass, + confidence: Math.max(analysis.confidence, parsed.confidence), + needsClassifierFallback: false, + needsSynthesis: + analysis.needsSynthesis || + parsed.queryClass === "comparison" || + parsed.queryClass === "broad_summary" || + parsed.queryClass === "medication_dose_risk", + expandedTerms: uniqueTextValues([...analysis.expandedTerms, ...parsed.expandedTerms], 36), + queryRewrite: { + ...analysis.queryRewrite, + expansions: uniqueTextValues([...analysis.queryRewrite.expansions, ...parsed.expandedTerms], 48), + searchQuery: uniqueTextValues( + [analysis.queryRewrite.searchQuery, ...analysis.queryRewrite.expansions, ...parsed.expandedTerms], + 60, + ).join(" "), + reasons: uniqueTextValues([...analysis.queryRewrite.reasons, ...parsed.reasons, "classifier_fallback"], 16), + }, + reasons: uniqueTextValues([...analysis.reasons, ...parsed.reasons, "classifier_fallback"], 12), + } satisfies ClinicalQueryAnalysis; +} + +/** Analyze query with classifier fallback. */ +export async function analyzeQueryWithClassifierFallback( + query: string, + analysis: ClinicalQueryAnalysis, + opts?: { + // Finding #11 corpus grounding: when provided, unsupported-soft-tail queries are checked + // against the corpus BEFORE the nondeterministic LLM classifier. Scoped with the exact + // owner_filter retrieval will use so grounding can never see documents retrieval cannot. + corpusGrounding?: { supabase: ReturnType; ownerFilter: string | null }; + ownerId?: string | null; + signal?: AbortSignal; + }, +) { + if ( + // Fail closed before any generative model call: an adversarial-manipulation + // query is routed to "unsupported" downstream, so never send its text to the + // LLM query classifier. (Embedding-based retrieval is non-generative and not + // an injection surface.) + hasAdversarialManipulationIntent(query) || + unavailableDocumentNoisePattern.test(query) || + (clearlyOutsideCorpusMedicalPattern.test(query) && analysis.documentTitleTerms.length === 0) + ) { + return { ...analysis, needsClassifierFallback: false } satisfies ClinicalQueryAnalysis; + } + + // Finding #11 corpus-grounded relevance: for queries that would hit the unsupported soft + // tail, the corpus — not the LLM — decides. An in-corpus bare topic ("bipolar disorder") + // deterministically reclassifies to broad_summary (mirroring what an accepted classifier + // verdict would have done, minus the coin flip); a corpus-absent query ("florbizone syndrome + // management") skips the LLM entirely so the soft-tail refusal is deterministic — and typos + // remain rescuable because the short-circuit path still runs trigram correction afterwards. + // "inconclusive" (including DB errors and an unapplied migration) keeps legacy behaviour. + // This deliberately runs before the OPENAI_API_KEY gate: offline/source-only deployments + // still retrieve lexically, so in-corpus bare topics should answer there too. + if (opts?.corpusGrounding && isUnsupportedSoftTailAnalysis(query, analysis)) { + const grounding = await classifyCorpusGrounding({ + supabase: opts.corpusGrounding.supabase, + query, + ownerFilter: opts.corpusGrounding.ownerFilter, + }); + if (grounding.verdict === "in_corpus_topic") { + return { + ...analysis, + queryClass: "broad_summary", + confidence: Math.max(analysis.confidence, 0.62), + needsSynthesis: true, + needsClassifierFallback: false, + corpusGrounding: "in_corpus_topic", + reasons: uniqueTextValues([...analysis.reasons, "corpus_topic_grounding"], 12), + } satisfies ClinicalQueryAnalysis; + } + if (grounding.verdict === "out_of_corpus") { + // Do NOT touch queryClass/confidence/reasons: the existing soft-tail short-circuit (and + // its alias-expansion + trigram-correction escape hatches) must keep firing exactly as + // before — only the LLM lottery is removed. + return { + ...analysis, + needsClassifierFallback: false, + corpusGrounding: "out_of_corpus", + } satisfies ClinicalQueryAnalysis; + } + analysis = { ...analysis, corpusGrounding: "inconclusive" }; + } + + // Finding #2: Deterministic fallback routing for short clinical queries. + // Short, bare clinical search queries (e.g., "bipolar disorder", "anorexia management") + // can be misclassified by the generative LLM. We route them deterministically. + if ( + analysis.needsClassifierFallback && + analysis.corpusGrounding !== "inconclusive" && + query.trim().split(/\s+/).length <= 4 && + (analysis.documentTitleTerms.length > 0 || analysis.canonicalTerms.length > 0) + ) { + return { + ...analysis, + queryClass: "broad_summary", + needsClassifierFallback: false, + reasons: uniqueTextValues([...analysis.reasons, "deterministic_short_clinical_query_fallback"], 12), + } satisfies ClinicalQueryAnalysis; + } + + if (!analysis.needsClassifierFallback || !env.OPENAI_API_KEY) return analysis; + + const memoKey = classifierVerdictMemoKey(query, analysis); + const memoized = classifierVerdictMemo.get(memoKey); + if (memoized) { + if (memoized.expiresAt > Date.now()) return applyClassifierVerdict(analysis, memoized.verdict); + classifierVerdictMemo.delete(memoKey); + } + + let pending = classifierVerdictInflight.get(memoKey); + if (!pending) { + pending = requestClassifierVerdict(query, analysis, opts?.ownerId).finally(() => { + classifierVerdictInflight.delete(memoKey); + }); + classifierVerdictInflight.set(memoKey, pending); + } + + try { + const verdict = await awaitWithCallerSignal(pending, opts?.signal); + // Finding #11 follow-up: bounded TTL for a rejected soft-tail verdict — see the constant above. + const rejected = verdict.confidence < 0.58 || verdict.queryClass === "unsupported_or_general"; + const softTail = rejected && isUnsupportedSoftTailAnalysis(query, analysis); + storeClassifierVerdictMemo(memoKey, verdict, softTail ? rejectedSoftTailMemoTtlMs : undefined); + return applyClassifierVerdict(analysis, verdict); + } catch (error) { + if ( + error && + (error instanceof DOMException || typeof error === "object") && + (error as { name?: string }).name === "AbortError" + ) + throw error; + // Transport/parse failures are deliberately NOT memoized: fall back to the deterministic + // analysis for this request only, and let the next request retry the classifier. + return analysis; + } +} diff --git a/src/lib/rag/rag-extractive-answer.ts b/src/lib/rag/rag-extractive-answer.ts index 6281869bb..b13466536 100644 --- a/src/lib/rag/rag-extractive-answer.ts +++ b/src/lib/rag/rag-extractive-answer.ts @@ -3521,6 +3521,7 @@ function finalQualityFailure(answer: RagAnswer, query: string, queryClass: RagQu return { ...answer, answer: finalQualityGapAnswer(query, queryClass), + rejectedCandidateText: answer.answer, grounded: false, confidence: "unsupported", answerSections: [], @@ -3972,6 +3973,7 @@ function finalizeRagAnswerQualityCore(answer: RagAnswer, query: string, queryCla return { ...answer, answer: gapAnswer, + rejectedCandidateText: answer.answer, grounded: false, confidence: "unsupported", citations: [], diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 94634b4b7..d103900fa 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -24,11 +24,9 @@ export { loadChunksForMemoryCards, loadChunksForSignalMatches, } from "@/lib/rag/rag-candidate-sources"; -import { classifyCorpusGrounding } from "@/lib/corpus-grounding"; import type { Database, Json } from "@/lib/supabase/database.types"; import { embedTextWithTelemetry, - generateParsedTextResult, generateStructuredTextResult, openAISafetyIdentifier, type OpenAITextResult, @@ -165,11 +163,7 @@ import { rankClinicalResults, } from "@/lib/clinical-search"; import { env, requestedOpenAIAnswerModels } from "@/lib/env"; -import { - ragAnswerPromptVersion, - ragQueryClassifierPromptVersion, - ragSummaryPromptVersion, -} from "@/lib/rag/rag-versioning"; +import { ragAnswerPromptVersion, ragSummaryPromptVersion } from "@/lib/rag/rag-versioning"; import { answerPrivacyMetadata, answerTextForStorage, @@ -219,13 +213,16 @@ export { import { retrievalPlanForQueryClass, type SearchChunksArgs, type SearchTelemetry } from "@/lib/rag/rag-contracts"; export { retrievalPlanForQueryClass, type SearchChunksArgs, type SearchTelemetry } from "@/lib/rag/rag-contracts"; import { - clearlyOutsideCorpusMedicalPattern, - isUnsupportedSoftTailAnalysis, shouldSkipUnsupportedSoftTailAnswerCacheWrite, shouldSkipUnsupportedSoftTailCacheWrite, - unavailableDocumentNoisePattern, } from "@/lib/rag/rag-query-guard"; export { shouldShortCircuitUnsupportedSearch } from "@/lib/rag/rag-query-guard"; +import { analyzeQueryWithClassifierFallback, uniqueTextValues } from "@/lib/rag/rag-classifier-fallback"; +export { + analyzeQueryWithClassifierFallback, + resetClassifierVerdictMemoForTests, +} from "@/lib/rag/rag-classifier-fallback"; +import { awaitWithCallerSignal } from "@/lib/rag/rag-abort-signal"; import { hasAdmissionCommunityLookupIntent, hasAdmissionCommunityTitleSupport, @@ -472,26 +469,6 @@ function throwIfAborted(signal?: AbortSignal) { } } -function awaitWithCallerSignal(pending: Promise, signal?: AbortSignal): Promise { - if (!signal) return pending; - if (signal.aborted) throw signal.reason ?? new DOMException("The operation was aborted.", "AbortError"); - - return new Promise((resolve, reject) => { - const onAbort = () => reject(signal.reason ?? new DOMException("The operation was aborted.", "AbortError")); - signal.addEventListener("abort", onAbort, { once: true }); - pending.then( - (value) => { - signal.removeEventListener("abort", onAbort); - resolve(value); - }, - (error) => { - signal.removeEventListener("abort", onAbort); - reject(error); - }, - ); - }); -} - export type AnswerProgressEvent = { stage: | "retrieved" @@ -922,276 +899,6 @@ function hasOpenAIUsage(usage: OpenAITokenUsage) { return Object.values(usage).some((value) => typeof value === "number" && value > 0); } -const queryClassifierParseSchema = z - .object({ - queryClass: z.enum([ - "document_lookup", - "table_threshold", - "medication_dose_risk", - "comparison", - "broad_summary", - "unsupported_or_general", - ]), - confidence: z.number(), - reasons: z.array(z.string()), - expandedTerms: z.array(z.string()), - }) - .strict(); - -const queryClassifierVerdictSchema = queryClassifierParseSchema.extend({ - confidence: z.number().min(0).max(1), - reasons: z.array(z.string().max(80)).max(4), - expandedTerms: z.array(z.string().max(60)).max(10), -}); - -/** Unique text values. */ -function uniqueTextValues(values: Array, limit = 32) { - const seen = new Set(); - const output: string[] = []; - for (const value of values) { - const normalized = value?.replace(/\s+/g, " ").trim(); - if (!normalized) continue; - const key = normalized.toLowerCase(); - if (seen.has(key)) continue; - seen.add(key); - output.push(normalized); - if (output.length >= limit) break; - } - return output; -} - -type ClassifierVerdict = z.infer; - -// Finding #11 interim fix (docs/process-hardening.md): the LLM classifier verdict flips -// run-to-run for the same query, so the unsupported short-circuit downstream intermittently -// returned 0 results for valid in-corpus topics. Memoizing successful verdicts makes the -// verdict — and therefore retrieval behaviour — deterministic per query for the TTL window. -// Only *successful* classifier calls are memoized (accepted and rejected verdicts alike); -// transport errors and timeouts stay retryable, otherwise one transient 6s timeout would pin -// a query's classification for the whole TTL. The full corpus-grounded relevance fix remains -// scoped to RAG optimisation Phase 2. -const classifierVerdictMemoTtlMs = 15 * 60 * 1000; -// Finding #11 follow-up: bounds retries for a rejected soft-tail verdict (isUnsupportedSoftTailAnalysis). -const rejectedSoftTailMemoTtlMs = 60 * 1000; -const classifierVerdictMemoMaxEntries = 500; -const classifierVerdictMemo = new Map(); -const classifierVerdictInflight = new Map>(); - -/** Classifier verdict memo key. */ -function classifierVerdictMemoKey(query: string, analysis: ClinicalQueryAnalysis) { - const normalizedQuery = query.normalize("NFKC").toLowerCase().replace(/\s+/g, " ").trim(); - // The deterministic class + confidence bucket are part of the key so a deterministic-analyzer - // change invalidates stale verdicts instead of replaying them against a different baseline. - return [ - env.OPENAI_QUERY_CLASSIFIER_MODEL, - ragQueryClassifierPromptVersion, - normalizedQuery, - analysis.queryClass, - analysis.confidence.toFixed(2), - ].join("::"); -} - -/** Store classifier verdict memo. */ -function storeClassifierVerdictMemo(key: string, verdict: ClassifierVerdict, ttlMs = classifierVerdictMemoTtlMs) { - if (classifierVerdictMemo.size >= classifierVerdictMemoMaxEntries) { - const oldestKey = classifierVerdictMemo.keys().next().value; - if (oldestKey !== undefined) classifierVerdictMemo.delete(oldestKey); - } - classifierVerdictMemo.set(key, { expiresAt: Date.now() + ttlMs, verdict }); -} - -/** Reset classifier verdict memo for tests. */ -export function resetClassifierVerdictMemoForTests() { - classifierVerdictMemo.clear(); - classifierVerdictInflight.clear(); -} - -/** Request classifier verdict. */ -async function requestClassifierVerdict( - query: string, - analysis: ClinicalQueryAnalysis, - ownerId?: string | null, -): Promise { - const result = await generateParsedTextResult( - [ - { - role: "user", - content: [ - { - type: "input_text", - text: [ - `Query: ${query}`, - `Deterministic query class: ${analysis.queryClass}`, - `Deterministic confidence: ${analysis.confidence}`, - `Known expanded terms: ${analysis.expandedTerms.join(", ") || "none"}`, - ].join("\n"), - }, - ], - }, - ], - queryClassifierParseSchema, - { - model: env.OPENAI_QUERY_CLASSIFIER_MODEL, - maxOutputTokens: 220, - operation: "text_generation", - instructions: - "Classify this query for retrieval routing only. Do not answer the clinical question. Prefer unsupported when the query is not about indexed clinical document retrieval.", - reasoningEffort: "low", - textVerbosity: "low", - schemaName: "clinical_rag_query_classifier", - promptCacheKey: ragQueryClassifierPromptVersion, - timeoutMs: 6000, - safetyIdentifier: env.OPENAI_SAFETY_IDENTIFIER_SECRET ? openAISafetyIdentifier(ownerId) : undefined, - }, - ); - return queryClassifierVerdictSchema.parse(result.parsed); -} - -/** Apply classifier verdict. */ -function applyClassifierVerdict(analysis: ClinicalQueryAnalysis, parsed: ClassifierVerdict): ClinicalQueryAnalysis { - if (parsed.confidence < 0.58 || parsed.queryClass === "unsupported_or_general") return analysis; - return { - ...analysis, - queryClass: parsed.queryClass, - confidence: Math.max(analysis.confidence, parsed.confidence), - needsClassifierFallback: false, - needsSynthesis: - analysis.needsSynthesis || - parsed.queryClass === "comparison" || - parsed.queryClass === "broad_summary" || - parsed.queryClass === "medication_dose_risk", - expandedTerms: uniqueTextValues([...analysis.expandedTerms, ...parsed.expandedTerms], 36), - queryRewrite: { - ...analysis.queryRewrite, - expansions: uniqueTextValues([...analysis.queryRewrite.expansions, ...parsed.expandedTerms], 48), - searchQuery: uniqueTextValues( - [analysis.queryRewrite.searchQuery, ...analysis.queryRewrite.expansions, ...parsed.expandedTerms], - 60, - ).join(" "), - reasons: uniqueTextValues([...analysis.queryRewrite.reasons, ...parsed.reasons, "classifier_fallback"], 16), - }, - reasons: uniqueTextValues([...analysis.reasons, ...parsed.reasons, "classifier_fallback"], 12), - } satisfies ClinicalQueryAnalysis; -} - -/** Analyze query with classifier fallback. */ -export async function analyzeQueryWithClassifierFallback( - query: string, - analysis: ClinicalQueryAnalysis, - opts?: { - // Finding #11 corpus grounding: when provided, unsupported-soft-tail queries are checked - // against the corpus BEFORE the nondeterministic LLM classifier. Scoped with the exact - // owner_filter retrieval will use so grounding can never see documents retrieval cannot. - corpusGrounding?: { supabase: ReturnType; ownerFilter: string | null }; - ownerId?: string | null; - signal?: AbortSignal; - }, -) { - if ( - // Fail closed before any generative model call: an adversarial-manipulation - // query is routed to "unsupported" downstream, so never send its text to the - // LLM query classifier. (Embedding-based retrieval is non-generative and not - // an injection surface.) - hasAdversarialManipulationIntent(query) || - unavailableDocumentNoisePattern.test(query) || - (clearlyOutsideCorpusMedicalPattern.test(query) && analysis.documentTitleTerms.length === 0) - ) { - return { ...analysis, needsClassifierFallback: false } satisfies ClinicalQueryAnalysis; - } - - // Finding #11 corpus-grounded relevance: for queries that would hit the unsupported soft - // tail, the corpus — not the LLM — decides. An in-corpus bare topic ("bipolar disorder") - // deterministically reclassifies to broad_summary (mirroring what an accepted classifier - // verdict would have done, minus the coin flip); a corpus-absent query ("florbizone syndrome - // management") skips the LLM entirely so the soft-tail refusal is deterministic — and typos - // remain rescuable because the short-circuit path still runs trigram correction afterwards. - // "inconclusive" (including DB errors and an unapplied migration) keeps legacy behaviour. - // This deliberately runs before the OPENAI_API_KEY gate: offline/source-only deployments - // still retrieve lexically, so in-corpus bare topics should answer there too. - if (opts?.corpusGrounding && isUnsupportedSoftTailAnalysis(query, analysis)) { - const grounding = await classifyCorpusGrounding({ - supabase: opts.corpusGrounding.supabase, - query, - ownerFilter: opts.corpusGrounding.ownerFilter, - }); - if (grounding.verdict === "in_corpus_topic") { - return { - ...analysis, - queryClass: "broad_summary", - confidence: Math.max(analysis.confidence, 0.62), - needsSynthesis: true, - needsClassifierFallback: false, - corpusGrounding: "in_corpus_topic", - reasons: uniqueTextValues([...analysis.reasons, "corpus_topic_grounding"], 12), - } satisfies ClinicalQueryAnalysis; - } - if (grounding.verdict === "out_of_corpus") { - // Do NOT touch queryClass/confidence/reasons: the existing soft-tail short-circuit (and - // its alias-expansion + trigram-correction escape hatches) must keep firing exactly as - // before — only the LLM lottery is removed. - return { - ...analysis, - needsClassifierFallback: false, - corpusGrounding: "out_of_corpus", - } satisfies ClinicalQueryAnalysis; - } - analysis = { ...analysis, corpusGrounding: "inconclusive" }; - } - - // Finding #2: Deterministic fallback routing for short clinical queries. - // Short, bare clinical search queries (e.g., "bipolar disorder", "anorexia management") - // can be misclassified by the generative LLM. We route them deterministically. - if ( - analysis.needsClassifierFallback && - analysis.corpusGrounding !== "inconclusive" && - query.trim().split(/\s+/).length <= 4 && - (analysis.documentTitleTerms.length > 0 || analysis.canonicalTerms.length > 0) - ) { - return { - ...analysis, - queryClass: "broad_summary", - needsClassifierFallback: false, - reasons: uniqueTextValues([...analysis.reasons, "deterministic_short_clinical_query_fallback"], 12), - } satisfies ClinicalQueryAnalysis; - } - - if (!analysis.needsClassifierFallback || !env.OPENAI_API_KEY) return analysis; - - const memoKey = classifierVerdictMemoKey(query, analysis); - const memoized = classifierVerdictMemo.get(memoKey); - if (memoized) { - if (memoized.expiresAt > Date.now()) return applyClassifierVerdict(analysis, memoized.verdict); - classifierVerdictMemo.delete(memoKey); - } - - let pending = classifierVerdictInflight.get(memoKey); - if (!pending) { - pending = requestClassifierVerdict(query, analysis, opts?.ownerId).finally(() => { - classifierVerdictInflight.delete(memoKey); - }); - classifierVerdictInflight.set(memoKey, pending); - } - - try { - const verdict = await awaitWithCallerSignal(pending, opts?.signal); - // Finding #11 follow-up: bounded TTL for a rejected soft-tail verdict — see the constant above. - const rejected = verdict.confidence < 0.58 || verdict.queryClass === "unsupported_or_general"; - const softTail = rejected && isUnsupportedSoftTailAnalysis(query, analysis); - storeClassifierVerdictMemo(memoKey, verdict, softTail ? rejectedSoftTailMemoTtlMs : undefined); - return applyClassifierVerdict(analysis, verdict); - } catch (error) { - if ( - error && - (error instanceof DOMException || typeof error === "object") && - (error as { name?: string }).name === "AbortError" - ) - throw error; - // Transport/parse failures are deliberately NOT memoized: fall back to the deterministic - // analysis for this request only, and let the next request retry the classifier. - return analysis; - } -} - /** Metadata expansion term score. */ function metadataExpansionTermScore(queryTokens: Set, value: string, sourceWeight: number) { const tokens = normalizedClinicalSearchTokens(value); @@ -3116,6 +2823,11 @@ async function answerQuestionWithScopeUncoalesced( "ungrounded_extractive_answer"; const reviewRouteReason = `${finalizedAnswer.routingReason ?? answer.routingReason ?? route.reason}; ${SOURCE_BACKED_REVIEW_FALLBACK_REASON}; extractive_quality_gate:${extractiveQualityReason}`; const reviewPlan = buildCurrentSmartApiPlan("extractive", reviewRouteReason); + // The candidate text the quality gate actually judged and rejected — captured before + // finalizeAnswer below builds a fresh fallback candidate from + // sourceBackedGenerationTimeoutAnswer() and overwrites `finalizedAnswer`. Debug-only; see + // RagAnswer.rejectedCandidateText. + const priorRejectedCandidateText = finalizedAnswer.rejectedCandidateText ?? finalizedAnswer.answer; finalizedAnswer = finalizeAnswer({ ...answer, answer: boldHighYieldClinicalText(sourceBackedGenerationTimeoutAnswer(args.query), args.query), @@ -3129,6 +2841,7 @@ async function answerQuestionWithScopeUncoalesced( smartApiPlan: reviewPlan, answerSections: [], }); + finalizedAnswer.rejectedCandidateText ??= priorRejectedCandidateText; } if (args.logQuery !== false) diff --git a/src/lib/types.ts b/src/lib/types.ts index 9b4854374..10dbbd767 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -1106,6 +1106,12 @@ export type RagAnswer = { // When non-empty the answer should be treated as needing source verification. unverifiedNumericTokens?: string[]; faithfulnessWarning?: string; + // Debug-only: when a final quality gate rejects a candidate's answer prose and replaces + // `answer` with an evidence-gap response, this preserves the pre-replacement candidate text + // so eval tooling (scripts/eval-quality.ts) can show what was actually rejected instead of + // only the generic fallback text the user was shown. Never read by product/UI code and never + // influences routing, scoring, or the delivered answer. + rejectedCandidateText?: string; }; export type ExtractedPage = { diff --git a/tests/eval-quality.test.ts b/tests/eval-quality.test.ts index 820e89567..0a2ccd6e4 100644 --- a/tests/eval-quality.test.ts +++ b/tests/eval-quality.test.ts @@ -16,6 +16,7 @@ import { sourceGovernanceDangerFailuresForAnswer, sourceWarningsForRagQualityAnswer, type RagQualityResult, + qualityThresholds, } from "../scripts/eval-quality"; import { evaluateGoldenRetrievalCase, type GoldenRetrievalResult } from "../scripts/eval-retrieval"; @@ -489,6 +490,10 @@ describe("eval quality reporting", () => { const allowed = buildEvalQualityReport({ generatedAt: "2026-08-22T00:00:00.000Z", retrievalResults: [], + sourceBackedReviewFallbackAllowance: [ + { id: "quality-antipsychotic-metabolic-monitoring", reason: "guidance_wrapper_fragment" }, + { id: "quality-discharge-documentation", reason: "guidance_wrapper_fragment" }, + ], ragResults: [ ragResult({ id: "quality-antipsychotic-metabolic-monitoring", @@ -512,6 +517,10 @@ describe("eval quality reporting", () => { const substituted = buildEvalQualityReport({ generatedAt: "2026-08-22T00:00:00.000Z", retrievalResults: [], + sourceBackedReviewFallbackAllowance: [ + { id: "quality-antipsychotic-metabolic-monitoring", reason: "guidance_wrapper_fragment" }, + { id: "quality-discharge-documentation", reason: "guidance_wrapper_fragment" }, + ], ragResults: [ ragResult({ id: "quality-discharge-documentation", @@ -538,6 +547,10 @@ describe("eval quality reporting", () => { const otherReason = buildEvalQualityReport({ generatedAt: "2026-08-22T00:00:00.000Z", retrievalResults: [], + sourceBackedReviewFallbackAllowance: [ + { id: "quality-antipsychotic-metabolic-monitoring", reason: "guidance_wrapper_fragment" }, + { id: "quality-discharge-documentation", reason: "guidance_wrapper_fragment" }, + ], ragResults: [ ragResult({ id: "quality-antipsychotic-metabolic-monitoring", @@ -1061,3 +1074,34 @@ describe("cross-region retrieval-exhausted carve-out (E-3b)", () => { ).toBe(true); }); }); + +describe("source-backed-review-fallback allowance must name real cases (#NPQJKP)", () => { + // PR #2301 added two allowance entries written as `quality-`, while every id in + // ragEvalCases is bare. `allowed.id === result.id` is an exact match, so neither entry has + // ever excused anything: the waiver reads as active in the source and is inert at runtime, + // and the canary stayed red on cases nobody had accepted. An allowance that cannot match is + // worse than no allowance, because it stops the reader looking further. + it("every allowance id exists in the eval fixture", async () => { + const { ragEvalCases } = await import("../src/lib/rag/rag-eval-cases"); + const known = new Set(ragEvalCases.map((evalCase) => evalCase.id)); + const unmatched = qualityThresholds.ragSourceBackedReviewFallbackAllowance + .map((allowed) => allowed.id) + .filter((id) => !known.has(id)); + expect(unmatched).toEqual([]); + }); + + it("every allowance reason is a reason the pipeline can actually emit", () => { + const emittable = new Set([ + "guidance_wrapper_fragment", + "bare_document_title_list", + "provider_source_gap", + "source_gap", + "empty_after_sanitize", + "generation_quality_failed", + "invalid_model_citation_ids", + ]); + for (const allowed of qualityThresholds.ragSourceBackedReviewFallbackAllowance) { + expect(emittable.has(allowed.reason)).toBe(true); + } + }); +}); diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index aa281f1eb..be87d5194 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -1941,6 +1941,13 @@ describe("RAG structured-output fallback", () => { expect(answer.routingReason).toContain("material_source_governance_gap"); expect(answer.routingReason).toContain("source_backed_review_fallback"); expect(answer.routingReason).toContain("extractive_quality_gate:"); + // The candidate text the quality gate actually judged and rejected on the first pass + // through this branch (route.mode === "extractive") survives in the debug-only field + // scripts/eval-quality.ts reads, even though `answer.answer` here is a second, unrelated + // fallback candidate built fresh by the review-fallback branch in rag.ts. See + // RagAnswer.rejectedCandidateText. + expect(answer.rejectedCandidateText).toBeTruthy(); + expect(answer.rejectedCandidateText).not.toBe(answer.answer); }); it("does not answer FBC withhold-threshold lookups from generic monitoring timing facts", async () => { diff --git a/tests/rag-guidance-wrapper-quality-gate.test.ts b/tests/rag-guidance-wrapper-quality-gate.test.ts index 5ff149f23..ecc2dcc98 100644 --- a/tests/rag-guidance-wrapper-quality-gate.test.ts +++ b/tests/rag-guidance-wrapper-quality-gate.test.ts @@ -202,6 +202,21 @@ describe("#NPQJKP — reachability of the enforcing gate on the grounded extract expect(finalized.answer).not.toContain("compliance, monitoring and evaluation"); }); + it("preserves the rejected candidate text in rejectedCandidateText, distinct from the delivered fallback", () => { + // scripts/eval-quality.ts records this field so a failed canary shows what the gate actually + // read (see its "answerText" field), rather than only the generic evidence-gap wrapper the + // user was shown. Debug-only: it must not change what is returned to the user. + const finalized = finalizeRagAnswerQuality( + capturedGroundedExtractiveAnswer(METABOLIC_ANSWER), + METABOLIC_QUERY, + METABOLIC_CLASS, + ); + + expect(finalized.rejectedCandidateText).toBe(METABOLIC_ANSWER); + expect(finalized.answer).not.toBe(METABOLIC_ANSWER); + expect(finalized.answer).not.toContain("compliance, monitoring and evaluation"); + }); + it("is bypassed only by the preformatted-and-grounded early return", () => { // The one remaining way past the quality gate on this path. Whether the two captured answers // were preformatted cannot be read from here — the dumps are gitignored on the owner's