From c064e052956126eeeb779afb95fbd9662cf0812f Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:53:16 +0800 Subject: [PATCH 1/4] fix(eval): record the answer text the quality gates judge, and delete a waiver that never matched MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Eval Canary has been red since 2026-08-22 — eleven days across four runs — on one step, "Answer-quality subset (live generation)". Golden retrieval passed in every one of those runs; only answer generation blocked. Two defects, both of which made the red uninvestigable rather than merely present. 1. The harness discards the text it judges. The blocking gate is a text-shape predicate: isLaunderedGuidanceWrapperAnswer inspects the answer's opening sentence. Nothing in the uploaded artifact records that sentence, or the answer at all — I downloaded both the failing and the last-good artifact to confirm it. So the only way to see what the gate rejected was to pay for another live run. Record it now: the opening sentence for every case, and the full answer only for cases a text-shape gate rejected, so reports stay small. 2. The allowance has never matched anything. PR #2301 added two entries keyed `quality-` while every id in ragEvalCases is bare, and the comparison is `allowed.id === result.id`. One entry names a case that does not exist in the fixture at all. So the file read as though two degradations were signed off while the gate ran at zero tolerance throughout. Emptying it is a no-op at runtime — proven, since neither entry could match — and stops the source asserting a waiver that is not real. The unit test made the SAME typo, inventing fixtures with the same prefixed ids, so config and test agreed and neither could fail. That is why review did not catch it. The mechanism test now supplies its own allowance through a new optional parameter, so it can no longer be satisfied by whatever the production list happens to contain, and a new test fails if any allowance id names no case in the fixture. Mutation-tested: it goes red on a bad id and green when restored. This deliberately does NOT make the canary green. Three real cases — clozapine-monitoring, clozapine-monitoring-paraphrase and nocc-requirements — are degrading to source-backed review, and whether their answers are genuinely poor or the predicate is over-firing cannot be judged until the next run records the text. Adding them to the allowance is what the threshold's own comment forbids: "Do not add an entry to clear a red run." Co-Authored-By: Claude Opus 5 --- scripts/eval-quality.ts | 64 +++++++++++++++++++++++++++++++++----- tests/eval-quality.test.ts | 44 ++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 8 deletions(-) diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 14ed5f94fd..b030bd003b 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -208,7 +208,7 @@ export function ragAnswerTimingDiagnostics( }; } -const qualityThresholds = { +export const qualityThresholds = { retrievalTopKHitRate: 0.8, retrievalDocumentRecallAt5: 0.8, retrievalContentRecallAt5: 0.8, @@ -234,10 +234,16 @@ const qualityThresholds = { // // Do not add an entry to clear a red run. A block here names the offending case and reason: // investigate that case, exactly as canary 32589154243 did for these two. - ragSourceBackedReviewFallbackAllowance: [ - { id: "quality-antipsychotic-metabolic-monitoring", reason: "guidance_wrapper_fragment" }, - { id: "quality-discharge-documentation", reason: "guidance_wrapper_fragment" }, - ] as ReadonlyArray<{ id: string; reason: string }>, + // + // EMPTIED 2026-09-02, and this is a no-op rather than a policy change. Both entries were + // written as `quality-` while every id in ragEvalCases is bare, and the match is + // `allowed.id === result.id`, so neither has excused a single case since PR #2301 landed. + // The canary has been red continuously since 2026-08-22 on cases nobody accepted, while the + // source read as though two degradations had been signed off. Removing dead entries changes + // no runtime behaviour and stops the file asserting a waiver that does not exist; the + // accompanying test in tests/eval-quality.test.ts fails if an id is ever added that names no + // case. Re-accepting a degradation now requires naming a real case id, with evidence. + ragSourceBackedReviewFallbackAllowance: [] as ReadonlyArray<{ id: string; reason: string }>, numericGroundingFailureRate: 0, staleTopResultRate: 0.25, reviewRequiredTopResultRate: 0.25, @@ -428,6 +434,27 @@ function rate(numerator: number, denominator: number) { return denominator === 0 ? 0 : Number((numerator / denominator).toFixed(4)); } +const TEXT_SHAPE_GATE_REASONS = [ + "guidance_wrapper_fragment", + "bare_document_title_list", + "provider_source_gap", + "empty_after_sanitize", +] as const; + +/** True when a gate that judges the answer PROSE rejected this answer. */ +function textShapeGateRejected(routingReason: string | undefined) { + const reason = (routingReason ?? "").toLowerCase(); + return TEXT_SHAPE_GATE_REASONS.some((token) => reason.includes(token)); +} + +/** The opening sentence, which is the span the wrapper predicate actually inspects. */ +function openingSentenceOf(answer: string | undefined | null) { + const text = (answer ?? "").replace(/\s+/g, " ").trim(); + if (!text) return null; + const stop = text.search(/[.!?](\s|$)/); + return (stop === -1 ? text : text.slice(0, stop + 1)).slice(0, 400); +} + function isSourceBackedReviewFallback(routingReason: string | undefined) { return (routingReason ?? "") .split(";") @@ -568,7 +595,12 @@ function topResultGovernanceCounts(results: GoldenRetrievalResult[]) { }; } -function summarizeRagQualityResults(results: RagQualityResult[], providerMode: EvalQualityProviderMode) { +function summarizeRagQualityResults( + results: RagQualityResult[], + providerMode: EvalQualityProviderMode, + allowanceOverride?: ReadonlyArray<{ id: string; reason: string }>, +) { + const fallbackAllowance = allowanceOverride ?? qualityThresholds.ragSourceBackedReviewFallbackAllowance; const supported = results.filter((result) => result.supported); const unsupported = results.filter((result) => !result.supported); // A supported case counts as grounded-supported when it grounds, OR — for @@ -614,7 +646,7 @@ function summarizeRagQualityResults(results: RagQualityResult[], providerMode: E const sourceBackedReviewFallbackUnaccounted = sourceBackedReviewFallbackResults .filter( (result) => - !qualityThresholds.ragSourceBackedReviewFallbackAllowance.some( + !fallbackAllowance.some( (allowed) => allowed.id === result.id && (result.routingReason ?? "").toLowerCase().includes(allowed.reason), ), ) @@ -678,10 +710,18 @@ export function buildEvalQualityReport(args: { ragResults: RagQualityResult[]; sourceMetadataDebtAcceptance?: SourceMetadataDebtAcceptance; providerMode?: EvalQualityProviderMode; + // Tests supply their own allowance so the mechanism can be exercised without depending on + // whatever the production list happens to contain. The production list and the test fixtures + // previously shared the same typo'd ids, so both agreed and neither could fail. + sourceBackedReviewFallbackAllowance?: ReadonlyArray<{ id: string; reason: string }>; }) { const providerMode = args.providerMode ?? "openai"; const retrievalSummary = summarizeGoldenRetrievalResults(args.retrievalResults); - const ragSummary = summarizeRagQualityResults(args.ragResults, providerMode); + const ragSummary = summarizeRagQualityResults( + args.ragResults, + providerMode, + args.sourceBackedReviewFallbackAllowance, + ); // `--rag-only` intentionally leaves retrieval metrics and gates empty. The // canary can still supply the preceding golden-retrieval artifact so this // report renders its source-governance metadata without rerunning retrieval @@ -1254,6 +1294,14 @@ async function runRagQualityCases(args: { unverifiedNumericTokenCount: answer.unverifiedNumericTokens?.length ?? 0, hasFaithfulnessWarning: Boolean(answer.faithfulnessWarning), routingReason: answer.routingReason, + // The text-shape gates (guidance_wrapper_fragment, bare_document_title_list, + // provider_source_gap) judge the ANSWER PROSE and then the harness discarded it, so a + // blocked canary could not be investigated without paying for another live run. That is + // why #NPQJKP sat red for eleven days. Record what the gate actually read: the opening + // sentence always (it is what isLaunderedGuidanceWrapperAnswer inspects), and the full + // text only for the cases a text-shape gate rejected, so reports stay small. + answerOpeningSentence: openingSentenceOf(answer.answer), + answerText: textShapeGateRejected(answer.routingReason) ? (answer.answer ?? null) : undefined, timings, routeCeilingExceeded, executionType: diff --git a/tests/eval-quality.test.ts b/tests/eval-quality.test.ts index 820e895670..0a2ccd6e44 100644 --- a/tests/eval-quality.test.ts +++ b/tests/eval-quality.test.ts @@ -16,6 +16,7 @@ import { sourceGovernanceDangerFailuresForAnswer, sourceWarningsForRagQualityAnswer, type RagQualityResult, + qualityThresholds, } from "../scripts/eval-quality"; import { evaluateGoldenRetrievalCase, type GoldenRetrievalResult } from "../scripts/eval-retrieval"; @@ -489,6 +490,10 @@ describe("eval quality reporting", () => { const allowed = buildEvalQualityReport({ generatedAt: "2026-08-22T00:00:00.000Z", retrievalResults: [], + sourceBackedReviewFallbackAllowance: [ + { id: "quality-antipsychotic-metabolic-monitoring", reason: "guidance_wrapper_fragment" }, + { id: "quality-discharge-documentation", reason: "guidance_wrapper_fragment" }, + ], ragResults: [ ragResult({ id: "quality-antipsychotic-metabolic-monitoring", @@ -512,6 +517,10 @@ describe("eval quality reporting", () => { const substituted = buildEvalQualityReport({ generatedAt: "2026-08-22T00:00:00.000Z", retrievalResults: [], + sourceBackedReviewFallbackAllowance: [ + { id: "quality-antipsychotic-metabolic-monitoring", reason: "guidance_wrapper_fragment" }, + { id: "quality-discharge-documentation", reason: "guidance_wrapper_fragment" }, + ], ragResults: [ ragResult({ id: "quality-discharge-documentation", @@ -538,6 +547,10 @@ describe("eval quality reporting", () => { const otherReason = buildEvalQualityReport({ generatedAt: "2026-08-22T00:00:00.000Z", retrievalResults: [], + sourceBackedReviewFallbackAllowance: [ + { id: "quality-antipsychotic-metabolic-monitoring", reason: "guidance_wrapper_fragment" }, + { id: "quality-discharge-documentation", reason: "guidance_wrapper_fragment" }, + ], ragResults: [ ragResult({ id: "quality-antipsychotic-metabolic-monitoring", @@ -1061,3 +1074,34 @@ describe("cross-region retrieval-exhausted carve-out (E-3b)", () => { ).toBe(true); }); }); + +describe("source-backed-review-fallback allowance must name real cases (#NPQJKP)", () => { + // PR #2301 added two allowance entries written as `quality-`, while every id in + // ragEvalCases is bare. `allowed.id === result.id` is an exact match, so neither entry has + // ever excused anything: the waiver reads as active in the source and is inert at runtime, + // and the canary stayed red on cases nobody had accepted. An allowance that cannot match is + // worse than no allowance, because it stops the reader looking further. + it("every allowance id exists in the eval fixture", async () => { + const { ragEvalCases } = await import("../src/lib/rag/rag-eval-cases"); + const known = new Set(ragEvalCases.map((evalCase) => evalCase.id)); + const unmatched = qualityThresholds.ragSourceBackedReviewFallbackAllowance + .map((allowed) => allowed.id) + .filter((id) => !known.has(id)); + expect(unmatched).toEqual([]); + }); + + it("every allowance reason is a reason the pipeline can actually emit", () => { + const emittable = new Set([ + "guidance_wrapper_fragment", + "bare_document_title_list", + "provider_source_gap", + "source_gap", + "empty_after_sanitize", + "generation_quality_failed", + "invalid_model_citation_ids", + ]); + for (const allowed of qualityThresholds.ragSourceBackedReviewFallbackAllowance) { + expect(emittable.has(allowed.reason)).toBe(true); + } + }); +}); From 6ec636d626dd53b8e492d165515391466a788fcb Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:00:54 +0800 Subject: [PATCH 2/4] fix(eval): declare the recorded answer fields on RagQualityResult MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The push guard caught this: the result object gained answerOpeningSentence and answerText without the type learning about them, so tsc failed while the unit tests passed. Tests exercise the report builder through its own fixtures and never typecheck the production write path — running them was not sufficient evidence for this change, and the guard was. Co-Authored-By: Claude Opus 5 --- scripts/eval-quality.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index b030bd003b..1fc1b0e626 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -85,6 +85,10 @@ export type RagQualityResult = { unverifiedNumericTokenCount: number; hasFaithfulnessWarning: boolean; routingReason?: string; + /** Opening sentence of the answer — the span the text-shape gates inspect. */ + answerOpeningSentence?: string | null; + /** Full answer text, recorded only when a text-shape gate rejected it. */ + answerText?: string | null; timings?: { retrievalMs: number; routingMs: number; From cc22c14e795d1d51cdd0af82d1dca9a78a6151ff Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 4 Sep 2026 11:46:25 +0000 Subject: [PATCH 3/4] fix(rag): capture pre-fallback rejected candidate text for eval-quality MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eval-quality.ts's answerText field was meant to record the answer prose a text-shape quality gate judged and rejected, but by the time it read answer.answer, finalizeRagAnswerQualityCore had already overwritten it with a generic evidence-gap response — and for the source-backed-review branch in rag.ts, a second overwrite replaced it again with an unrelated fallback wrapper. Neither overwrite left the originally rejected text anywhere the harness could read. Add RagAnswer.rejectedCandidateText, populated at the two points in rag-extractive-answer.ts that discard a rejected candidate's answer text, and threaded through rag.ts's source-backed-review branch so it survives that branch's second candidate substitution. eval-quality.ts now prefers this field over answer.answer when recording what a text-shape gate rejected. Purely additive: no change to routing, scoring, or the answer delivered to users. RAG impact: no retrieval behaviour change — captures the pre-fallback rejected candidate's text into a new debug-only field for the eval harness; does not alter which candidate is chosen, its content, or any scoring/ranking/routing decision. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01ENQDEPFuwDZNoNssvV1PTc --- scripts/eval-quality.ts | 18 ++++++++++++++++-- src/lib/rag/rag-extractive-answer.ts | 2 ++ src/lib/rag/rag.ts | 6 ++++++ src/lib/types.ts | 6 ++++++ tests/rag-answer-fallback.test.ts | 7 +++++++ .../rag-guidance-wrapper-quality-gate.test.ts | 15 +++++++++++++++ 6 files changed, 52 insertions(+), 2 deletions(-) diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 1fc1b0e626..b3e9953037 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -87,7 +87,11 @@ export type RagQualityResult = { routingReason?: string; /** Opening sentence of the answer — the span the text-shape gates inspect. */ answerOpeningSentence?: string | null; - /** Full answer text, recorded only when a text-shape gate rejected it. */ + /** + * Full answer text, recorded only when a text-shape gate rejected it. This is the + * pre-fallback candidate text the gate actually judged (RagAnswer.rejectedCandidateText), + * not the generic fallback text ultimately shown to the user. + */ answerText?: string | null; timings?: { retrievalMs: number; @@ -1304,8 +1308,18 @@ async function runRagQualityCases(args: { // why #NPQJKP sat red for eleven days. Record what the gate actually read: the opening // sentence always (it is what isLaunderedGuidanceWrapperAnswer inspects), and the full // text only for the cases a text-shape gate rejected, so reports stay small. + // + // The rejected candidate's text is not `answer.answer` by the time we get here — for a + // gate rejection, `finalizeRagAnswerQualityCore` has already replaced `answer.answer` + // with a generic evidence-gap response, and for a source-backed-review rejection, rag.ts + // has additionally replaced it again with a different generic fallback answer. Prefer + // `answer.rejectedCandidateText`, the pre-replacement text the gate actually judged + // (see RagAnswer.rejectedCandidateText); fall back to `answer.answer` only for the rare + // case a text-shape reason fired without that field being populated. answerOpeningSentence: openingSentenceOf(answer.answer), - answerText: textShapeGateRejected(answer.routingReason) ? (answer.answer ?? null) : undefined, + answerText: textShapeGateRejected(answer.routingReason) + ? (answer.rejectedCandidateText ?? answer.answer ?? null) + : undefined, timings, routeCeilingExceeded, executionType: diff --git a/src/lib/rag/rag-extractive-answer.ts b/src/lib/rag/rag-extractive-answer.ts index 6281869bb6..b134665368 100644 --- a/src/lib/rag/rag-extractive-answer.ts +++ b/src/lib/rag/rag-extractive-answer.ts @@ -3521,6 +3521,7 @@ function finalQualityFailure(answer: RagAnswer, query: string, queryClass: RagQu return { ...answer, answer: finalQualityGapAnswer(query, queryClass), + rejectedCandidateText: answer.answer, grounded: false, confidence: "unsupported", answerSections: [], @@ -3972,6 +3973,7 @@ function finalizeRagAnswerQualityCore(answer: RagAnswer, query: string, queryCla return { ...answer, answer: gapAnswer, + rejectedCandidateText: answer.answer, grounded: false, confidence: "unsupported", citations: [], diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 94634b4b7c..e00fab3f3f 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -3116,6 +3116,11 @@ async function answerQuestionWithScopeUncoalesced( "ungrounded_extractive_answer"; const reviewRouteReason = `${finalizedAnswer.routingReason ?? answer.routingReason ?? route.reason}; ${SOURCE_BACKED_REVIEW_FALLBACK_REASON}; extractive_quality_gate:${extractiveQualityReason}`; const reviewPlan = buildCurrentSmartApiPlan("extractive", reviewRouteReason); + // The candidate text the quality gate actually judged and rejected — captured before + // finalizeAnswer below builds a fresh fallback candidate from + // sourceBackedGenerationTimeoutAnswer() and overwrites `finalizedAnswer`. Debug-only; see + // RagAnswer.rejectedCandidateText. + const priorRejectedCandidateText = finalizedAnswer.rejectedCandidateText ?? finalizedAnswer.answer; finalizedAnswer = finalizeAnswer({ ...answer, answer: boldHighYieldClinicalText(sourceBackedGenerationTimeoutAnswer(args.query), args.query), @@ -3129,6 +3134,7 @@ async function answerQuestionWithScopeUncoalesced( smartApiPlan: reviewPlan, answerSections: [], }); + finalizedAnswer.rejectedCandidateText ??= priorRejectedCandidateText; } if (args.logQuery !== false) diff --git a/src/lib/types.ts b/src/lib/types.ts index 9b48543742..10dbbd767f 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -1106,6 +1106,12 @@ export type RagAnswer = { // When non-empty the answer should be treated as needing source verification. unverifiedNumericTokens?: string[]; faithfulnessWarning?: string; + // Debug-only: when a final quality gate rejects a candidate's answer prose and replaces + // `answer` with an evidence-gap response, this preserves the pre-replacement candidate text + // so eval tooling (scripts/eval-quality.ts) can show what was actually rejected instead of + // only the generic fallback text the user was shown. Never read by product/UI code and never + // influences routing, scoring, or the delivered answer. + rejectedCandidateText?: string; }; export type ExtractedPage = { diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index aa281f1eb6..be87d51943 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -1941,6 +1941,13 @@ describe("RAG structured-output fallback", () => { expect(answer.routingReason).toContain("material_source_governance_gap"); expect(answer.routingReason).toContain("source_backed_review_fallback"); expect(answer.routingReason).toContain("extractive_quality_gate:"); + // The candidate text the quality gate actually judged and rejected on the first pass + // through this branch (route.mode === "extractive") survives in the debug-only field + // scripts/eval-quality.ts reads, even though `answer.answer` here is a second, unrelated + // fallback candidate built fresh by the review-fallback branch in rag.ts. See + // RagAnswer.rejectedCandidateText. + expect(answer.rejectedCandidateText).toBeTruthy(); + expect(answer.rejectedCandidateText).not.toBe(answer.answer); }); it("does not answer FBC withhold-threshold lookups from generic monitoring timing facts", async () => { diff --git a/tests/rag-guidance-wrapper-quality-gate.test.ts b/tests/rag-guidance-wrapper-quality-gate.test.ts index 5ff149f23f..ecc2dcc980 100644 --- a/tests/rag-guidance-wrapper-quality-gate.test.ts +++ b/tests/rag-guidance-wrapper-quality-gate.test.ts @@ -202,6 +202,21 @@ describe("#NPQJKP — reachability of the enforcing gate on the grounded extract expect(finalized.answer).not.toContain("compliance, monitoring and evaluation"); }); + it("preserves the rejected candidate text in rejectedCandidateText, distinct from the delivered fallback", () => { + // scripts/eval-quality.ts records this field so a failed canary shows what the gate actually + // read (see its "answerText" field), rather than only the generic evidence-gap wrapper the + // user was shown. Debug-only: it must not change what is returned to the user. + const finalized = finalizeRagAnswerQuality( + capturedGroundedExtractiveAnswer(METABOLIC_ANSWER), + METABOLIC_QUERY, + METABOLIC_CLASS, + ); + + expect(finalized.rejectedCandidateText).toBe(METABOLIC_ANSWER); + expect(finalized.answer).not.toBe(METABOLIC_ANSWER); + expect(finalized.answer).not.toContain("compliance, monitoring and evaluation"); + }); + it("is bypassed only by the preformatted-and-grounded early return", () => { // The one remaining way past the quality gate on this path. Whether the two captured answers // were preformatted cannot be read from here — the dumps are gitignored on the owner's From 236bac0351703130a3238477bb292f21af558680 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 4 Sep 2026 12:19:49 +0000 Subject: [PATCH 4/4] refactor(rag): extract classifier fallback out of rag.ts to fix maintainability budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit's rejectedCandidateText debug field pushed rag.ts to 4368 lines, 6 over the 4362-line no-growth budget enforced by check-maintainability-budgets.mjs. Tightening the new code alone (shortening its comment) could not close the gap without shrinking below the minimum Prettier-formatted footprint of the capture/restore statements it needs. Mechanically move the self-contained LLM query-classifier fallback cluster (schemas, memo cache, analyzeQueryWithClassifierFallback, and the shared uniqueTextValues helper) into a new src/lib/rag/rag-classifier-fallback.ts, matching the existing pattern of rag.ts delegating to sibling rag-*.ts modules. Also extract the small generic awaitWithCallerSignal helper into src/lib/rag/rag-abort-signal.ts so both rag.ts and the new module can import it without a circular dependency. rag.ts re-exports analyzeQueryWithClassifierFallback and resetClassifierVerdictMemoForTests so existing test imports (rag.analyzeQueryWithClassifierFallback, etc.) are unaffected. No behavior change: this is a pure code move plus import/export bookkeeping. rag.ts drops from 4368 to 4075 lines, well under the 4362 ceiling. RAG impact: no retrieval behaviour change — mechanical extraction of the classifier-fallback cluster into its own module; the code and its call sites are unchanged, only their file location and import paths move. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01ENQDEPFuwDZNoNssvV1PTc --- src/lib/rag/rag-abort-signal.ts | 19 ++ src/lib/rag/rag-classifier-fallback.ts | 284 +++++++++++++++++++++++ src/lib/rag/rag.ts | 307 +------------------------ 3 files changed, 310 insertions(+), 300 deletions(-) create mode 100644 src/lib/rag/rag-abort-signal.ts create mode 100644 src/lib/rag/rag-classifier-fallback.ts diff --git a/src/lib/rag/rag-abort-signal.ts b/src/lib/rag/rag-abort-signal.ts new file mode 100644 index 0000000000..5d2255f9b1 --- /dev/null +++ b/src/lib/rag/rag-abort-signal.ts @@ -0,0 +1,19 @@ +export function awaitWithCallerSignal(pending: Promise, signal?: AbortSignal): Promise { + if (!signal) return pending; + if (signal.aborted) throw signal.reason ?? new DOMException("The operation was aborted.", "AbortError"); + + return new Promise((resolve, reject) => { + const onAbort = () => reject(signal.reason ?? new DOMException("The operation was aborted.", "AbortError")); + signal.addEventListener("abort", onAbort, { once: true }); + pending.then( + (value) => { + signal.removeEventListener("abort", onAbort); + resolve(value); + }, + (error) => { + signal.removeEventListener("abort", onAbort); + reject(error); + }, + ); + }); +} diff --git a/src/lib/rag/rag-classifier-fallback.ts b/src/lib/rag/rag-classifier-fallback.ts new file mode 100644 index 0000000000..37246e6009 --- /dev/null +++ b/src/lib/rag/rag-classifier-fallback.ts @@ -0,0 +1,284 @@ +import { createAdminClient } from "@/lib/supabase/admin"; +import { classifyCorpusGrounding } from "@/lib/corpus-grounding"; +import { generateParsedTextResult, openAISafetyIdentifier } from "@/lib/openai"; +import { env } from "@/lib/env"; +import { ragQueryClassifierPromptVersion } from "@/lib/rag/rag-versioning"; +import { hasAdversarialManipulationIntent } from "@/lib/rag/rag-routing"; +import { + clearlyOutsideCorpusMedicalPattern, + isUnsupportedSoftTailAnalysis, + unavailableDocumentNoisePattern, +} from "@/lib/rag/rag-query-guard"; +import { awaitWithCallerSignal } from "@/lib/rag/rag-abort-signal"; +import type { ClinicalQueryAnalysis } from "@/lib/types"; +import { z } from "zod"; + +const queryClassifierParseSchema = z + .object({ + queryClass: z.enum([ + "document_lookup", + "table_threshold", + "medication_dose_risk", + "comparison", + "broad_summary", + "unsupported_or_general", + ]), + confidence: z.number(), + reasons: z.array(z.string()), + expandedTerms: z.array(z.string()), + }) + .strict(); + +const queryClassifierVerdictSchema = queryClassifierParseSchema.extend({ + confidence: z.number().min(0).max(1), + reasons: z.array(z.string().max(80)).max(4), + expandedTerms: z.array(z.string().max(60)).max(10), +}); + +/** Unique text values. */ +export function uniqueTextValues(values: Array, limit = 32) { + const seen = new Set(); + const output: string[] = []; + for (const value of values) { + const normalized = value?.replace(/\s+/g, " ").trim(); + if (!normalized) continue; + const key = normalized.toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + output.push(normalized); + if (output.length >= limit) break; + } + return output; +} + +type ClassifierVerdict = z.infer; + +// Finding #11 interim fix (docs/process-hardening.md): the LLM classifier verdict flips +// run-to-run for the same query, so the unsupported short-circuit downstream intermittently +// returned 0 results for valid in-corpus topics. Memoizing successful verdicts makes the +// verdict — and therefore retrieval behaviour — deterministic per query for the TTL window. +// Only *successful* classifier calls are memoized (accepted and rejected verdicts alike); +// transport errors and timeouts stay retryable, otherwise one transient 6s timeout would pin +// a query's classification for the whole TTL. The full corpus-grounded relevance fix remains +// scoped to RAG optimisation Phase 2. +const classifierVerdictMemoTtlMs = 15 * 60 * 1000; +// Finding #11 follow-up: bounds retries for a rejected soft-tail verdict (isUnsupportedSoftTailAnalysis). +const rejectedSoftTailMemoTtlMs = 60 * 1000; +const classifierVerdictMemoMaxEntries = 500; +const classifierVerdictMemo = new Map(); +const classifierVerdictInflight = new Map>(); + +/** Classifier verdict memo key. */ +function classifierVerdictMemoKey(query: string, analysis: ClinicalQueryAnalysis) { + const normalizedQuery = query.normalize("NFKC").toLowerCase().replace(/\s+/g, " ").trim(); + // The deterministic class + confidence bucket are part of the key so a deterministic-analyzer + // change invalidates stale verdicts instead of replaying them against a different baseline. + return [ + env.OPENAI_QUERY_CLASSIFIER_MODEL, + ragQueryClassifierPromptVersion, + normalizedQuery, + analysis.queryClass, + analysis.confidence.toFixed(2), + ].join("::"); +} + +/** Store classifier verdict memo. */ +function storeClassifierVerdictMemo(key: string, verdict: ClassifierVerdict, ttlMs = classifierVerdictMemoTtlMs) { + if (classifierVerdictMemo.size >= classifierVerdictMemoMaxEntries) { + const oldestKey = classifierVerdictMemo.keys().next().value; + if (oldestKey !== undefined) classifierVerdictMemo.delete(oldestKey); + } + classifierVerdictMemo.set(key, { expiresAt: Date.now() + ttlMs, verdict }); +} + +/** Reset classifier verdict memo for tests. */ +export function resetClassifierVerdictMemoForTests() { + classifierVerdictMemo.clear(); + classifierVerdictInflight.clear(); +} + +/** Request classifier verdict. */ +async function requestClassifierVerdict( + query: string, + analysis: ClinicalQueryAnalysis, + ownerId?: string | null, +): Promise { + const result = await generateParsedTextResult( + [ + { + role: "user", + content: [ + { + type: "input_text", + text: [ + `Query: ${query}`, + `Deterministic query class: ${analysis.queryClass}`, + `Deterministic confidence: ${analysis.confidence}`, + `Known expanded terms: ${analysis.expandedTerms.join(", ") || "none"}`, + ].join("\n"), + }, + ], + }, + ], + queryClassifierParseSchema, + { + model: env.OPENAI_QUERY_CLASSIFIER_MODEL, + maxOutputTokens: 220, + operation: "text_generation", + instructions: + "Classify this query for retrieval routing only. Do not answer the clinical question. Prefer unsupported when the query is not about indexed clinical document retrieval.", + reasoningEffort: "low", + textVerbosity: "low", + schemaName: "clinical_rag_query_classifier", + promptCacheKey: ragQueryClassifierPromptVersion, + timeoutMs: 6000, + safetyIdentifier: env.OPENAI_SAFETY_IDENTIFIER_SECRET ? openAISafetyIdentifier(ownerId) : undefined, + }, + ); + return queryClassifierVerdictSchema.parse(result.parsed); +} + +/** Apply classifier verdict. */ +function applyClassifierVerdict(analysis: ClinicalQueryAnalysis, parsed: ClassifierVerdict): ClinicalQueryAnalysis { + if (parsed.confidence < 0.58 || parsed.queryClass === "unsupported_or_general") return analysis; + return { + ...analysis, + queryClass: parsed.queryClass, + confidence: Math.max(analysis.confidence, parsed.confidence), + needsClassifierFallback: false, + needsSynthesis: + analysis.needsSynthesis || + parsed.queryClass === "comparison" || + parsed.queryClass === "broad_summary" || + parsed.queryClass === "medication_dose_risk", + expandedTerms: uniqueTextValues([...analysis.expandedTerms, ...parsed.expandedTerms], 36), + queryRewrite: { + ...analysis.queryRewrite, + expansions: uniqueTextValues([...analysis.queryRewrite.expansions, ...parsed.expandedTerms], 48), + searchQuery: uniqueTextValues( + [analysis.queryRewrite.searchQuery, ...analysis.queryRewrite.expansions, ...parsed.expandedTerms], + 60, + ).join(" "), + reasons: uniqueTextValues([...analysis.queryRewrite.reasons, ...parsed.reasons, "classifier_fallback"], 16), + }, + reasons: uniqueTextValues([...analysis.reasons, ...parsed.reasons, "classifier_fallback"], 12), + } satisfies ClinicalQueryAnalysis; +} + +/** Analyze query with classifier fallback. */ +export async function analyzeQueryWithClassifierFallback( + query: string, + analysis: ClinicalQueryAnalysis, + opts?: { + // Finding #11 corpus grounding: when provided, unsupported-soft-tail queries are checked + // against the corpus BEFORE the nondeterministic LLM classifier. Scoped with the exact + // owner_filter retrieval will use so grounding can never see documents retrieval cannot. + corpusGrounding?: { supabase: ReturnType; ownerFilter: string | null }; + ownerId?: string | null; + signal?: AbortSignal; + }, +) { + if ( + // Fail closed before any generative model call: an adversarial-manipulation + // query is routed to "unsupported" downstream, so never send its text to the + // LLM query classifier. (Embedding-based retrieval is non-generative and not + // an injection surface.) + hasAdversarialManipulationIntent(query) || + unavailableDocumentNoisePattern.test(query) || + (clearlyOutsideCorpusMedicalPattern.test(query) && analysis.documentTitleTerms.length === 0) + ) { + return { ...analysis, needsClassifierFallback: false } satisfies ClinicalQueryAnalysis; + } + + // Finding #11 corpus-grounded relevance: for queries that would hit the unsupported soft + // tail, the corpus — not the LLM — decides. An in-corpus bare topic ("bipolar disorder") + // deterministically reclassifies to broad_summary (mirroring what an accepted classifier + // verdict would have done, minus the coin flip); a corpus-absent query ("florbizone syndrome + // management") skips the LLM entirely so the soft-tail refusal is deterministic — and typos + // remain rescuable because the short-circuit path still runs trigram correction afterwards. + // "inconclusive" (including DB errors and an unapplied migration) keeps legacy behaviour. + // This deliberately runs before the OPENAI_API_KEY gate: offline/source-only deployments + // still retrieve lexically, so in-corpus bare topics should answer there too. + if (opts?.corpusGrounding && isUnsupportedSoftTailAnalysis(query, analysis)) { + const grounding = await classifyCorpusGrounding({ + supabase: opts.corpusGrounding.supabase, + query, + ownerFilter: opts.corpusGrounding.ownerFilter, + }); + if (grounding.verdict === "in_corpus_topic") { + return { + ...analysis, + queryClass: "broad_summary", + confidence: Math.max(analysis.confidence, 0.62), + needsSynthesis: true, + needsClassifierFallback: false, + corpusGrounding: "in_corpus_topic", + reasons: uniqueTextValues([...analysis.reasons, "corpus_topic_grounding"], 12), + } satisfies ClinicalQueryAnalysis; + } + if (grounding.verdict === "out_of_corpus") { + // Do NOT touch queryClass/confidence/reasons: the existing soft-tail short-circuit (and + // its alias-expansion + trigram-correction escape hatches) must keep firing exactly as + // before — only the LLM lottery is removed. + return { + ...analysis, + needsClassifierFallback: false, + corpusGrounding: "out_of_corpus", + } satisfies ClinicalQueryAnalysis; + } + analysis = { ...analysis, corpusGrounding: "inconclusive" }; + } + + // Finding #2: Deterministic fallback routing for short clinical queries. + // Short, bare clinical search queries (e.g., "bipolar disorder", "anorexia management") + // can be misclassified by the generative LLM. We route them deterministically. + if ( + analysis.needsClassifierFallback && + analysis.corpusGrounding !== "inconclusive" && + query.trim().split(/\s+/).length <= 4 && + (analysis.documentTitleTerms.length > 0 || analysis.canonicalTerms.length > 0) + ) { + return { + ...analysis, + queryClass: "broad_summary", + needsClassifierFallback: false, + reasons: uniqueTextValues([...analysis.reasons, "deterministic_short_clinical_query_fallback"], 12), + } satisfies ClinicalQueryAnalysis; + } + + if (!analysis.needsClassifierFallback || !env.OPENAI_API_KEY) return analysis; + + const memoKey = classifierVerdictMemoKey(query, analysis); + const memoized = classifierVerdictMemo.get(memoKey); + if (memoized) { + if (memoized.expiresAt > Date.now()) return applyClassifierVerdict(analysis, memoized.verdict); + classifierVerdictMemo.delete(memoKey); + } + + let pending = classifierVerdictInflight.get(memoKey); + if (!pending) { + pending = requestClassifierVerdict(query, analysis, opts?.ownerId).finally(() => { + classifierVerdictInflight.delete(memoKey); + }); + classifierVerdictInflight.set(memoKey, pending); + } + + try { + const verdict = await awaitWithCallerSignal(pending, opts?.signal); + // Finding #11 follow-up: bounded TTL for a rejected soft-tail verdict — see the constant above. + const rejected = verdict.confidence < 0.58 || verdict.queryClass === "unsupported_or_general"; + const softTail = rejected && isUnsupportedSoftTailAnalysis(query, analysis); + storeClassifierVerdictMemo(memoKey, verdict, softTail ? rejectedSoftTailMemoTtlMs : undefined); + return applyClassifierVerdict(analysis, verdict); + } catch (error) { + if ( + error && + (error instanceof DOMException || typeof error === "object") && + (error as { name?: string }).name === "AbortError" + ) + throw error; + // Transport/parse failures are deliberately NOT memoized: fall back to the deterministic + // analysis for this request only, and let the next request retry the classifier. + return analysis; + } +} diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index e00fab3f3f..d103900fae 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -24,11 +24,9 @@ export { loadChunksForMemoryCards, loadChunksForSignalMatches, } from "@/lib/rag/rag-candidate-sources"; -import { classifyCorpusGrounding } from "@/lib/corpus-grounding"; import type { Database, Json } from "@/lib/supabase/database.types"; import { embedTextWithTelemetry, - generateParsedTextResult, generateStructuredTextResult, openAISafetyIdentifier, type OpenAITextResult, @@ -165,11 +163,7 @@ import { rankClinicalResults, } from "@/lib/clinical-search"; import { env, requestedOpenAIAnswerModels } from "@/lib/env"; -import { - ragAnswerPromptVersion, - ragQueryClassifierPromptVersion, - ragSummaryPromptVersion, -} from "@/lib/rag/rag-versioning"; +import { ragAnswerPromptVersion, ragSummaryPromptVersion } from "@/lib/rag/rag-versioning"; import { answerPrivacyMetadata, answerTextForStorage, @@ -219,13 +213,16 @@ export { import { retrievalPlanForQueryClass, type SearchChunksArgs, type SearchTelemetry } from "@/lib/rag/rag-contracts"; export { retrievalPlanForQueryClass, type SearchChunksArgs, type SearchTelemetry } from "@/lib/rag/rag-contracts"; import { - clearlyOutsideCorpusMedicalPattern, - isUnsupportedSoftTailAnalysis, shouldSkipUnsupportedSoftTailAnswerCacheWrite, shouldSkipUnsupportedSoftTailCacheWrite, - unavailableDocumentNoisePattern, } from "@/lib/rag/rag-query-guard"; export { shouldShortCircuitUnsupportedSearch } from "@/lib/rag/rag-query-guard"; +import { analyzeQueryWithClassifierFallback, uniqueTextValues } from "@/lib/rag/rag-classifier-fallback"; +export { + analyzeQueryWithClassifierFallback, + resetClassifierVerdictMemoForTests, +} from "@/lib/rag/rag-classifier-fallback"; +import { awaitWithCallerSignal } from "@/lib/rag/rag-abort-signal"; import { hasAdmissionCommunityLookupIntent, hasAdmissionCommunityTitleSupport, @@ -472,26 +469,6 @@ function throwIfAborted(signal?: AbortSignal) { } } -function awaitWithCallerSignal(pending: Promise, signal?: AbortSignal): Promise { - if (!signal) return pending; - if (signal.aborted) throw signal.reason ?? new DOMException("The operation was aborted.", "AbortError"); - - return new Promise((resolve, reject) => { - const onAbort = () => reject(signal.reason ?? new DOMException("The operation was aborted.", "AbortError")); - signal.addEventListener("abort", onAbort, { once: true }); - pending.then( - (value) => { - signal.removeEventListener("abort", onAbort); - resolve(value); - }, - (error) => { - signal.removeEventListener("abort", onAbort); - reject(error); - }, - ); - }); -} - export type AnswerProgressEvent = { stage: | "retrieved" @@ -922,276 +899,6 @@ function hasOpenAIUsage(usage: OpenAITokenUsage) { return Object.values(usage).some((value) => typeof value === "number" && value > 0); } -const queryClassifierParseSchema = z - .object({ - queryClass: z.enum([ - "document_lookup", - "table_threshold", - "medication_dose_risk", - "comparison", - "broad_summary", - "unsupported_or_general", - ]), - confidence: z.number(), - reasons: z.array(z.string()), - expandedTerms: z.array(z.string()), - }) - .strict(); - -const queryClassifierVerdictSchema = queryClassifierParseSchema.extend({ - confidence: z.number().min(0).max(1), - reasons: z.array(z.string().max(80)).max(4), - expandedTerms: z.array(z.string().max(60)).max(10), -}); - -/** Unique text values. */ -function uniqueTextValues(values: Array, limit = 32) { - const seen = new Set(); - const output: string[] = []; - for (const value of values) { - const normalized = value?.replace(/\s+/g, " ").trim(); - if (!normalized) continue; - const key = normalized.toLowerCase(); - if (seen.has(key)) continue; - seen.add(key); - output.push(normalized); - if (output.length >= limit) break; - } - return output; -} - -type ClassifierVerdict = z.infer; - -// Finding #11 interim fix (docs/process-hardening.md): the LLM classifier verdict flips -// run-to-run for the same query, so the unsupported short-circuit downstream intermittently -// returned 0 results for valid in-corpus topics. Memoizing successful verdicts makes the -// verdict — and therefore retrieval behaviour — deterministic per query for the TTL window. -// Only *successful* classifier calls are memoized (accepted and rejected verdicts alike); -// transport errors and timeouts stay retryable, otherwise one transient 6s timeout would pin -// a query's classification for the whole TTL. The full corpus-grounded relevance fix remains -// scoped to RAG optimisation Phase 2. -const classifierVerdictMemoTtlMs = 15 * 60 * 1000; -// Finding #11 follow-up: bounds retries for a rejected soft-tail verdict (isUnsupportedSoftTailAnalysis). -const rejectedSoftTailMemoTtlMs = 60 * 1000; -const classifierVerdictMemoMaxEntries = 500; -const classifierVerdictMemo = new Map(); -const classifierVerdictInflight = new Map>(); - -/** Classifier verdict memo key. */ -function classifierVerdictMemoKey(query: string, analysis: ClinicalQueryAnalysis) { - const normalizedQuery = query.normalize("NFKC").toLowerCase().replace(/\s+/g, " ").trim(); - // The deterministic class + confidence bucket are part of the key so a deterministic-analyzer - // change invalidates stale verdicts instead of replaying them against a different baseline. - return [ - env.OPENAI_QUERY_CLASSIFIER_MODEL, - ragQueryClassifierPromptVersion, - normalizedQuery, - analysis.queryClass, - analysis.confidence.toFixed(2), - ].join("::"); -} - -/** Store classifier verdict memo. */ -function storeClassifierVerdictMemo(key: string, verdict: ClassifierVerdict, ttlMs = classifierVerdictMemoTtlMs) { - if (classifierVerdictMemo.size >= classifierVerdictMemoMaxEntries) { - const oldestKey = classifierVerdictMemo.keys().next().value; - if (oldestKey !== undefined) classifierVerdictMemo.delete(oldestKey); - } - classifierVerdictMemo.set(key, { expiresAt: Date.now() + ttlMs, verdict }); -} - -/** Reset classifier verdict memo for tests. */ -export function resetClassifierVerdictMemoForTests() { - classifierVerdictMemo.clear(); - classifierVerdictInflight.clear(); -} - -/** Request classifier verdict. */ -async function requestClassifierVerdict( - query: string, - analysis: ClinicalQueryAnalysis, - ownerId?: string | null, -): Promise { - const result = await generateParsedTextResult( - [ - { - role: "user", - content: [ - { - type: "input_text", - text: [ - `Query: ${query}`, - `Deterministic query class: ${analysis.queryClass}`, - `Deterministic confidence: ${analysis.confidence}`, - `Known expanded terms: ${analysis.expandedTerms.join(", ") || "none"}`, - ].join("\n"), - }, - ], - }, - ], - queryClassifierParseSchema, - { - model: env.OPENAI_QUERY_CLASSIFIER_MODEL, - maxOutputTokens: 220, - operation: "text_generation", - instructions: - "Classify this query for retrieval routing only. Do not answer the clinical question. Prefer unsupported when the query is not about indexed clinical document retrieval.", - reasoningEffort: "low", - textVerbosity: "low", - schemaName: "clinical_rag_query_classifier", - promptCacheKey: ragQueryClassifierPromptVersion, - timeoutMs: 6000, - safetyIdentifier: env.OPENAI_SAFETY_IDENTIFIER_SECRET ? openAISafetyIdentifier(ownerId) : undefined, - }, - ); - return queryClassifierVerdictSchema.parse(result.parsed); -} - -/** Apply classifier verdict. */ -function applyClassifierVerdict(analysis: ClinicalQueryAnalysis, parsed: ClassifierVerdict): ClinicalQueryAnalysis { - if (parsed.confidence < 0.58 || parsed.queryClass === "unsupported_or_general") return analysis; - return { - ...analysis, - queryClass: parsed.queryClass, - confidence: Math.max(analysis.confidence, parsed.confidence), - needsClassifierFallback: false, - needsSynthesis: - analysis.needsSynthesis || - parsed.queryClass === "comparison" || - parsed.queryClass === "broad_summary" || - parsed.queryClass === "medication_dose_risk", - expandedTerms: uniqueTextValues([...analysis.expandedTerms, ...parsed.expandedTerms], 36), - queryRewrite: { - ...analysis.queryRewrite, - expansions: uniqueTextValues([...analysis.queryRewrite.expansions, ...parsed.expandedTerms], 48), - searchQuery: uniqueTextValues( - [analysis.queryRewrite.searchQuery, ...analysis.queryRewrite.expansions, ...parsed.expandedTerms], - 60, - ).join(" "), - reasons: uniqueTextValues([...analysis.queryRewrite.reasons, ...parsed.reasons, "classifier_fallback"], 16), - }, - reasons: uniqueTextValues([...analysis.reasons, ...parsed.reasons, "classifier_fallback"], 12), - } satisfies ClinicalQueryAnalysis; -} - -/** Analyze query with classifier fallback. */ -export async function analyzeQueryWithClassifierFallback( - query: string, - analysis: ClinicalQueryAnalysis, - opts?: { - // Finding #11 corpus grounding: when provided, unsupported-soft-tail queries are checked - // against the corpus BEFORE the nondeterministic LLM classifier. Scoped with the exact - // owner_filter retrieval will use so grounding can never see documents retrieval cannot. - corpusGrounding?: { supabase: ReturnType; ownerFilter: string | null }; - ownerId?: string | null; - signal?: AbortSignal; - }, -) { - if ( - // Fail closed before any generative model call: an adversarial-manipulation - // query is routed to "unsupported" downstream, so never send its text to the - // LLM query classifier. (Embedding-based retrieval is non-generative and not - // an injection surface.) - hasAdversarialManipulationIntent(query) || - unavailableDocumentNoisePattern.test(query) || - (clearlyOutsideCorpusMedicalPattern.test(query) && analysis.documentTitleTerms.length === 0) - ) { - return { ...analysis, needsClassifierFallback: false } satisfies ClinicalQueryAnalysis; - } - - // Finding #11 corpus-grounded relevance: for queries that would hit the unsupported soft - // tail, the corpus — not the LLM — decides. An in-corpus bare topic ("bipolar disorder") - // deterministically reclassifies to broad_summary (mirroring what an accepted classifier - // verdict would have done, minus the coin flip); a corpus-absent query ("florbizone syndrome - // management") skips the LLM entirely so the soft-tail refusal is deterministic — and typos - // remain rescuable because the short-circuit path still runs trigram correction afterwards. - // "inconclusive" (including DB errors and an unapplied migration) keeps legacy behaviour. - // This deliberately runs before the OPENAI_API_KEY gate: offline/source-only deployments - // still retrieve lexically, so in-corpus bare topics should answer there too. - if (opts?.corpusGrounding && isUnsupportedSoftTailAnalysis(query, analysis)) { - const grounding = await classifyCorpusGrounding({ - supabase: opts.corpusGrounding.supabase, - query, - ownerFilter: opts.corpusGrounding.ownerFilter, - }); - if (grounding.verdict === "in_corpus_topic") { - return { - ...analysis, - queryClass: "broad_summary", - confidence: Math.max(analysis.confidence, 0.62), - needsSynthesis: true, - needsClassifierFallback: false, - corpusGrounding: "in_corpus_topic", - reasons: uniqueTextValues([...analysis.reasons, "corpus_topic_grounding"], 12), - } satisfies ClinicalQueryAnalysis; - } - if (grounding.verdict === "out_of_corpus") { - // Do NOT touch queryClass/confidence/reasons: the existing soft-tail short-circuit (and - // its alias-expansion + trigram-correction escape hatches) must keep firing exactly as - // before — only the LLM lottery is removed. - return { - ...analysis, - needsClassifierFallback: false, - corpusGrounding: "out_of_corpus", - } satisfies ClinicalQueryAnalysis; - } - analysis = { ...analysis, corpusGrounding: "inconclusive" }; - } - - // Finding #2: Deterministic fallback routing for short clinical queries. - // Short, bare clinical search queries (e.g., "bipolar disorder", "anorexia management") - // can be misclassified by the generative LLM. We route them deterministically. - if ( - analysis.needsClassifierFallback && - analysis.corpusGrounding !== "inconclusive" && - query.trim().split(/\s+/).length <= 4 && - (analysis.documentTitleTerms.length > 0 || analysis.canonicalTerms.length > 0) - ) { - return { - ...analysis, - queryClass: "broad_summary", - needsClassifierFallback: false, - reasons: uniqueTextValues([...analysis.reasons, "deterministic_short_clinical_query_fallback"], 12), - } satisfies ClinicalQueryAnalysis; - } - - if (!analysis.needsClassifierFallback || !env.OPENAI_API_KEY) return analysis; - - const memoKey = classifierVerdictMemoKey(query, analysis); - const memoized = classifierVerdictMemo.get(memoKey); - if (memoized) { - if (memoized.expiresAt > Date.now()) return applyClassifierVerdict(analysis, memoized.verdict); - classifierVerdictMemo.delete(memoKey); - } - - let pending = classifierVerdictInflight.get(memoKey); - if (!pending) { - pending = requestClassifierVerdict(query, analysis, opts?.ownerId).finally(() => { - classifierVerdictInflight.delete(memoKey); - }); - classifierVerdictInflight.set(memoKey, pending); - } - - try { - const verdict = await awaitWithCallerSignal(pending, opts?.signal); - // Finding #11 follow-up: bounded TTL for a rejected soft-tail verdict — see the constant above. - const rejected = verdict.confidence < 0.58 || verdict.queryClass === "unsupported_or_general"; - const softTail = rejected && isUnsupportedSoftTailAnalysis(query, analysis); - storeClassifierVerdictMemo(memoKey, verdict, softTail ? rejectedSoftTailMemoTtlMs : undefined); - return applyClassifierVerdict(analysis, verdict); - } catch (error) { - if ( - error && - (error instanceof DOMException || typeof error === "object") && - (error as { name?: string }).name === "AbortError" - ) - throw error; - // Transport/parse failures are deliberately NOT memoized: fall back to the deterministic - // analysis for this request only, and let the next request retry the classifier. - return analysis; - } -} - /** Metadata expansion term score. */ function metadataExpansionTermScore(queryTokens: Set, value: string, sourceWeight: number) { const tokens = normalizedClinicalSearchTokens(value);