From 5bb99c07e2def50cd51be0750a3d69182abda292 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 14:31:55 +0000 Subject: [PATCH] refactor(rag): extract per-request hydration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the per-request hydration cluster out of the rag.ts monolith into src/lib/rag/rag-hydration.ts: DocumentRankingMetadataCache, createDocumentRankingMetadataCache, attachDocumentRankingMetadata, withCachedIndexQuality, attachIndexQualityMetadata, attachPageVisualEvidence. All six move byte-for-byte; the only edit is the export keyword added to the type and to createDocumentRankingMetadataCache, which now cross the module boundary. withCachedIndexQuality and attachIndexQualityMetadata stay private — nothing outside the cluster calls them. The cluster referenced no rag.ts-local symbol, so the new module imports only stable siblings and carries no back-edge. rag.ts re-exports attachDocumentRankingMetadata and attachPageVisualEvidence so the public @/lib/rag/rag API is unchanged for tests/rag-query-concurrency.test.ts. Seven import bindings orphaned by the move were pruned from rag.ts. This does not unblock prepareCoverageGateResults, which stays in rag.ts: hydration covered only two of its five rag.ts-only dependencies. The remaining three — selectRankedRetrievalResults, applySecondStageRerankIfNeeded and measureSearchPhase — are a separate ranking/timing seam, as the Codex review on PR #1461 predicted. rag.ts 4780 -> 4543 lines; maintainability budget ratcheted 4780 -> 4543. RAG impact: no retrieval behaviour change — pure module extraction Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GGEBHp4Seoh1jK1vGTNtYS --- docs/codebase-index.md | 23 +- docs/maturity-backlog-workorders.md | 10 + docs/outstanding-issues.md | 2 +- scripts/check-maintainability-budgets.mjs | 7 +- src/lib/rag/rag-hydration.ts | 252 +++++++++++++++++++++ src/lib/rag/rag.ts | 257 +--------------------- 6 files changed, 289 insertions(+), 262 deletions(-) create mode 100644 src/lib/rag/rag-hydration.ts diff --git a/docs/codebase-index.md b/docs/codebase-index.md index b7e027ac3e..bdeeb08544 100644 --- a/docs/codebase-index.md +++ b/docs/codebase-index.md @@ -115,17 +115,18 @@ The `rag.ts` orchestrator and its `rag-*` cluster live in **`src/lib/rag/`** (th domain-extracted directory; imported as `@/lib/rag/rag*`). Other modules below remain flat in `src/lib/`. -| Module | Role | -| ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | -| `rag.ts` | Main answer pipeline orchestrator | -| `rag-routing.ts`, `rag-provider.ts`, `rag-answer-text.ts`, `smart-rag-api.ts` | Model routing, provider modes, API surface | -| `rag-contracts.ts`, `rag-answer-support.ts`, `rag-query-guard.ts` | Shared RAG contracts and pure answer/query policy | -| `rag-evidence-gates.ts`, `rag-coverage-gate.ts` | Evidence-sufficiency predicates and the fast-path evidence coverage gate | -| `rag-cache.ts`, `rag-retrieval-variants.ts` | Bounded caches and retrieval variants | -| `clinical-search.ts`, `clinical-query-mode.ts`, `retrieval-selection.ts` | Query modes and retrieval selection | -| `answer-ranking.ts`, `answer-verification.ts`, `answer-formatting.ts`, `answer-follow-up.ts`, `answer-render-policy.ts` | Answer quality and rendering | -| `citations.ts`, `cross-document-synthesis.ts`, `evidence-relevance.ts` | Evidence and synthesis | -| `ranking-config.ts`, `search-scope.ts`, `rag-eval-cases.ts` | Ranking tuning and eval fixtures | +| Module | Role | +| ----------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | +| `rag.ts` | Main answer pipeline orchestrator | +| `rag-routing.ts`, `rag-provider.ts`, `rag-answer-text.ts`, `smart-rag-api.ts` | Model routing, provider modes, API surface | +| `rag-contracts.ts`, `rag-answer-support.ts`, `rag-query-guard.ts` | Shared RAG contracts and pure answer/query policy | +| `rag-evidence-gates.ts`, `rag-coverage-gate.ts` | Evidence-sufficiency predicates and the fast-path evidence coverage gate | +| `rag-hydration.ts` | Per-request hydration: document ranking metadata, cached index quality, page visual evidence | +| `rag-cache.ts`, `rag-retrieval-variants.ts` | Bounded caches and retrieval variants | +| `clinical-search.ts`, `clinical-query-mode.ts`, `retrieval-selection.ts` | Query modes and retrieval selection | +| `answer-ranking.ts`, `answer-verification.ts`, `answer-formatting.ts`, `answer-follow-up.ts`, `answer-render-policy.ts` | Answer quality and rendering | +| `citations.ts`, `cross-document-synthesis.ts`, `evidence-relevance.ts` | Evidence and synthesis | +| `ranking-config.ts`, `search-scope.ts`, `rag-eval-cases.ts` | Ranking tuning and eval fixtures | ### Ingestion and indexing diff --git a/docs/maturity-backlog-workorders.md b/docs/maturity-backlog-workorders.md index 05463d089d..357c6ca22e 100644 --- a/docs/maturity-backlog-workorders.md +++ b/docs/maturity-backlog-workorders.md @@ -109,6 +109,16 @@ structural change, not a single mixed PR. extracted hook's `useSyncExternalStore` wiring. The residual is a tightly-coupled orchestrator core; further safe extractions are smaller, incremental units. `rag.ts` remains the largest open target. +- **Progress (#101):** extracted per-request hydration from `rag.ts` into + `src/lib/rag/rag-hydration.ts` — `DocumentRankingMetadataCache`, + `createDocumentRankingMetadataCache`, `attachDocumentRankingMetadata`, `withCachedIndexQuality`, + `attachIndexQualityMetadata`, `attachPageVisualEvidence`, moved byte-for-byte (rag.ts 4,780 → + 4,543, budget ratcheted to 4,543). Cycle-free: the cluster referenced no `rag.ts`-local symbol, + so the module imports only stable siblings and `rag.ts` re-exports the two names + `tests/rag-query-concurrency.test.ts` imports. `prepareCoverageGateResults` still cannot move — + hydration covered only two of its five `rag.ts`-only dependencies; the rest + (`selectRankedRetrievalResults`, `applySecondStageRerankIfNeeded`, `measureSearchPhase`) are a + separate ranking/timing seam. - **Approach:** extract cohesive units behind the existing budgets; the components decompose into their `*/` sibling directories, and `rag.ts` is the natural seam now that X2 has landed. - **Risk:** HIGH (behavioural surface). One file per PR. diff --git a/docs/outstanding-issues.md b/docs/outstanding-issues.md index c8f738fbd9..68f8158b7e 100644 --- a/docs/outstanding-issues.md +++ b/docs/outstanding-issues.md @@ -127,7 +127,7 @@ removed after current-main verification; it is not missing recommended work. | #040 | P3 | rec | Add targeted visual-regression baselines | Keep a small approved baseline set for high-value desktop/mobile surfaces and accessibility modes instead of screenshotting every route. Start with account/settings, document viewer, mode homes and bottom-composer interactions; define an intentional-update workflow before enabling blocking comparisons. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | | #079 | P3 | task | Disposition retained worktrees in bounded cleanup batches | **Outcome:** the retained reconciliation tail is gradually classified without another disruptive all-worktree sweep. **Next:** process no more than ten worktrees per explicitly scheduled pass using current owner/process metadata, open-PR state, exact review-ledger coverage, ancestry, and cherry-pick-aware content proof. **Success:** remove only clean, inactive, bundled worktrees whose content is merged or explicitly rejected; record every disposition and retain recovery evidence. **Stop:** preserve dirty, active, secret-bearing, post-freeze, paused, or ambiguous work and never use reset, force deletion, broad clean, or process killing. | final reconciliation inventory retained 104 independent worktrees; session 2026-07-24 | 2026-07-24 | | #085 | P3 | rec | Upload-limit client/server sync is unguarded | `NEXT_PUBLIC_MAX_UPLOAD_MB` (client, build-time inlined) and `MAX_UPLOAD_MB` (server, runtime) default 150/150 but nothing keeps them in sync — no zod link, gate, or test. Lower server-only → the 413-after-full-transfer UX FV-04 (#1064/#1069) was built to prevent; lower client-only → false pre-check rejection of files the server would accept (breaks #1064's client-is-a-strict-superset invariant). Both documented in `.env.example`/`docs/deployment-architecture.md` but unenforced; the client value is also frozen at `next build`, so changing the Railway service var without an image rebuild silently no-ops. Cheapest guard: a `check:*` script (or CI assertion) that fails when the two configured values disagree. | session 2026-07-28 (FV-04 adversarial workflow, PR #1069); ID #085 after #084 claimed on main | 2026-07-27 | -| #086 | P3 | task | Repository maturity backlog — remaining structural work | **Outcome:** the deferred repository-maturity backlog ships as verified draft PRs, one per structural change. **Canonical runbook:** [`docs/maturity-backlog-workorders.md`](maturity-backlog-workorders.md). **Remaining:** X3 `rag.ts` decomposition (in progress); X7 finish the `src/lib` domain reorg; X6 clinical/retrieval/answer coverage floors; X5 ACL-migration consolidation (provider-gated); L1 archive the retired `backfill:*` one-shots + the dead `ci-change-scope` token; M1 repo-host hardening (maintainer, audit §8). **Shipped:** L4 ledger rotation (#1418 — `ledger:rotate`, live/archive corpus, `merge=ledger`). **X3 progress:** the evidence coverage gate shipped as `src/lib/rag/rag-coverage-gate.ts` (PR #1454, squashed `102bb1f`) — `evaluateEvidenceCoverageGate` + `applyCoverageGateTelemetry` moved byte-identically, `rag.ts` 5,030 → 4,780, budget ratcheted to 4,780, no back-edge, `evaluateEvidenceCoverageGate` still re-exported from `@/lib/rag/rag`. **Next X3 unit — `rag-hydration.ts`:** the hydration cluster is `createDocumentRankingMetadataCache` / `attachDocumentRankingMetadata` / `withCachedIndexQuality` / `attachIndexQualityMetadata` / `attachPageVisualEvidence` plus the `DocumentRankingMetadataCache` type (`rag.ts:1487-1718` as of `102bb1f`). **It does NOT on its own unblock `prepareCoverageGateResults`** (corrected 2026-07-30 after a Codex finding on PR #1461 — the earlier claim that it re-homed all five dependencies was wrong). That function needs five `rag.ts`-only runtime symbols, and hydration accounts for only two of them (`attachDocumentRankingMetadata`, `attachPageVisualEvidence`). The other three sit outside the cluster and are a separate seam: `selectRankedRetrievalResults` (`rag.ts:1825`, retrieval selection), `applySecondStageRerankIfNeeded` (`rag.ts:679`, second-stage ranking), and `measureSearchPhase` (`rag.ts:1975`, the shared pipeline timing wrapper — 21 references across the file, of which only `metadata_hydration` and `visual_hydration` are hydration phases, so it belongs with the search orchestrator/telemetry, not with hydration). So `prepareCoverageGateResults` can only move after BOTH hydration and that ranking/timing seam are re-homed; it stayed in `rag.ts` for #086 because a back-edge and a signature change were both refused. Verify with a symbol-location plus call-site search before planning the boundary — do not treat hydration alone as sufficient. **Next:** remaining X3 units on user go-ahead. **Stop:** RAG/retrieval items need the flag + go-ahead; X5 is live-DB provider-gated. | `docs/maturity-backlog-workorders.md`; audit §8/§10; session 2026-07-28 | 2026-07-28 | +| #086 | P3 | task | Repository maturity backlog — remaining structural work | **Outcome:** the deferred repository-maturity backlog ships as verified draft PRs, one per structural change. **Canonical runbook:** [`docs/maturity-backlog-workorders.md`](maturity-backlog-workorders.md). **Remaining:** X3 `rag.ts` decomposition (in progress); X7 finish the `src/lib` domain reorg; X6 clinical/retrieval/answer coverage floors; X5 ACL-migration consolidation (provider-gated); L1 archive the retired `backfill:*` one-shots + the dead `ci-change-scope` token; M1 repo-host hardening (maintainer, audit §8). **Shipped:** L4 ledger rotation (#1418 — `ledger:rotate`, live/archive corpus, `merge=ledger`). **X3 progress:** the evidence coverage gate shipped as `src/lib/rag/rag-coverage-gate.ts` (PR #1454, squashed `102bb1f`) — `evaluateEvidenceCoverageGate` + `applyCoverageGateTelemetry` moved byte-identically, `rag.ts` 5,030 → 4,780, budget ratcheted to 4,780, no back-edge, `evaluateEvidenceCoverageGate` still re-exported from `@/lib/rag/rag`. **Hydration SHIPPED (#101):** `src/lib/rag/rag-hydration.ts` now owns the hydration cluster is `createDocumentRankingMetadataCache` / `attachDocumentRankingMetadata` / `withCachedIndexQuality` / `attachIndexQualityMetadata` / `attachPageVisualEvidence` plus the `DocumentRankingMetadataCache` type (byte-identical move; `rag.ts` 4,780 → 4,543, budget ratcheted). **It did NOT on its own unblock `prepareCoverageGateResults`**, exactly as the 2026-07-30 Codex finding on PR #1461 predicted (the original claim that it re-homed all five dependencies was wrong, and shipping it confirmed that). That function needs five `rag.ts`-only runtime symbols, and hydration accounts for only two of them (`attachDocumentRankingMetadata`, `attachPageVisualEvidence`). The other three sit outside the cluster and are a separate seam: `selectRankedRetrievalResults` (`rag.ts:1825`, retrieval selection), `applySecondStageRerankIfNeeded` (`rag.ts:679`, second-stage ranking), and `measureSearchPhase` (`rag.ts:1975`, the shared pipeline timing wrapper — 21 references across the file, of which only `metadata_hydration` and `visual_hydration` are hydration phases, so it belongs with the search orchestrator/telemetry, not with hydration). So `prepareCoverageGateResults` can only move after BOTH hydration and that ranking/timing seam are re-homed; it stayed in `rag.ts` for #086 because a back-edge and a signature change were both refused. Verify with a symbol-location plus call-site search before planning the boundary — do not treat hydration alone as sufficient. **Next:** remaining X3 units on user go-ahead. **Stop:** RAG/retrieval items need the flag + go-ahead; X5 is live-DB provider-gated. | `docs/maturity-backlog-workorders.md`; audit §8/§10; session 2026-07-28 | 2026-07-28 | | #090 | P2 | task | Upgrade the eslint ecosystem to clear remaining dev-scoped high advisories | **Outcome:** full `npm audit` reports zero high advisories from the eslint toolchain. **Next:** in a dedicated dependency pass, upgrade eslint and its plugin/config set together (npm offers `eslint@10.8.0`, `isSemVerMajor`); residual highs (`@eslint/config-array`, `@eslint/eslintrc`, `eslint`, `eslint-config-next`, `eslint-plugin-import`, `eslint-plugin-jsx-a11y`, plus the advisory's numeric `<=5.0.7` hit on the unused `brace-expansion@1.1.16` / `2.1.2` maintenance lines that still ship an unpatched `main`) cascade from this toolchain. **Success:** `npm run lint` clean, `verify:cheap` green, full-audit highs cleared, no rule-config regressions. **Stop:** an eslint major previously broke `eslint-plugin-react` here — do not bundle into an unrelated PR, and do not force with `npm audit fix --force`. Production `npm audit --omit=dev` is already clean after the exceljs `archiver@8` / `unzipper@0.12.5` overrides on PR #1314; this item is eslint-dev cleanup only. | session 2026-07-28 brace-expansion triage (PR #1314) | 2026-07-28 | | #091 | P2 | issue | Results band cannot express a partial-source failure | **Outcome:** a favourites/results count is never asserted while some of its sources failed to load. **Detail:** `SearchResultsBandStatus` is a single flat status, so a page fed by several registries has no way to say "3 loaded, 1 failed". `saved-registry-favourites-status.ts:29` works around this with `itemCount > 0 && folded !== "ready" ? "ready" : folded`, and `favourites-command-library-page.tsx` applies the same mask a second time — so a partial failure renders a confident count with no fault indication, which is the exact defect class PR #1316 removed everywhere else. Neither favourites page consumes the true `registryStatus` the fold already returns. **Next:** decide between (a) a `partial` arm on the status union carrying a "some sources unavailable" note beside an honest count, or (b) surfacing `registryStatus` as a separate inline notice and dropping the mask. **Stop:** do not widen the mask to more surfaces before the shape is decided. | PR #1316 review thread PRRT_kwDOSh5Fis6UUf-k; session 2026-07-28 | 2026-07-28 | | #092 | P3 | task | Refetch pulse deferred on auth-backed registries (privacy invariant) | **Outcome:** a background refresh keeps the prior count visible instead of a skeleton, without weakening identity clearing. **Detail:** the `refetching` status is built in the band and adopted only on `formulation-home-page.tsx`, where the lag is `useDeferredValue` over static data. It is deliberately NOT adopted on `use-registry-records.ts:85`, `use-medication-catalog.ts:76` or `use-differential-catalog.ts:133`, which all clear data on entering loading. `use-differential-catalog.ts:122` states why: "Auth must clear prior identity's matches immediately", and `:164` that "a later retype of any prior query cannot resurrect authorized matches." **Next:** if adopted, guard preservation on identity AND query equality, and pin with a test that an identity change still clears immediately. **Stop:** never hold records across an auth transition. | PR #1316 plan phase 6; session 2026-07-28 | 2026-07-28 | diff --git a/scripts/check-maintainability-budgets.mjs b/scripts/check-maintainability-budgets.mjs index 3d190cf457..421aee7ae6 100644 --- a/scripts/check-maintainability-budgets.mjs +++ b/scripts/check-maintainability-budgets.mjs @@ -5,9 +5,10 @@ const budgets = new Map([ // Chrome ownership/reporting lives in use-dashboard-chrome-coordinator; keep // the reclaimed monolith budget so it cannot silently drift back to 4160. ["src/components/ClinicalDashboard.tsx", 4140], - // The evidence coverage gate lives in rag-coverage-gate; keep the reclaimed - // budget so it cannot silently drift back to 5030. - ["src/lib/rag/rag.ts", 4780], + // The evidence coverage gate lives in rag-coverage-gate and per-request + // hydration in rag-hydration; keep the reclaimed budget so it cannot silently + // drift back to 5030. + ["src/lib/rag/rag.ts", 4543], ["src/components/DocumentViewer.tsx", 1734], ["supabase/functions/indexing-v3-agent/index.ts", 2191], ]); diff --git a/src/lib/rag/rag-hydration.ts b/src/lib/rag/rag-hydration.ts new file mode 100644 index 0000000000..f4d4916da8 --- /dev/null +++ b/src/lib/rag/rag-hydration.ts @@ -0,0 +1,252 @@ +import type { ChunkImage, ClinicalImageUseClass, SearchResult } from "@/lib/types"; +import { createAdminClient } from "@/lib/supabase/admin"; +import { fetchRelatedDocumentMetadata } from "@/lib/document-enrichment"; +import { normalizeImageBbox } from "@/lib/image-filtering"; +import { committedIndexGeneration, isCommittedGenerationMetadata } from "@/lib/reindex-pipeline"; +import { metadataText, safeRecord } from "@/lib/rag/rag-answer-text"; +import { compactContextText } from "@/lib/rag/rag-source-block"; + +// Extracted from rag.ts (maturity X3 / #101): per-request hydration of retrieved +// results — document ranking metadata, cached index quality, and page visual +// evidence — behind one bounded per-request cache. Behaviour-preserving: the +// function bodies are byte-identical to their previous rag.ts definitions. + +export type DocumentRankingMetadataCache = { + documentMetadata: Map< + string, + { labels: SearchResult["document_labels"]; summary: SearchResult["document_summary"] } | null + >; + indexQuality: Map; +}; + +/** Create document ranking metadata cache. */ +export function createDocumentRankingMetadataCache(): DocumentRankingMetadataCache { + return { + documentMetadata: new Map(), + indexQuality: new Map(), + }; +} + +/** Attach document ranking metadata. */ +export async function attachDocumentRankingMetadata( + supabase: ReturnType, + results: SearchResult[], + ownerId?: string, + cache = createDocumentRankingMetadataCache(), +) { + const documentIds = Array.from(new Set(results.map((result) => result.document_id))); + if (documentIds.length === 0) return results; + const missingDocumentIds = documentIds.filter( + (documentId) => + !cache.documentMetadata.has(documentId) && + results.some( + (result) => + result.document_id === documentId && + (result.document_labels === undefined || result.document_labels.length === 0) && + (result.document_summary === undefined || result.document_summary === null), + ), + ); + if (missingDocumentIds.length === 0) { + const enriched = results.map((result) => { + const metadata = cache.documentMetadata.get(result.document_id); + if (!metadata) return result; + if ( + (result.document_labels !== undefined && result.document_labels.length > 0) || + (result.document_summary !== undefined && result.document_summary !== null) + ) { + return result; + } + return { + ...result, + document_labels: metadata.labels, + document_summary: metadata.summary, + }; + }); + return attachIndexQualityMetadata(supabase, enriched, ownerId, cache); + } + + const [metadataRows, indexedResults] = await Promise.all([ + fetchRelatedDocumentMetadata({ + supabase, + ownerId, + documentIds: missingDocumentIds, + }).catch(() => null), + attachIndexQualityMetadata(supabase, results, ownerId, cache), + ]); + if (!metadataRows) return indexedResults; + + try { + for (const documentId of missingDocumentIds) cache.documentMetadata.set(documentId, null); + for (const row of metadataRows) { + cache.documentMetadata.set(row.document_id, { labels: row.labels, summary: row.summary }); + } + return indexedResults.map((result) => { + const metadata = cache.documentMetadata.get(result.document_id); + if (!metadata) return result; + return { + ...result, + document_labels: metadata.labels, + document_summary: metadata.summary, + }; + }); + } catch { + return indexedResults; + } +} + +/** With cached index quality. */ +function withCachedIndexQuality(results: SearchResult[], cache: DocumentRankingMetadataCache) { + return results.map((result) => ({ + ...result, + indexing_quality: cache.indexQuality.get(result.document_id) ?? result.indexing_quality ?? null, + })); +} + +/** Attach index quality metadata. */ +async function attachIndexQualityMetadata( + supabase: ReturnType, + results: SearchResult[], + ownerId?: string, + cache = createDocumentRankingMetadataCache(), +): Promise { + const documentIds = Array.from(new Set(results.map((result) => result.document_id))); + if (documentIds.length === 0) return results; + const missingDocumentIds = documentIds.filter((documentId) => !cache.indexQuality.has(documentId)); + if (missingDocumentIds.length === 0) return withCachedIndexQuality(results, cache); + try { + let query = supabase + .from("document_index_quality") + .select("document_id,owner_id,quality_score,extraction_quality,metrics,issues,updated_at") + .in("document_id", missingDocumentIds); + if (ownerId) query = query.eq("owner_id", ownerId); + const { data, error } = await query; + if (error) return results; + for (const documentId of missingDocumentIds) cache.indexQuality.set(documentId, null); + for (const row of data ?? []) cache.indexQuality.set(row.document_id, row as SearchResult["indexing_quality"]); + return withCachedIndexQuality(results, cache); + } catch { + return results; + } +} + +/** Attach page visual evidence. */ +export async function attachPageVisualEvidence( + supabase: ReturnType, + results: SearchResult[], +): Promise { + const documentIds = Array.from(new Set(results.map((result) => result.document_id))); + const pageNumbers = Array.from( + new Set(results.map((result) => result.page_number).filter((page): page is number => Boolean(page))), + ); + const sourceImageIds = Array.from( + new Set( + results.flatMap((result) => [ + result.index_unit?.source_image_id ?? null, + ...(result.table_facts ?? []).map((fact) => fact.source_image_id), + ]), + ), + ) + .filter((id): id is string => Boolean(id)) + .slice(0, 80); + if (documentIds.length === 0 || (pageNumbers.length === 0 && sourceImageIds.length === 0)) return results; + + const selectColumns = + "id,document_id,page_number,storage_path,caption,bbox,image_type,searchable,clinical_relevance_score,source_kind,width,height,labels,metadata"; + const [pageData, directData] = await Promise.all([ + pageNumbers.length > 0 + ? supabase + .from("document_images") + .select(selectColumns) + .in("document_id", documentIds) + .in("page_number", pageNumbers) + .eq("searchable", true) + .neq("image_type", "logo_decorative") + .order("clinical_relevance_score", { ascending: false }) + .limit(80) + : Promise.resolve({ data: [], error: null }), + sourceImageIds.length > 0 + ? supabase + .from("document_images") + .select(selectColumns) + .in("id", sourceImageIds) + .eq("searchable", true) + .neq("image_type", "logo_decorative") + .limit(sourceImageIds.length) + : Promise.resolve({ data: [], error: null }), + ]); + + const data = [...(pageData.data ?? []), ...(directData.data ?? [])]; + if ((pageData.error && directData.error) || data.length === 0) return results; + + const committedGenerationByDocument = new Map( + results.map((result) => [result.document_id, committedIndexGeneration(result.source_metadata)] as const), + ); + const imagesByPage = new Map(); + const imagesById = new Map(); + for (const image of data) { + if (imagesById.has(image.id)) continue; + const metadata = safeRecord(image.metadata); + if ( + !isCommittedGenerationMetadata({ + rowMetadata: metadata, + committedGeneration: committedGenerationByDocument.get(image.document_id), + }) + ) { + continue; + } + const rawTableText = metadataText(metadata, "table_text"); + const tableText = metadataText(metadata, "table_text_snippet") ?? rawTableText; + const publicImage: ChunkImage = { + id: image.id, + page_number: image.page_number, + storage_path: image.storage_path, + caption: image.caption, + bbox: normalizeImageBbox(image.bbox), + image_type: image.image_type as ChunkImage["image_type"], + searchable: image.searchable, + clinical_relevance_score: image.clinical_relevance_score, + source_kind: image.source_kind, + sourceKind: image.source_kind, + tableLabel: metadataText(metadata, "table_label"), + tableTitle: metadataText(metadata, "table_title"), + tableRole: metadataText(metadata, "table_role"), + clinicalUseClass: + typeof metadata.clinical_use_class === "string" ? (metadata.clinical_use_class as ClinicalImageUseClass) : null, + clinicalUseReason: typeof metadata.clinical_use_reason === "string" ? metadata.clinical_use_reason : null, + accessibleTableMarkdown: + typeof metadata.accessible_table_markdown === "string" ? metadata.accessible_table_markdown : rawTableText, + tableRows: Array.isArray(metadata.table_rows) ? (metadata.table_rows as string[][]) : null, + tableColumns: Array.isArray(metadata.table_columns) ? (metadata.table_columns as string[]) : null, + tableTextSnippet: tableText ? compactContextText(tableText, 500) : null, + labels: Array.isArray(image.labels) ? image.labels : [], + metadata, + }; + imagesById.set(image.id, publicImage); + const key = `${image.document_id}:${image.page_number}`; + imagesByPage.set(key, [...(imagesByPage.get(key) ?? []), publicImage]); + } + + return results.map((result) => { + const pageImages = imagesByPage.get(`${result.document_id}:${result.page_number}`) ?? []; + const directImages = [ + result.index_unit?.source_image_id ? imagesById.get(result.index_unit.source_image_id) : null, + ...(result.table_facts ?? []).map((fact) => (fact.source_image_id ? imagesById.get(fact.source_image_id) : null)), + ].filter((image): image is ChunkImage => Boolean(image)); + if (pageImages.length === 0 && directImages.length === 0) return result; + const seen = new Set((result.images ?? []).map((image) => image.id)); + const mergedImages = [ + ...(result.images ?? []), + ...directImages.filter((image) => { + if (seen.has(image.id)) return false; + seen.add(image.id); + return true; + }), + ...pageImages.filter((image) => { + if (seen.has(image.id)) return false; + seen.add(image.id); + return true; + }), + ].slice(0, 4); + return { ...result, images: mergedImages }; + }); +} diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 276b66199a..4200cc3270 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -134,7 +134,7 @@ import { recordAnswerOriginationFinished, recordCoalescedAnswerWaiter, } from "@/lib/observability/answer-coalescing-metrics"; -import { buildRagSourceBlock, compactContextText, neutralizeIdentityField } from "@/lib/rag/rag-source-block"; +import { buildRagSourceBlock, neutralizeIdentityField } from "@/lib/rag/rag-source-block"; export { buildRagSourceBlock, truncateForModel } from "@/lib/rag/rag-source-block"; import { buildClinicalTextSearchQuery, @@ -161,7 +161,6 @@ import { } from "@/lib/query-privacy"; import { normalizeSourceMetadata } from "@/lib/source-metadata"; import { safeErrorLogDetails } from "@/lib/privacy"; -import { normalizeImageBbox } from "@/lib/image-filtering"; import { SOURCE_BACKED_REVIEW_FALLBACK_REASON, chooseAnswerRoute, @@ -175,7 +174,7 @@ import { deadlineAllowsGenerationRetry, isAnswerRouteDeadlineExceeded, } from "@/lib/rag/rag-route-budget"; -import { fetchRelatedDocumentMetadata, fetchRelatedDocuments } from "@/lib/document-enrichment"; +import { fetchRelatedDocuments } from "@/lib/document-enrichment"; import { boldHighYieldClinicalText, boldRagAnswerHighYieldText, rankAnswerEvidence } from "@/lib/answer-ranking"; import { ragDeepMemoryVersion } from "@/lib/deep-memory"; import { @@ -218,6 +217,13 @@ import { } from "@/lib/rag/rag-evidence-gates"; import { applyCoverageGateTelemetry, evaluateEvidenceCoverageGate } from "@/lib/rag/rag-coverage-gate"; export { evaluateEvidenceCoverageGate } from "@/lib/rag/rag-coverage-gate"; +import { + attachDocumentRankingMetadata, + attachPageVisualEvidence, + createDocumentRankingMetadataCache, + type DocumentRankingMetadataCache, +} from "@/lib/rag/rag-hydration"; +export { attachDocumentRankingMetadata, attachPageVisualEvidence } from "@/lib/rag/rag-hydration"; import { cleanClinicalSummaryText, isLowYieldClinicalText } from "@/lib/source-text-sanitizer"; import { hasClinicalAnswerQualityIssue, @@ -225,7 +231,6 @@ import { looksLikeJsonArtifact, sanitizeAnswerText, sanitizeStructuredText, - metadataText, safeRecord, } from "@/lib/rag/rag-answer-text"; import { @@ -236,7 +241,7 @@ import { import { buildSmartRagApiPlan } from "@/lib/smart-rag-api"; import { clinicalModePrompt, queryClassForClinicalMode, queryForClinicalMode } from "@/lib/clinical-query-mode"; import { annotateSearchResults, buildEvidenceRelevance } from "@/lib/evidence-relevance"; -import { committedIndexGeneration, isCommittedGenerationMetadata } from "@/lib/reindex-pipeline"; +import { committedIndexGeneration } from "@/lib/reindex-pipeline"; import { buildRetrievalIntent, selectRetrievalEvidence } from "@/lib/retrieval-selection"; import { rankingConfig } from "@/lib/ranking-config"; import { resultsHaveReleaseRankScore, stabilizeReleasedSearchOrder } from "@/lib/released-search-order"; @@ -258,8 +263,6 @@ import type { AnswerSection, AnswerSectionKind, AnswerSectionSupportLevel, - ChunkImage, - ClinicalImageUseClass, Citation, ClinicalQueryAnalysis, EvidenceRelevance, @@ -1475,246 +1478,6 @@ function scoreExplanationLogMetadata(scoreExplanations: NonNullable; - indexQuality: Map; -}; - -/** Create document ranking metadata cache. */ -function createDocumentRankingMetadataCache(): DocumentRankingMetadataCache { - return { - documentMetadata: new Map(), - indexQuality: new Map(), - }; -} - -/** Attach document ranking metadata. */ -export async function attachDocumentRankingMetadata( - supabase: ReturnType, - results: SearchResult[], - ownerId?: string, - cache = createDocumentRankingMetadataCache(), -) { - const documentIds = Array.from(new Set(results.map((result) => result.document_id))); - if (documentIds.length === 0) return results; - const missingDocumentIds = documentIds.filter( - (documentId) => - !cache.documentMetadata.has(documentId) && - results.some( - (result) => - result.document_id === documentId && - (result.document_labels === undefined || result.document_labels.length === 0) && - (result.document_summary === undefined || result.document_summary === null), - ), - ); - if (missingDocumentIds.length === 0) { - const enriched = results.map((result) => { - const metadata = cache.documentMetadata.get(result.document_id); - if (!metadata) return result; - if ( - (result.document_labels !== undefined && result.document_labels.length > 0) || - (result.document_summary !== undefined && result.document_summary !== null) - ) { - return result; - } - return { - ...result, - document_labels: metadata.labels, - document_summary: metadata.summary, - }; - }); - return attachIndexQualityMetadata(supabase, enriched, ownerId, cache); - } - - const [metadataRows, indexedResults] = await Promise.all([ - fetchRelatedDocumentMetadata({ - supabase, - ownerId, - documentIds: missingDocumentIds, - }).catch(() => null), - attachIndexQualityMetadata(supabase, results, ownerId, cache), - ]); - if (!metadataRows) return indexedResults; - - try { - for (const documentId of missingDocumentIds) cache.documentMetadata.set(documentId, null); - for (const row of metadataRows) { - cache.documentMetadata.set(row.document_id, { labels: row.labels, summary: row.summary }); - } - return indexedResults.map((result) => { - const metadata = cache.documentMetadata.get(result.document_id); - if (!metadata) return result; - return { - ...result, - document_labels: metadata.labels, - document_summary: metadata.summary, - }; - }); - } catch { - return indexedResults; - } -} - -/** With cached index quality. */ -function withCachedIndexQuality(results: SearchResult[], cache: DocumentRankingMetadataCache) { - return results.map((result) => ({ - ...result, - indexing_quality: cache.indexQuality.get(result.document_id) ?? result.indexing_quality ?? null, - })); -} - -/** Attach index quality metadata. */ -async function attachIndexQualityMetadata( - supabase: ReturnType, - results: SearchResult[], - ownerId?: string, - cache = createDocumentRankingMetadataCache(), -): Promise { - const documentIds = Array.from(new Set(results.map((result) => result.document_id))); - if (documentIds.length === 0) return results; - const missingDocumentIds = documentIds.filter((documentId) => !cache.indexQuality.has(documentId)); - if (missingDocumentIds.length === 0) return withCachedIndexQuality(results, cache); - try { - let query = supabase - .from("document_index_quality") - .select("document_id,owner_id,quality_score,extraction_quality,metrics,issues,updated_at") - .in("document_id", missingDocumentIds); - if (ownerId) query = query.eq("owner_id", ownerId); - const { data, error } = await query; - if (error) return results; - for (const documentId of missingDocumentIds) cache.indexQuality.set(documentId, null); - for (const row of data ?? []) cache.indexQuality.set(row.document_id, row as SearchResult["indexing_quality"]); - return withCachedIndexQuality(results, cache); - } catch { - return results; - } -} - -/** Attach page visual evidence. */ -export async function attachPageVisualEvidence( - supabase: ReturnType, - results: SearchResult[], -): Promise { - const documentIds = Array.from(new Set(results.map((result) => result.document_id))); - const pageNumbers = Array.from( - new Set(results.map((result) => result.page_number).filter((page): page is number => Boolean(page))), - ); - const sourceImageIds = Array.from( - new Set( - results.flatMap((result) => [ - result.index_unit?.source_image_id ?? null, - ...(result.table_facts ?? []).map((fact) => fact.source_image_id), - ]), - ), - ) - .filter((id): id is string => Boolean(id)) - .slice(0, 80); - if (documentIds.length === 0 || (pageNumbers.length === 0 && sourceImageIds.length === 0)) return results; - - const selectColumns = - "id,document_id,page_number,storage_path,caption,bbox,image_type,searchable,clinical_relevance_score,source_kind,width,height,labels,metadata"; - const [pageData, directData] = await Promise.all([ - pageNumbers.length > 0 - ? supabase - .from("document_images") - .select(selectColumns) - .in("document_id", documentIds) - .in("page_number", pageNumbers) - .eq("searchable", true) - .neq("image_type", "logo_decorative") - .order("clinical_relevance_score", { ascending: false }) - .limit(80) - : Promise.resolve({ data: [], error: null }), - sourceImageIds.length > 0 - ? supabase - .from("document_images") - .select(selectColumns) - .in("id", sourceImageIds) - .eq("searchable", true) - .neq("image_type", "logo_decorative") - .limit(sourceImageIds.length) - : Promise.resolve({ data: [], error: null }), - ]); - - const data = [...(pageData.data ?? []), ...(directData.data ?? [])]; - if ((pageData.error && directData.error) || data.length === 0) return results; - - const committedGenerationByDocument = new Map( - results.map((result) => [result.document_id, committedIndexGeneration(result.source_metadata)] as const), - ); - const imagesByPage = new Map(); - const imagesById = new Map(); - for (const image of data) { - if (imagesById.has(image.id)) continue; - const metadata = safeRecord(image.metadata); - if ( - !isCommittedGenerationMetadata({ - rowMetadata: metadata, - committedGeneration: committedGenerationByDocument.get(image.document_id), - }) - ) { - continue; - } - const rawTableText = metadataText(metadata, "table_text"); - const tableText = metadataText(metadata, "table_text_snippet") ?? rawTableText; - const publicImage: ChunkImage = { - id: image.id, - page_number: image.page_number, - storage_path: image.storage_path, - caption: image.caption, - bbox: normalizeImageBbox(image.bbox), - image_type: image.image_type as ChunkImage["image_type"], - searchable: image.searchable, - clinical_relevance_score: image.clinical_relevance_score, - source_kind: image.source_kind, - sourceKind: image.source_kind, - tableLabel: metadataText(metadata, "table_label"), - tableTitle: metadataText(metadata, "table_title"), - tableRole: metadataText(metadata, "table_role"), - clinicalUseClass: - typeof metadata.clinical_use_class === "string" ? (metadata.clinical_use_class as ClinicalImageUseClass) : null, - clinicalUseReason: typeof metadata.clinical_use_reason === "string" ? metadata.clinical_use_reason : null, - accessibleTableMarkdown: - typeof metadata.accessible_table_markdown === "string" ? metadata.accessible_table_markdown : rawTableText, - tableRows: Array.isArray(metadata.table_rows) ? (metadata.table_rows as string[][]) : null, - tableColumns: Array.isArray(metadata.table_columns) ? (metadata.table_columns as string[]) : null, - tableTextSnippet: tableText ? compactContextText(tableText, 500) : null, - labels: Array.isArray(image.labels) ? image.labels : [], - metadata, - }; - imagesById.set(image.id, publicImage); - const key = `${image.document_id}:${image.page_number}`; - imagesByPage.set(key, [...(imagesByPage.get(key) ?? []), publicImage]); - } - - return results.map((result) => { - const pageImages = imagesByPage.get(`${result.document_id}:${result.page_number}`) ?? []; - const directImages = [ - result.index_unit?.source_image_id ? imagesById.get(result.index_unit.source_image_id) : null, - ...(result.table_facts ?? []).map((fact) => (fact.source_image_id ? imagesById.get(fact.source_image_id) : null)), - ].filter((image): image is ChunkImage => Boolean(image)); - if (pageImages.length === 0 && directImages.length === 0) return result; - const seen = new Set((result.images ?? []).map((image) => image.id)); - const mergedImages = [ - ...(result.images ?? []), - ...directImages.filter((image) => { - if (seen.has(image.id)) return false; - seen.add(image.id); - return true; - }), - ...pageImages.filter((image) => { - if (seen.has(image.id)) return false; - seen.add(image.id); - return true; - }), - ].slice(0, 4); - return { ...result, images: mergedImages }; - }); -} - /** Decide text fast path. */ export function decideTextFastPath( query: string,