chore: organize dirty work from codex/comparison-alias-distinctness

src/lib/eval-document-matching.ts

@@ -111,17 +111,63 @@ function resultDocumentText(source: Pick<SearchResult, "file_name" | "title">) {
   return normalizedDocumentName(`${source.title} ${source.file_name}`);
 }
 
+type ExpectedFileSource = Pick<SearchResult, "file_name" | "title"> & Partial<Pick<SearchResult, "document_id">>;
+
+function distinctSourceDocuments(sources: ExpectedFileSource[], limit: number) {
+  const documents = new Map<string, string>();
+  sources.slice(0, limit).forEach((source, index) => {
+    const identity =
+      normalizedDocumentName(source.document_id ?? "") ||
+      normalizedDocumentName(source.file_name) ||
+      normalizedDocumentName(source.title) ||
+      `source ${index}`;
+    const text = resultDocumentText(source);
+    const previous = documents.get(identity);
+    documents.set(identity, previous ? `${previous} ${text}` : text);
+  });
+  return [...documents.values()];
+}
+
+// Coverage is a one-to-one assignment, not independent substring checks: a broad alias may
+// match several expectations, but one retrieved document can provide evidence for only one slot.
+// The augmenting-path pass avoids a greedy false miss when a flexible expectation can move to a
+// second source and free the only valid source for a narrower expectation.
+function matchExpectedFilesToDistinctSources(expectedFiles: string[], sourceDocuments: string[]) {
+  const alternatives = expectedFiles.map(documentExpectationAlternatives);
+  const sourceForExpected = Array<number>(expectedFiles.length).fill(-1);
+  const expectedForSource = Array<number>(sourceDocuments.length).fill(-1);
+
+  function assign(expectedIndex: number, visitedSources: Set<number>): boolean {
+    for (let sourceIndex = 0; sourceIndex < sourceDocuments.length; sourceIndex += 1) {
+      if (visitedSources.has(sourceIndex)) continue;
+      if (!alternatives[expectedIndex]?.some((alternative) => sourceDocuments[sourceIndex]?.includes(alternative))) {
+        continue;
+      }
+
+      visitedSources.add(sourceIndex);
+      const currentExpected = expectedForSource[sourceIndex] ?? -1;
+      if (currentExpected !== -1 && !assign(currentExpected, visitedSources)) continue;
+
+      expectedForSource[sourceIndex] = expectedIndex;
+      sourceForExpected[expectedIndex] = sourceIndex;
+      return true;
+    }
+    return false;
+  }
+
+  expectedFiles.forEach((_, expectedIndex) => {
+    assign(expectedIndex, new Set());
+  });
+
+  return expectedFiles.filter((_, expectedIndex) => sourceForExpected[expectedIndex] !== -1);
+}
+
 export function expectedFileCoverage(
   expectedFiles: string[],
-  sources: Array<Pick<SearchResult, "file_name" | "title">>,
+  sources: ExpectedFileSource[],
   limit = 3,
 ): ExpectedFileCoverage {
-  const topFiles = sources.slice(0, limit).map(resultDocumentText);
-  const matchedFiles = expectedFiles.filter((expected) =>
-    documentExpectationAlternatives(expected).some((alternative) =>
-      topFiles.some((file) => file.includes(alternative)),
-    ),
-  );
+  const matchedFiles = matchExpectedFilesToDistinctSources(expectedFiles, distinctSourceDocuments(sources, limit));
 
   return {
     expectedFiles,

tests/eval-utils.test.ts

@@ -54,6 +54,103 @@ describe("RAG eval source identity matching", () => {
     expect(coverage.missingFiles).toEqual(["MHSP.Discharge.pdf"]);
   });
 
+  it("does not let one retrieved document satisfy both sides of a comparison", () => {
+    const coverage = expectedFileCoverage(
+      ["MHSP.AdmissionCommunityPts.pdf", "MHSP.Discharge.pdf"],
+      [
+        {
+          title: "Admission to Discharge for Mental Health Inpatients",
+          file_name: "Admission to Discharge for Mental Health Inpatients (NMHS).pdf",
+        },
+      ],
+      5,
+    );
+
+    expect(coverage.matchedFiles).toEqual(["MHSP.AdmissionCommunityPts.pdf"]);
+    expect(coverage.missingFiles).toEqual(["MHSP.Discharge.pdf"]);
+    expect(coverage.anyHit).toBe(true);
+    expect(coverage.allHit).toBe(false);
+  });
+
+  it("finds a complete one-to-one assignment when an overlapping alias has a distinct counterpart", () => {
+    const coverage = expectedFileCoverage(
+      ["MHSP.Discharge.pdf", "MHSP.AdmissionCommunityPts.pdf"],
+      [
+        {
+          title: "Admission to Discharge for Mental Health Inpatients",
+          file_name: "Admission to Discharge for Mental Health Inpatients (NMHS).pdf",
+        },
+        {
+          title: "Discharge Planning for Community Patients",
+          file_name: "Discharge Planning for Community Patients (NMHS).pdf",
+        },
+      ],
+      5,
+    );
+
+    expect(coverage.matchedFiles).toEqual(["MHSP.Discharge.pdf", "MHSP.AdmissionCommunityPts.pdf"]);
+    expect(coverage.missingFiles).toEqual([]);
+    expect(coverage.allHit).toBe(true);
+  });
+
+  it("does not count repeated chunks from one file as distinct expected documents", () => {
+    const repeatedSource = {
+      title: "Admission to Discharge for Mental Health Inpatients",
+      file_name: "Admission to Discharge for Mental Health Inpatients (NMHS).pdf",
+    };
+    const coverage = expectedFileCoverage(
+      ["MHSP.AdmissionCommunityPts.pdf", "MHSP.Discharge.pdf"],
+      [repeatedSource, repeatedSource],
+      5,
+    );
+
+    expect(coverage.matchedFiles).toHaveLength(1);
+    expect(coverage.missingFiles).toHaveLength(1);
+    expect(coverage.allHit).toBe(false);
+  });
+
+  it("uses document identity when repeated chunks have different display text", () => {
+    const coverage = expectedFileCoverage(
+      ["MHSP.AdmissionCommunityPts.pdf", "MHSP.Discharge.pdf"],
+      [
+        {
+          document_id: "11111111-1111-4111-8111-111111111111",
+          title: "Admission to Discharge for Mental Health Inpatients",
+          file_name: "admission-policy.pdf",
+        },
+        {
+          document_id: "11111111-1111-4111-8111-111111111111",
+          title: "Discharge Planning for Community Patients",
+          file_name: "discharge-policy.pdf",
+        },
+      ],
+      5,
+    );
+
+    expect(coverage.matchedFiles).toEqual(["MHSP.AdmissionCommunityPts.pdf"]);
+    expect(coverage.missingFiles).toEqual(["MHSP.Discharge.pdf"]);
+    expect(coverage.allHit).toBe(false);
+  });
+
+  it("keeps distinct document identities separate even when their display names match", () => {
+    const sharedDisplay = {
+      title: "Admission to Discharge for Mental Health Inpatients",
+      file_name: "Admission to Discharge for Mental Health Inpatients (NMHS).pdf",
+    };
+    const coverage = expectedFileCoverage(
+      ["MHSP.AdmissionCommunityPts.pdf", "MHSP.Discharge.pdf"],
+      [
+        { ...sharedDisplay, document_id: "11111111-1111-4111-8111-111111111111" },
+        { ...sharedDisplay, document_id: "22222222-2222-4222-8222-222222222222" },
+      ],
+      5,
+    );
+
+    expect(coverage.matchedFiles).toEqual(["MHSP.AdmissionCommunityPts.pdf", "MHSP.Discharge.pdf"]);
+    expect(coverage.missingFiles).toEqual([]);
+    expect(coverage.allHit).toBe(true);
+  });
+
   it("fails supported clinical eval cases when numeric faithfulness warnings are present", () => {
     const testCase: RagEvalCase = {
       id: "dose-warning",