diff --git a/src/review/content-lane/duplicates.ts b/src/review/content-lane/duplicates.ts index 9c8251da51..64fb1cdcbd 100644 --- a/src/review/content-lane/duplicates.ts +++ b/src/review/content-lane/duplicates.ts @@ -528,25 +528,22 @@ export type DirectoryIndexEntry = Record & { canonicalUrl?: unknown; }; -/** URL signal fields read off a directory entry, in order. */ -const DIRECTORY_ENTRY_URL_SIGNAL_FIELDS = [ - "documentationUrl", - "docsUrl", - "downloadUrl", - "githubUrl", - "packageUrl", - "repoUrl", - "repositoryUrl", - "sourceUrl", - "websiteUrl", -] as const; - function yamlScalar(value: unknown): string { return JSON.stringify(String(value || "")); } -/** Synthesize the per-entry frontmatter block. */ -function contentSignalSourceFromDirectoryEntry(entry: DirectoryIndexEntry): string { +/** + * Synthesize the per-entry frontmatter block for directory-index corpus extraction. + * URL lines come from `spec.urlFields` (not a hardcoded list) so they agree with + * `extractContentDuplicateSignals`'s filtering — custom ContentRepoSpec URL keys must + * reach the synthesized frontmatter or corpus `urls` stay silently empty (#5941). + * + * Exported for unit tests so the default-`spec` parameter branch is exercised (#5941 Codecov). + */ +export function contentSignalSourceFromDirectoryEntry( + entry: DirectoryIndexEntry, + spec: ContentRepoSpec = AWESOME_CLAUDE_CONTENT_SPEC, +): string { const lines = [ "---", `title: ${yamlScalar(entry.title)}`, @@ -554,7 +551,7 @@ function contentSignalSourceFromDirectoryEntry(entry: DirectoryIndexEntry): stri `category: ${yamlScalar(entry.category)}`, `slug: ${yamlScalar(entry.slug)}`, ]; - for (const field of DIRECTORY_ENTRY_URL_SIGNAL_FIELDS) { + for (const field of spec.urlFields) { const value = entry[field]; if (value) lines.push(`${field}: ${yamlScalar(value)}`); } @@ -588,7 +585,7 @@ export function directoryIndexToSignals( extractContentDuplicateSignals( { filePath, - content: contentSignalSourceFromDirectoryEntry(entry), + content: contentSignalSourceFromDirectoryEntry(entry, spec), label: `accepted entry ${filePath}`, url: String(entry.canonicalUrl || "") || `${siteUrl}/entry/${String(entry.category)}/${String(entry.slug)}`, }, diff --git a/test/unit/content-lane-duplicates.test.ts b/test/unit/content-lane-duplicates.test.ts index 59a5039e61..49b83c73e9 100644 --- a/test/unit/content-lane-duplicates.test.ts +++ b/test/unit/content-lane-duplicates.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { buildContentDuplicateReview, + contentSignalSourceFromDirectoryEntry, directoryIndexToSignals, extractContentDuplicateSignals, findContentDuplicateMatch, @@ -837,4 +838,59 @@ describe("per-repo ContentRepoSpec override (a self-hosted curated list re-param expect(directoryIndexToSignals(entries)[0]?.urls).toEqual(["https://github.com/o/r"]); // default url fields read githubUrl expect(directoryIndexToSignals(entries, {}, customSpec({ urlFields: new Set() }))[0]?.urls).toEqual([]); // custom empty set → none }); + + // #5941 — before the fix, directory-index synthesis always emitted a hardcoded URL-field list, so a custom + // ContentRepoSpec.urlFields set never reached extractContentDuplicateSignals and corpus urls stayed []. + it("directoryIndexToSignals emits custom ContentRepoSpec urlFields into corpus urls (regression #5941)", () => { + const entries = [ + { + category: "tools", + slug: "x", + title: "X", + description: "d", + myLink: "https://example.com/custom", + githubUrl: "https://github.com/o/r", + }, + ]; + const spec = customSpec({ urlFields: new Set(["myLink"]) }); + const signals = directoryIndexToSignals(entries, {}, spec); + expect(signals[0]?.urls).toEqual(["https://example.com/custom"]); + expect(signals[0]?.urls).not.toContain("https://github.com/o/r"); + }); + + it("default AWESOME_CLAUDE_CONTENT_SPEC still synthesizes the prior camelCase URL fields unchanged (#5941)", () => { + const entries = [ + { + category: "skills", + slug: "foo", + title: "Foo", + description: "d", + githubUrl: "https://github.com/acme/foo", + websiteUrl: "https://acme.example", + myLink: "https://should-not-appear.example", + }, + ]; + const signals = directoryIndexToSignals(entries); + expect(signals[0]?.urls).toEqual(["https://github.com/acme/foo", "https://acme.example"]); + expect(signals[0]?.urls).not.toContain("https://should-not-appear.example"); + }); + + it("contentSignalSourceFromDirectoryEntry defaults spec to AWESOME_CLAUDE_CONTENT_SPEC and skips empty URL values (#5941)", () => { + // One-arg call covers the default-parameter branch; empty-string githubUrl covers the falsy `if (value)` arm. + const synthesized = contentSignalSourceFromDirectoryEntry({ + category: "skills", + slug: "foo", + title: "Foo", + description: "d", + githubUrl: "", + websiteUrl: "https://acme.example", + }); + expect(synthesized).toContain('websiteUrl: "https://acme.example"'); + expect(synthesized).not.toContain("githubUrl:"); + const viaDefaultSpec = extractContentDuplicateSignals({ + filePath: "content/skills/foo.mdx", + content: synthesized, + }); + expect(viaDefaultSpec.urls).toEqual(["https://acme.example"]); + }); });