diff --git a/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogEntryFetcher.cs b/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogEntryFetcher.cs index f511292fdd..9e69933f21 100644 --- a/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogEntryFetcher.cs +++ b/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogEntryFetcher.cs @@ -376,6 +376,99 @@ private static Uri CombineSegments(Uri baseUri, IReadOnlyList segments) return new Uri($"{basePath}/{suffix}"); } + /// + /// + /// Fetches all note-*.yml entries for / at + /// from the CDN. Reads the notes-{target}.json index to enumerate + /// the pool-relative note paths; a missing index means no notes (not an error). A listed note that + /// cannot be fetched is a hard error — the index is an authoritative promise that the note exists. + /// + /// CDN base URI. + /// Repository org (e.g. elastic). + /// Repository name (e.g. kibana). + /// Target version string (e.g. 9.0.0). + /// Called once per hard error; caller decides how to surface it. + /// Cancellation token. + /// The fetched note entries, keyed by pool-relative path (main/note-foo.yml). + public async Task> FetchNotesAsync( + Uri baseUri, + string org, + string repo, + string target, + Action emitError, + Cancel ctx) + { + if (!ChangelogKeys.IsValidOrg(org) || !ChangelogKeys.IsValidRepo(repo)) + { + emitError($"Invalid org/repo '{org}/{repo}' for notes fetch: must be non-empty ASCII letters, digits, '.', '_' or '-'."); + return []; + } + + var indexUri = CombineSegments(baseUri, ["changelog", org, repo, $"notes-{target}.json"]); + NotesIndex? index; + try + { + var (notFound, content) = await FetchTextOrNotFoundAsync(indexUri, 1, ctx).ConfigureAwait(false); + if (notFound) + { + _logger.LogDebug("Notes index for {Org}/{Repo}@{Target} not found at {Uri}; no notes to bundle", org, repo, target, indexUri); + return []; + } + index = JsonSerializer.Deserialize(content, NotesIndexJsonContext.Default.NotesIndex); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + emitError($"Could not fetch notes index for {org}/{repo}@{target} from {indexUri}: {ex.Message}"); + return []; + } + + if (index is null || index.Notes.Count == 0) + return []; + + var repoLabel = $"{org}/{repo}"; + var entries = new List(index.Notes.Count); + foreach (var poolRelativePath in index.Notes) + { + ctx.ThrowIfCancellationRequested(); + + // Pool-relative path is "{branch}/note-{name}.yml"; split on first '/' only. + var slash = poolRelativePath.IndexOf('/', StringComparison.Ordinal); + if (slash <= 0 || slash == poolRelativePath.Length - 1) + { + emitError($"Notes index for {repoLabel}@{target} lists an invalid pool-relative path '{poolRelativePath}'; expected {{branch}}/{{file}}."); + return []; + } + var branch = poolRelativePath[..slash]; + var noteFileName = poolRelativePath[(slash + 1)..]; + + if (!ChangelogKeys.IsValidBranch(branch)) + { + emitError($"Notes index for {repoLabel}@{target} lists path '{poolRelativePath}' with an invalid branch segment."); + return []; + } + + var poolSegments = ChangelogKeys.PoolSegments(org, repo, branch); + var noteUri = CombineSegments(baseUri, [.. poolSegments, noteFileName]); + var poolLabel = $"{repoLabel}/{branch}"; + + var (fetched, content, lastError) = await TryFetchNamedEntryAsync(noteUri, noteFileName, poolLabel, ctx).ConfigureAwait(false); + if (fetched) + { + entries.Add(new CdnChangelogEntry(poolRelativePath, content)); + continue; + } + + // The notes index asserts this note exists — a miss is a real pipeline error. + emitError( + $"Note '{poolRelativePath}' for {repoLabel}@{target} is listed in the notes index but could not be fetched from {noteUri}: {lastError}. " + + "Ensure the note was uploaded and scrubbed; if it persists check the changelog scrubber pipeline."); + return []; + } + + _logger.LogInformation("Fetched {Count} note(s) for {Repo}@{Target} from {BaseUri}", entries.Count, repoLabel, target, baseUri); + return entries; + } + /// /// Disposes the per-instance created for an injected handler. The shared /// production client () is process-lived and intentionally not disposed. diff --git a/src/Elastic.Documentation.Configuration/ReleaseNotes/NotesIndex.cs b/src/Elastic.Documentation.Configuration/ReleaseNotes/NotesIndex.cs new file mode 100644 index 0000000000..8eaac0bcd7 --- /dev/null +++ b/src/Elastic.Documentation.Configuration/ReleaseNotes/NotesIndex.cs @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json.Serialization; + +namespace Elastic.Documentation.Configuration.ReleaseNotes; + +/// +/// Notes index published at changelog/{org}/{repo}/notes-{target}.json. +/// Lists pool-relative paths of all note-*.yml fragments for one target, +/// across every branch of the repo. +/// +/// +/// Contents are paths, not bodies — the note files remain the single source of truth. +/// A stale index can only omit or over-list, never serve stale prose. Bundling a target +/// is therefore 1 GET for the index + one GET per listed note. +/// +public sealed record NotesIndex +{ + /// Pool-relative paths of notes for this target, e.g. ["main/note-slow-rollover.yml"]. + public required IReadOnlyList Notes { get; init; } +} + +[JsonSourceGenerationOptions( + WriteIndented = true, + PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull +)] +[JsonSerializable(typeof(NotesIndex))] +public sealed partial class NotesIndexJsonContext : JsonSerializerContext; diff --git a/src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs b/src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs index 832e95b1b5..3e9dcea447 100644 --- a/src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs +++ b/src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs @@ -411,13 +411,19 @@ public async Task BundleChangelogs(IDiagnosticsCollector collector, Bundle if (collector.Errors > 0) return false; - if (matchResult.Entries.Count == 0) + // Merge notes for the target when sourcing from CDN and an explicit target is available. + // Notes are target-scoped by their index, so they are never filtered by PR/issue criteria. + var allEntries = await MergeNotesAsync(collector, matchResult.Entries, useCdn, authoringOwner, authoringRepo, input, ctx); + if (allEntries == null) + return false; + + if (allEntries.Count == 0) { collector.EmitError(string.Empty, "No changelog entries matched the filter criteria"); return false; } - return await BuildAndWriteBundle(collector, input, config, matchResult.Entries, outputPath, ctx); + return await BuildAndWriteBundle(collector, input, config, allEntries, outputPath, ctx); } catch (IOException ioEx) { @@ -1216,6 +1222,108 @@ private BundleChangelogsArguments ApplyConfigDefaults(BundleChangelogsArguments return byName.Select(kv => (kv.Key, kv.Value)).ToList(); } + /// + /// Fetches notes for from the CDN and converts them to matched entries. + /// An absent notes index is not an error (most targets have no notes). Returns null after + /// emitting an error when the index exists but a listed note cannot be fetched. + /// + private async Task?> FetchCdnNotesAsync( + IDiagnosticsCollector collector, + string? org, + string? repo, + string target, + Cancel ctx) + { + if (string.IsNullOrWhiteSpace(repo)) + return []; + + var resolvedOrg = string.IsNullOrWhiteSpace(org) ? DefaultOwner : org; + + var baseUri = ChangelogCdn.ResolveBaseUri(); + if (baseUri is null) + return []; + + var hadError = false; + var cdnEntries = await _entryFetcher.FetchNotesAsync( + baseUri, resolvedOrg, repo, target, + msg => { hadError = true; collector.EmitError(string.Empty, msg); }, + ctx); + + if (hadError) + return null; + + if (cdnEntries.Count == 0) + return []; + + var matchedNotes = new List(cdnEntries.Count); + foreach (var entry in cdnEntries) + { + try + { + var normalized = ReleaseNotesSerialization.NormalizeYaml(entry.Content); + var dto = ReleaseNotesSerialization.GetEntryDeserializer().Deserialize(normalized); + var data = ReleaseNotesSerialization.ConvertEntry(dto); + var checksum = ComputeSha1(entry.Content); + matchedNotes.Add(new MatchedChangelogFile + { + Data = data, + FilePath = entry.FileName, + FileName = entry.FileName, + Checksum = checksum + }); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Failed to parse note '{FileName}' for {Repo}@{Target}; skipping", entry.FileName, repo, target); + collector.EmitError(string.Empty, $"Note '{entry.FileName}' for {repo}@{target} could not be parsed: {ex.Message}"); + return null; + } + } + + _logger.LogInformation("Resolved {Count} note(s) for {Repo}@{Target} from CDN", matchedNotes.Count, repo, target); + return matchedNotes; + } + + /// + /// Fetches notes from the CDN (when applicable) and appends them to , + /// deduplicating by checksum. Returns null after emitting an error on any fatal failure; + /// returns the original list unchanged when CDN notes are not applicable. + /// + private async Task?> MergeNotesAsync( + IDiagnosticsCollector collector, + IReadOnlyList entries, + bool useCdn, + string? org, + string? repo, + BundleChangelogsArguments input, + Cancel ctx) + { + if (!useCdn) + return entries; + + var noteTargets = ResolveNoteTargets(input); + if (noteTargets.Count == 0) + return entries; + + // Dedup by checksum: a note body identical to a PR entry (edge case) should appear once. + var seen = new HashSet(entries.Select(e => e.Checksum), StringComparer.OrdinalIgnoreCase); + var combined = new List(entries); + + foreach (var noteTarget in noteTargets) + { + var noteEntries = await FetchCdnNotesAsync(collector, org, repo, noteTarget, ctx); + if (noteEntries == null) + return null; + + foreach (var note in noteEntries) + { + if (seen.Add(note.Checksum)) + combined.Add(note); + } + } + return combined; + } + /// Gate for repo-scoped CDN entry sourcing: true when the authoring repo resolves, local sourcing is not forced (bundle.use_local_changelogs/--force-local/--directory), and a CDN base is configured. private static bool ShouldSourceFromCdn(string? authoringRepo, bool useLocalChangelogs, bool explicitDirectory) { @@ -1334,6 +1442,22 @@ private static bool ValidatePlaceholderUsage(IDiagnosticsCollector collector, Bu return true; } + /// + /// Returns all distinct, explicit, non-wildcard targets from . + /// Notes are fetched for every resolved target so multi-target bundles are fully covered. + /// Returns an empty list when no concrete targets are available. + /// + private static IReadOnlyList ResolveNoteTargets(BundleChangelogsArguments input) + { + if (input.OutputProducts is not { Count: > 0 }) + return []; + return input.OutputProducts + .Where(p => !string.IsNullOrWhiteSpace(p.Target) && p.Target != "*") + .Select(p => p.Target!) + .Distinct(StringComparer.Ordinal) + .ToList(); + } + private static ChangelogFilterCriteria BuildFilterCriteria( BundleChangelogsArguments input, HashSet prsToMatch, diff --git a/src/services/Elastic.Changelog/Reconciliation/NotesIndex.cs b/src/services/Elastic.Changelog/Reconciliation/NotesIndex.cs index b541de8a20..91193c1f25 100644 --- a/src/services/Elastic.Changelog/Reconciliation/NotesIndex.cs +++ b/src/services/Elastic.Changelog/Reconciliation/NotesIndex.cs @@ -2,30 +2,6 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information -using System.Text.Json.Serialization; - -namespace Elastic.Changelog.Reconciliation; - -/// -/// Notes index published at changelog/{org}/{repo}/notes-{target}.json. -/// Lists pool-relative paths of all note-*.yml fragments for one target, -/// across every branch of the repo. -/// -/// -/// Contents are paths, not bodies — the note files remain the single source of truth. -/// A stale index can only omit or over-list, never serve stale prose. Bundling a target -/// is therefore 1 GET for the index + one GET per listed note. -/// -public sealed record NotesIndex -{ - /// Pool-relative paths of notes for this target, e.g. ["main/note-slow-rollover.yml"]. - public required IReadOnlyList Notes { get; init; } -} - -[JsonSourceGenerationOptions( - WriteIndented = true, - PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull -)] -[JsonSerializable(typeof(NotesIndex))] -public sealed partial class NotesIndexJsonContext : JsonSerializerContext; +// NotesIndex and NotesIndexJsonContext are defined in +// Elastic.Documentation.Configuration.ReleaseNotes (NotesIndex.cs in that project), +// so the CDN entry fetcher and the reconciler share one definition. diff --git a/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogEntryFetcherTests.cs b/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogEntryFetcherTests.cs index 7d2f182786..d3420acb1c 100644 --- a/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogEntryFetcherTests.cs +++ b/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogEntryFetcherTests.cs @@ -188,6 +188,77 @@ public async Task FetchAsync_UnsafeFileName_EmitsWarningAndSkips() warnings.Should().ContainSingle().Which.Should().Contain("escape.yaml"); } + [Fact] + public async Task FetchNotesAsync_IndexAbsent_ReturnsEmptyWithNoError() + { + // A missing notes index means "no notes for this target" — not a pipeline error. + var handler = new StubHandler(_ => new HttpResponseMessage(HttpStatusCode.NotFound)); + var (errors, _, emitError, _) = Diagnostics(); + + using var fetcher = CreateFetcher(handler); + var entries = await fetcher.FetchNotesAsync(BaseUri, "elastic", "elasticsearch", "9.0.0", emitError, TestContext.Current.CancellationToken); + + entries.Should().BeEmpty(); + errors.Should().BeEmpty("a missing index is expected for targets with no notes"); + handler.RequestedPaths.Should().ContainSingle() + .Which.Should().EndWith("/changelog/elastic/elasticsearch/notes-9.0.0.json"); + } + + [Fact] + public async Task FetchNotesAsync_HappyPath_FetchesAllListedNotes() + { + var handler = new StubHandler(req => + { + var path = req.RequestUri!.AbsolutePath; + if (path.EndsWith("/notes-9.0.0.json", StringComparison.Ordinal)) + return Json(/*lang=json,strict*/ """{"notes":["main/note-slow-rollover.yml","9.0/note-gap.yml"]}"""); + return Yaml(SampleEntry); + }); + var (errors, _, emitError, _) = Diagnostics(); + + using var fetcher = CreateFetcher(handler); + var entries = await fetcher.FetchNotesAsync(BaseUri, "elastic", "elasticsearch", "9.0.0", emitError, TestContext.Current.CancellationToken); + + errors.Should().BeEmpty(); + entries.Select(e => e.FileName).Should().BeEquivalentTo("main/note-slow-rollover.yml", "9.0/note-gap.yml"); + // Verify the actual note URLs contain branch segments + handler.RequestedPaths.Should().Contain(p => p.EndsWith("/main/note-slow-rollover.yml", StringComparison.Ordinal)); + handler.RequestedPaths.Should().Contain(p => p.EndsWith("/9.0/note-gap.yml", StringComparison.Ordinal)); + } + + [Fact] + public async Task FetchNotesAsync_ListedNoteNotFound_EmitsErrorAndReturnsEmpty() + { + // A note listed in the index that cannot be fetched is a hard error — the index promises it exists. + var handler = new StubHandler(req => + { + var path = req.RequestUri!.AbsolutePath; + if (path.EndsWith("/notes-9.0.0.json", StringComparison.Ordinal)) + return Json(/*lang=json,strict*/ """{"notes":["main/note-missing.yml"]}"""); + return new HttpResponseMessage(HttpStatusCode.NotFound); + }); + var (errors, _, emitError, _) = Diagnostics(); + + using var fetcher = CreateFetcher(handler, maxAttempts: 2); + var entries = await fetcher.FetchNotesAsync(BaseUri, "elastic", "elasticsearch", "9.0.0", emitError, TestContext.Current.CancellationToken); + + entries.Should().BeEmpty(); + errors.Should().ContainSingle().Which.Should().Contain("note-missing.yml"); + } + + [Fact] + public async Task FetchNotesAsync_EmptyIndex_ReturnsEmpty() + { + var handler = new StubHandler(_ => Json(/*lang=json,strict*/ """{"notes":[]}""")); + var (errors, _, emitError, _) = Diagnostics(); + + using var fetcher = CreateFetcher(handler); + var entries = await fetcher.FetchNotesAsync(BaseUri, "elastic", "elasticsearch", "9.0.0", emitError, TestContext.Current.CancellationToken); + + entries.Should().BeEmpty(); + errors.Should().BeEmpty(); + } + private static HttpResponseMessage Json(string body) => new(HttpStatusCode.OK) { Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") };