Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public sealed record BundledEntryDto
public string? Pr { get; set; }
public List<string>? Prs { get; set; }
public List<string>? Issues { get; set; }
public string? Link { get; set; }
}

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,54 @@ public async Task<IReadOnlyList<CdnChangelogEntry>> FetchAsync(
return (false, await response.Content.ReadAsStringAsync(ctx).ConfigureAwait(false));
}

/// <summary>
/// Probes for a PR's changelog entry by number. Returns <c>null</c> when the entry does not exist (404);
/// this is the normal "no changelog for this PR" case and is not an error. Transient failures
/// (5xx, timeouts, transport errors) are retried with the same budget as other fetches.
/// Unlike <see cref="TryFetchNamedEntryAsync"/>, a 404 here is authoritative-null, not an error.
/// </summary>
public async Task<CdnChangelogEntry?> FetchPrEntryAsync(
Uri baseUri, string org, string repo, string branch, int prNumber, Cancel ctx)
{
var fileName = $"{prNumber}.yaml";
var poolSegments = ChangelogKeys.PoolSegments(org, repo, branch);
var uri = CombineSegments(baseUri, [.. poolSegments, fileName]);
var poolLabel = $"{org}/{repo}/{branch}";
var (fetched, content, _) = await TryProbeEntryAsync(uri, fileName, poolLabel, ctx).ConfigureAwait(false);
return fetched ? new CdnChangelogEntry(fileName, content) : null;
}

private async Task<(bool Fetched, string Content, string? LastError)> TryProbeEntryAsync(Uri uri, string fileName, string poolLabel, Cancel ctx)
{
string? lastError = null;
for (var attempt = 1; attempt <= _maxAttempts; attempt++)
{
ctx.ThrowIfCancellationRequested();
try
{
var (notFound, content) = await FetchTextOrNotFoundAsync(uri, attempt, ctx).ConfigureAwait(false);
if (notFound)
return (false, string.Empty, null); // 404 = probe miss, not an error
if (attempt > 1)
_logger.LogInformation("Probed changelog entry '{File}' for {Pool} on attempt {Attempt}/{Max}", fileName, poolLabel, attempt, _maxAttempts);
return (true, content, null);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
if (ex is HttpRequestException { StatusCode: >= HttpStatusCode.BadRequest and < HttpStatusCode.InternalServerError })
return (false, string.Empty, ex.Message);
lastError = ex.Message;
if (attempt >= _maxAttempts)
break;
var delay = RetryDelay(attempt);
_logger.LogDebug("Probe for changelog entry '{File}' for {Pool} failed (attempt {Attempt}/{Max}: {Error}); retrying in {Delay}",
fileName, poolLabel, attempt, _maxAttempts, ex.Message, delay);
await _sleep(delay, ctx).ConfigureAwait(false);
}
}
return (false, string.Empty, lastError);
}

/// <summary>
/// Fetches a single entry, retrying transient failures (most importantly a not-yet-propagated 404)
/// up to <see cref="_maxAttempts"/> times with exponential backoff. Retry requests are cache-busted
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@ public record ChangelogEntryDto
[YamlMember(Alias = "feature-id", ApplyNamingConventions = false)]
public string? FeatureId { get; set; }
public bool? Highlight { get; set; }
/// <summary>
/// Marker reference: a bare PR number pointing to the authoritative entry in the same pool.
/// A marker carries <c>link:</c> and nothing else; any other field alongside it is invalid.
/// Written by the pipeline for non-primary PRs in a multi-PR entry; never hand-authored.
/// </summary>
public string? Link { get; set; }
}

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,8 @@ private static string ToYamlDoubleQuotedString(string s)
Impact = dto.Impact,
Action = dto.Action,
FeatureId = dto.FeatureId,
Highlight = dto.Highlight
Highlight = dto.Highlight,
Link = dto.Link
};

private static ChangelogEntry ToEntry(BundledEntry entry) => new()
Expand All @@ -181,7 +182,8 @@ private static string ToYamlDoubleQuotedString(string s)
Impact = entry.Impact,
Action = entry.Action,
FeatureId = entry.FeatureId,
Highlight = entry.Highlight
Highlight = entry.Highlight,
Link = entry.Link
};

private static ProductReference ToProductReference(ProductInfoDto dto) => new()
Expand Down Expand Up @@ -225,7 +227,8 @@ private static string ToYamlDoubleQuotedString(string s)
Subtype = ParseEntrySubtype(dto.Subtype),
Areas = dto.Areas,
Prs = dto.Prs ?? (dto.Pr != null ? [dto.Pr] : null),
Issues = dto.Issues
Issues = dto.Issues,
Link = dto.Link
};

private static BundledFile ToBundledFile(BundledFileDto dto) => new()
Expand Down Expand Up @@ -281,21 +284,29 @@ private static ChangelogEntryType ParseEntryType(string? value)

// Reverse mappings (Domain → DTO) for serialization

private static ChangelogEntryDto ToDto(ChangelogEntry entry) => new()
private static ChangelogEntryDto ToDto(ChangelogEntry entry)
{
Prs = entry.Prs?.ToList(),
Issues = entry.Issues?.ToList(),
Type = EntryTypeToString(entry.Type),
Subtype = EntrySubtypeToString(entry.Subtype),
Products = entry.Products?.Select(ToDto).ToList(),
Areas = entry.Areas?.ToList(),
Title = entry.Title,
Description = entry.Description,
Impact = entry.Impact,
Action = entry.Action,
FeatureId = entry.FeatureId,
Highlight = entry.Highlight
};
// Marker entries are link-only; emitting any other field would violate the marker contract.
if (entry.IsMarker)
return new ChangelogEntryDto { Link = entry.Link };

return new ChangelogEntryDto
{
Prs = entry.Prs?.ToList(),
Issues = entry.Issues?.ToList(),
Type = EntryTypeToString(entry.Type),
Subtype = EntrySubtypeToString(entry.Subtype),
Products = entry.Products?.Select(ToDto).ToList(),
Areas = entry.Areas?.ToList(),
Title = entry.Title,
Description = entry.Description,
Impact = entry.Impact,
Action = entry.Action,
FeatureId = entry.FeatureId,
Highlight = entry.Highlight,
Link = entry.Link
};
}

private static ProductInfoDto ToDto(ProductReference product) => new()
{
Expand Down Expand Up @@ -338,7 +349,8 @@ private static ChangelogEntryType ParseEntryType(string? value)
Subtype = EntrySubtypeToString(entry.Subtype),
Areas = entry.Areas?.ToList(),
Prs = entry.Prs?.ToList(),
Issues = entry.Issues?.ToList()
Issues = entry.Issues?.ToList(),
Link = entry.Link
};

private static BundledFileDto ToDto(BundledFile file) => new()
Expand Down
6 changes: 6 additions & 0 deletions src/Elastic.Documentation/ReleaseNotes/BundledEntry.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,10 @@ public record BundledEntry

/// <summary>Related issue URLs or references.</summary>
public IReadOnlyList<string>? Issues { get; init; }

/// <summary>
/// Marker reference: a bare PR number pointing to the authoritative entry in the same pool.
/// Preserved through the scrub round-trip so the public pool can resolve markers end-to-end.
/// </summary>
public string? Link { get; init; }
}
13 changes: 12 additions & 1 deletion src/Elastic.Documentation/ReleaseNotes/ChangelogEntry.cs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,16 @@ public record ChangelogEntry
/// <summary>Whether this entry should be highlighted.</summary>
public bool? Highlight { get; init; }

/// <summary>
/// Marker reference: a bare PR number pointing to the authoritative entry in the same pool.
/// Non-null only on machine-written marker objects emitted by the pipeline for non-primary PRs
/// in a multi-PR entry. A marker carries <c>link:</c> and nothing else.
/// </summary>
public string? Link { get; init; }

/// <summary>True when this entry is a pipeline-written marker that redirects to another PR's entry.</summary>
public bool IsMarker => Link is not null;

/// <summary>
/// Converts this ChangelogEntry to a BundledEntry for embedding in bundles.
/// File property is set to null; set it separately using a 'with' expression.
Expand All @@ -64,6 +74,7 @@ public record ChangelogEntry
Subtype = Subtype,
Areas = Areas,
Prs = Prs,
Issues = Issues
Issues = Issues,
Link = Link
};
}
133 changes: 124 additions & 9 deletions src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -289,11 +289,11 @@ public async Task<bool> BundleChangelogs(IDiagnosticsCollector collector, Bundle
if (!ValidateInput(collector, input, requireDirectoryExists: !useCdn))
return false;

// --all and --input-products require reading every entry body; that is only possible locally.
// On the CDN path entries are probed by key, so there is nothing to enumerate without a PR list.
if (useCdn && (input.All || input.InputProducts is { Count: > 0 }))
// --all, --input-products, and --issues require reading every entry body; that is only possible locally.
// On the CDN path entries are probed by key (one per PR), so there is nothing to enumerate without a PR list.
if (useCdn && (input.All || input.InputProducts is { Count: > 0 } || input.Issues is { Length: > 0 }))
{
var flag = input.All ? "--all" : "--input-products";
var flag = input.All ? "--all" : input.Issues is { Length: > 0 } ? "--issues" : "--input-products";
collector.EmitError(string.Empty,
$"{flag} is not supported when sourcing changelog entries from the CDN, because entries are fetched by key (one per PR) and there is no pool enumeration. " +
"Pass --force-local or --directory to bundle from a local checkout instead.");
Expand Down Expand Up @@ -380,12 +380,16 @@ public async Task<bool> BundleChangelogs(IDiagnosticsCollector collector, Bundle
}
else
{
// --prs / --issues on the CDN path: still uses the pool registry for now (Step 9 will
// switch this to per-PR probing once canonical keys and markers are in place).
var contents = await FetchCdnEntriesAsync(collector, authoringOwner, authoringRepo, authoringBranch, ctx);
if (contents == null)
// --prs / --report / --release-version on the CDN path: probe one key per PR number.
// Each entry lives at changelog/{org}/{repo}/{branch}/{pr}.yaml; 404 = no entry for that PR.
var probed = await FetchCdnProbedEntriesAsync(collector, authoringOwner, authoringRepo, authoringBranch, prsToMatch, ctx);
if (probed == null)
return false;
matchResult = entryMatcher.MatchChangelogContents(collector, contents, filterCriteria, ctx);
_logger.LogInformation("Probed {Count} changelog entry(ies) for {Pool} from CDN",
probed.Count, $"{authoringOwner}/{authoringRepo}/{authoringBranch}");
// Entries are already selected by probe; pass IncludeAll so the matcher skips content re-filtering.
var probeCriteria = filterCriteria with { IncludeAll = true };
matchResult = entryMatcher.MatchChangelogContents(collector, probed, probeCriteria, ctx);
}
}
else
Expand Down Expand Up @@ -1324,6 +1328,117 @@ private BundleChangelogsArguments ApplyConfigDefaults(BundleChangelogsArguments
return combined;
}

private async Task<IReadOnlyList<(string FileName, string Content)>?> FetchCdnProbedEntriesAsync(
IDiagnosticsCollector collector,
string? org,
string? repo,
string? branch,
HashSet<string> prsToMatch,
Cancel ctx)
{
if (string.IsNullOrWhiteSpace(repo))
{
collector.EmitError(string.Empty,
"Sourcing changelog entries from the CDN requires a resolvable authoring repository. " +
"Set bundle.repo in changelog.yml (or pass --repo), or pass --force-local / --directory to bundle local changelog files.");
return null;
}

var resolvedOrg = string.IsNullOrWhiteSpace(org) ? DefaultOwner : org;
var resolvedBranch = string.IsNullOrWhiteSpace(branch) ? DefaultBranch : branch;
var poolLabel = $"{resolvedOrg}/{repo}/{resolvedBranch}";

var baseUri = ChangelogCdn.ResolveBaseUri();
if (baseUri is null)
{
collector.EmitError(string.Empty,
$"No valid changelog CDN base URL is configured. Set the {ChangelogCdn.BaseUrlEnvironmentVariable} environment variable to an absolute http(s) URL.");
return null;
}

var prNumbers = new List<int>();
foreach (var pr in prsToMatch)
{
if (TryExtractPrNumber(pr, resolvedOrg, repo, out var prNumber))
prNumbers.Add(prNumber);
}

if (prNumbers.Count == 0)
{
_logger.LogInformation("No PR numbers for {Pool} found in filter; returning empty probe result", poolLabel);
return [];
}

var tasks = prNumbers.Select(pr =>
_entryFetcher.FetchPrEntryAsync(baseUri, resolvedOrg, repo, resolvedBranch, pr, ctx)).ToArray();
var results = await Task.WhenAll(tasks).ConfigureAwait(false);

var entries = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var hasError = false;

for (var i = 0; i < prNumbers.Count; i++)
{
var prNumber = prNumbers[i];
var cdnEntry = results[i];

if (cdnEntry is null)
{
_logger.LogWarning("No changelog entry found for PR {PrNumber} in {Pool}", prNumber, poolLabel);
continue;
}

var entry = ReleaseNotesSerialization.DeserializeEntry(cdnEntry.Value.Content);
if (!entry.IsMarker)
{
_ = entries.TryAdd(cdnEntry.Value.FileName, cdnEntry.Value.Content);
continue;
}

if (!int.TryParse(entry.Link, out var parentPr) || parentPr <= 0)
{
collector.EmitError(string.Empty,
$"Changelog entry '{cdnEntry.Value.FileName}' contains an invalid link: '{entry.Link}'. Expected a positive PR number.");
hasError = true;
continue;
}

var parentCdnEntry = await _entryFetcher.FetchPrEntryAsync(baseUri, resolvedOrg, repo, resolvedBranch, parentPr, ctx).ConfigureAwait(false);
if (parentCdnEntry is null)
{
collector.EmitError(string.Empty,
$"Changelog entry '{cdnEntry.Value.FileName}' is a marker pointing to PR {parentPr}, but that entry does not exist in {poolLabel}.");
hasError = true;
continue;
}

var parentEntry = ReleaseNotesSerialization.DeserializeEntry(parentCdnEntry.Value.Content);
if (parentEntry.IsMarker)
{
collector.EmitError(string.Empty,
$"Marker chain detected: '{cdnEntry.Value.FileName}' → '{parentCdnEntry.Value.FileName}' is also a marker. Marker chains are not allowed.");
hasError = true;
continue;
}

_ = entries.TryAdd(parentCdnEntry.Value.FileName, parentCdnEntry.Value.Content);
}

return hasError ? null : entries.Select(kv => (kv.Key, kv.Value)).ToList();
}

private static bool TryExtractPrNumber(string pr, string authoringOwner, string authoringRepo, out int prNumber)
{
prNumber = 0;
// Normalize to {owner}/{repo}#{number} form; the org/repo may differ from the CDN pool
// (e.g. --owner override renames the pool but the PR URLs still carry the GitHub org).
// We only need the number to build the probe key.
var normalized = NormalizePrForComparison(pr, authoringOwner, authoringRepo);
var hashIndex = normalized.LastIndexOf('#');
if (hashIndex < 0)
return false;
return int.TryParse(normalized[(hashIndex + 1)..], out prNumber) && prNumber > 0;
}

/// <summary>Gate for repo-scoped CDN entry sourcing: true when the authoring repo resolves, local sourcing is not forced (<c>bundle.use_local_changelogs</c>/<c>--force-local</c>/<c>--directory</c>), and a CDN base is configured.</summary>
private static bool ShouldSourceFromCdn(string? authoringRepo, bool useLocalChangelogs, bool explicitDirectory)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,8 @@ private async Task<string> ScrubChangelog(string content, Cancel ctx)
Issues = entry.Issues,
Areas = entry.Areas,
Highlight = entry.Highlight,
Subtype = entry.Subtype
Subtype = entry.Subtype,
Link = entry.Link
};

await using var collector = new DiagnosticsCollector([]);
Expand All @@ -107,7 +108,8 @@ private async Task<string> ScrubChangelog(string content, Cancel ctx)
Impact = sanitized.Impact,
Action = sanitized.Action,
Prs = sanitized.Prs,
Issues = sanitized.Issues
Issues = sanitized.Issues,
Link = entry.Link
};

var result = ReleaseNotesSerialization.SerializeEntry(scrubEntry);
Expand Down
Loading
Loading