Skip to content

feat(orchestrator): add soft-delete marker label to the check metric #3144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ValentaTomas merged 1 commit into from
Jun 29, 2026
Merged

feat(orchestrator): add soft-delete marker label to the check metric #3144

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions packages/orchestrator/pkg/sandbox/build/softdelete.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,16 @@ var softDeleteCheckMetric = utils.Must(meter.Int64Counter(

// recordCheck emits one metric point per check. result distinguishes a read
// (checked) from an unreadable object (not_found/error); soft_deleted/failed
// are only meaningful when result==checked.
func (b *StorageDiff) recordCheck(ctx context.Context, result string, softDeleted, failed bool) {
// are only meaningful when result==checked. marker carries the tombstone value
// (matching the log), set only on a hit so its action_id doesn't inflate
// cardinality on the common no-tombstone path.
func (b *StorageDiff) recordCheck(ctx context.Context, result string, softDeleted, failed bool, marker string) {
softDeleteCheckMetric.Add(ctx, 1, metric.WithAttributes(
attribute.String("artifact", string(b.diffType)),
attribute.String("result", result),
attribute.Bool("soft_deleted", softDeleted),
attribute.Bool("failed", failed),
attribute.String("marker", marker),
))
}

Expand Down Expand Up @@ -87,7 +90,7 @@ func (b *StorageDiff) softDeleteCheck(ctx context.Context, ff *featureflags.Clie
blob, err := b.persistence.OpenBlob(ctx, path)
if err != nil {
result := classifyCheckError(err)
b.recordCheck(ctx, result, false, false)
b.recordCheck(ctx, result, false, false, "")
logger.L().Warn(ctx, "storage-index soft-delete check could not open object",
logger.WithBuildID(b.buildID), zap.String("artifact", string(b.diffType)),
zap.String("result", result), zap.String("object", path), zap.Error(err))
Expand All @@ -103,15 +106,15 @@ func (b *StorageDiff) softDeleteCheck(ctx context.Context, ff *featureflags.Clie
// possibly-tombstoned layer (BlobCustomMetadata used to return no error
// here, which silently failed open).
if result == checkResultUnsupported && ff.BoolFlag(ctx, featureflags.StorageSoftDeleteEnforceFlag) {
b.recordCheck(ctx, result, false, true)
b.recordCheck(ctx, result, false, true, "")
b.softDeletedPath.Store(&path)
logger.L().Error(ctx, "storage-index soft-delete unverifiable; failing closed",
logger.WithBuildID(b.buildID), zap.String("artifact", string(b.diffType)),
zap.String("result", result), zap.String("object", path), zap.Error(err))

return
}
b.recordCheck(ctx, result, false, false)
b.recordCheck(ctx, result, false, false, "")
logger.L().Warn(ctx, "storage-index soft-delete check could not read object metadata",
logger.WithBuildID(b.buildID), zap.String("artifact", string(b.diffType)),
zap.String("result", result),
Expand All @@ -125,7 +128,7 @@ func (b *StorageDiff) softDeleteCheck(ctx context.Context, ff *featureflags.Clie
enforce := ff.BoolFlag(ctx, featureflags.StorageSoftDeleteEnforceFlag)
failed := softDeleted && enforce

b.recordCheck(ctx, checkResultChecked, softDeleted, failed)
b.recordCheck(ctx, checkResultChecked, softDeleted, failed, marker)

if !softDeleted {
return
Expand Down
Loading