diff --git a/pkg/cli/audit_cache.go b/pkg/cli/audit_cache.go index 7fb4be0f8f2..586ed035e9c 100644 --- a/pkg/cli/audit_cache.go +++ b/pkg/cli/audit_cache.go @@ -80,7 +80,13 @@ func writeLogsAuditFiles(processedRuns []ProcessedRun, verbose bool) { func writeLogsAuditFile(processedRun ProcessedRun, processedRuns []ProcessedRun, verbose bool) { runOutputDir := processedRun.Run.LogsPath - auditData, ok := loadCachedAuditData(runOutputDir, processedRun.Run, auditCacheSourceLogs) + var auditData AuditData + ok := processedRun.cachedAudit != nil + if ok { + auditData = *processedRun.cachedAudit + } else { + auditData, ok = loadCachedAuditData(runOutputDir, processedRun.Run, auditCacheSourceLogs) + } if !ok { metrics := LogMetrics{} if summary, ok := loadRunSummary(runOutputDir, verbose); ok { @@ -89,8 +95,41 @@ func writeLogsAuditFile(processedRun ProcessedRun, processedRuns []ProcessedRun, auditData, _ = buildLocalAuditData(processedRun, metrics, processedRun.MCPToolUsage) auditData.CacheSource = auditCacheSourceLogs } - auditData.Comparison = buildAuditComparisonForProcessedRuns(processedRun, processedRuns) + hydratedProcessedRuns := hydrateProcessedRunsWithCachedAudit(processedRuns) + auditData.Comparison = buildAuditComparisonForProcessedRuns(hydrateProcessedRunWithCachedAudit(processedRun), hydratedProcessedRuns) if err := writeAuditData(runOutputDir, auditData); err != nil { logsOrchestratorLog.Printf("Failed to write audit file for run %d: %v", processedRun.Run.DatabaseID, err) + return + } + if processedRun.cachedData != nil { + processedRun.cachedData.AuditPath = auditPath(runOutputDir) + } +} + +func hydrateProcessedRunsWithCachedAudit(processedRuns []ProcessedRun) []ProcessedRun { + hydrated := make([]ProcessedRun, len(processedRuns)) + for i, processedRun := range processedRuns { + hydrated[i] = hydrateProcessedRunWithCachedAudit(processedRun) + } + return hydrated +} + +func hydrateProcessedRunWithCachedAudit(processedRun ProcessedRun) ProcessedRun { + if processedRun.cachedAudit == nil { + return processedRun + } + audit := processedRun.cachedAudit + if processedRun.Run.Turns == 0 { + processedRun.Run.Turns = audit.Metrics.Turns + } + if len(processedRun.SafeOutputs) == 0 { + processedRun.SafeOutputs = audit.CreatedItems + } + if processedRun.FirewallAnalysis == nil { + processedRun.FirewallAnalysis = audit.FirewallAnalysis + } + if len(processedRun.MCPFailures) == 0 { + processedRun.MCPFailures = audit.MCPFailures } + return processedRun } diff --git a/pkg/cli/logs_cached_json.go b/pkg/cli/logs_cached_json.go index 444ff250af6..fb660afa202 100644 --- a/pkg/cli/logs_cached_json.go +++ b/pkg/cli/logs_cached_json.go @@ -799,12 +799,13 @@ func normalizeCachedLogRun(run *RunData) error { // cachedJSONLCanSatisfy permits cached records only for the compact usage // artifact, whose JSON includes the metadata required for cached reports. -// Parsing, auditing, training, and tool graphs require raw artifact files. -func cachedJSONLCanSatisfy(artifactFilter []string, parse, audit, train, toolGraph bool) bool { - return isUsageOnlyArtifactFilter(artifactFilter) && !parse && !audit && !train && !toolGraph +// Audit mode uses the available cached data on a best-effort basis. Parsing, +// explicit training, and tool graphs require raw artifact files. +func cachedJSONLCanSatisfy(artifactFilter []string, parse, train, toolGraph bool) bool { + return isUsageOnlyArtifactFilter(artifactFilter) && !parse && !train && !toolGraph } -func processedRunFromCachedData(data RunData) ProcessedRun { +func processedRunFromCachedData(data RunData, audit *AuditData, outputDir string) ProcessedRun { return ProcessedRun{ Run: WorkflowRun{ DatabaseID: data.RunID, @@ -832,6 +833,7 @@ func processedRunFromCachedData(data RunData) ProcessedRun { MissingToolCount: data.MissingToolCount, MissingDataCount: data.MissingDataCount, SafeItemsCount: data.SafeItemsCount, + LogsPath: filepath.Join(outputDir, fmt.Sprintf("run-%d", data.RunID)), }, AwContext: data.AwContext, TaskDomain: data.TaskDomain, @@ -840,5 +842,6 @@ func processedRunFromCachedData(data RunData) ProcessedRun { TokenUsage: data.TokenUsageSummary, WorkingSet: data.WorkingSet, cachedData: &data, + cachedAudit: audit, } } diff --git a/pkg/cli/logs_cached_json_test.go b/pkg/cli/logs_cached_json_test.go index 1ae434915db..369a8fb087f 100644 --- a/pkg/cli/logs_cached_json_test.go +++ b/pkg/cli/logs_cached_json_test.go @@ -10,9 +10,11 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "testing" "time" + "github.com/github/gh-aw/pkg/console" "github.com/github/gh-aw/pkg/constants" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -688,16 +690,22 @@ func TestCachedLogsLookupRejectsUnknownIdentity(t *testing.T) { func TestCachedJSONLCanSatisfy(t *testing.T) { usageFilter := []string{constants.UsageArtifactName.String()} agentFilter := []string{constants.AgentArtifactName.String()} - assert.True(t, cachedJSONLCanSatisfy(usageFilter, false, false, false, false)) - assert.False(t, cachedJSONLCanSatisfy(agentFilter, false, false, false, false)) - assert.False(t, cachedJSONLCanSatisfy(nil, false, false, false, false)) - assert.False(t, cachedJSONLCanSatisfy(usageFilter, true, false, false, false)) - assert.False(t, cachedJSONLCanSatisfy(usageFilter, false, true, false, false)) - assert.False(t, cachedJSONLCanSatisfy(usageFilter, false, false, true, false)) - assert.False(t, cachedJSONLCanSatisfy(usageFilter, false, false, false, true)) + assert.True(t, cachedJSONLCanSatisfy(usageFilter, false, false, false)) + assert.False(t, cachedJSONLCanSatisfy(agentFilter, false, false, false)) + assert.False(t, cachedJSONLCanSatisfy(nil, false, false, false)) + assert.False(t, cachedJSONLCanSatisfy(usageFilter, true, false, false)) + assert.False(t, cachedJSONLCanSatisfy(usageFilter, false, true, false)) + assert.False(t, cachedJSONLCanSatisfy(usageFilter, false, false, true)) } func TestDownloadRunArtifactsConcurrentReusesCachedJSONRecord(t *testing.T) { + originalProcess := processConcurrentRunDownload + t.Cleanup(func() { processConcurrentRunDownload = originalProcess }) + processConcurrentRunDownload = func(context.Context, WorkflowRun, concurrentRunDownloadParams, *atomic.Int64, *console.ProgressBar) (DownloadResult, error) { + t.Fatal("cached run should not be downloaded") + return DownloadResult{}, nil + } + cached := RunData{ RunID: 42, WorkflowName: "cached-workflow", @@ -708,11 +716,12 @@ func TestDownloadRunArtifactsConcurrentReusesCachedJSONRecord(t *testing.T) { UpdatedAt: time.Date(2026, time.September, 1, 0, 0, 0, 0, time.UTC), LogsPath: "/previous/run-42", } + cachedAudit := &AuditData{Overview: OverviewData{RunID: 42}} results := downloadRunArtifactsConcurrent(context.Background(), []WorkflowRun{{DatabaseID: 42, Repository: "github/gh-aw", Status: "completed", Conclusion: "success", Attempt: 1, UpdatedAt: cached.UpdatedAt}}, runArtifactsConcurrentOptions{ outputDir: t.TempDir(), maxRuns: 1, - cachedRuns: cachedLogsRuns{42: {RunData: cached}}, + cachedRuns: cachedLogsRuns{42: {RunData: cached, Audit: cachedAudit}}, storageLimit: newLogsStorageLimit(t.TempDir(), 0, false), }) @@ -720,6 +729,82 @@ func TestDownloadRunArtifactsConcurrentReusesCachedJSONRecord(t *testing.T) { require.NotNil(t, results[0].CachedRun) assert.True(t, results[0].Cached) assert.Equal(t, cached, *results[0].CachedRun) + assert.Same(t, cachedAudit, results[0].cachedAudit) +} + +func TestPrepareLogsDataAuditUsesCachedDataBestEffort(t *testing.T) { + outputDir := t.TempDir() + baseline := RunData{ + RunID: 41, + WorkflowName: "cached-workflow", + Status: "completed", + Conclusion: "success", + CreatedAt: time.Date(2026, time.September, 1, 0, 0, 0, 0, time.UTC), + UpdatedAt: time.Date(2026, time.September, 1, 0, 0, 0, 0, time.UTC), + } + cached := RunData{ + RunID: 42, + WorkflowName: "cached-workflow", + Status: "completed", + Conclusion: "success", + CreatedAt: baseline.CreatedAt.Add(time.Minute), + UpdatedAt: baseline.UpdatedAt, + } + baselineAudit := &AuditData{ + CacheSource: auditCacheSourceLogs, + Overview: OverviewData{ + RunID: baseline.RunID, + WorkflowName: baseline.WorkflowName, + Status: baseline.Status, + Conclusion: baseline.Conclusion, + CreatedAt: baseline.CreatedAt, + UpdatedAt: baseline.UpdatedAt, + }, + Metrics: MetricsData{Turns: 2}, + FirewallAnalysis: &FirewallAnalysis{AnalysisBase: AnalysisBase{BlockedRequests: 1}}, + } + cachedAudit := &AuditData{ + CacheSource: auditCacheSourceLogs, + Overview: OverviewData{ + RunID: cached.RunID, + WorkflowName: cached.WorkflowName, + Status: cached.Status, + Conclusion: cached.Conclusion, + CreatedAt: cached.CreatedAt, + UpdatedAt: cached.UpdatedAt, + }, + Metrics: MetricsData{Turns: 5}, + CreatedItems: []CreatedItemReport{{ + Type: "create_issue", + Timestamp: "cache-only-created-item", + }}, + FirewallAnalysis: &FirewallAnalysis{AnalysisBase: AnalysisBase{BlockedRequests: 7}}, + MCPFailures: []MCPFailureReport{{ServerName: "cache-only-mcp", Status: "failed"}}, + } + baselineRun := processedRunFromCachedData(baseline, baselineAudit, outputDir) + processedRun := processedRunFromCachedData(cached, cachedAudit, outputDir) + + logsData, err := prepareLogsData([]ProcessedRun{baselineRun, processedRun}, renderLogsOutputOptions{ + audit: true, + outputDir: outputDir, + }) + require.NoError(t, err) + require.Len(t, logsData.Runs, 2) + assert.Equal(t, filepath.Join(outputDir, "run-42", auditFileName), logsData.Runs[1].AuditPath) + + written, ok := loadCachedAuditData(processedRun.Run.LogsPath, processedRun.Run, auditCacheSourceLogs) + require.True(t, ok) + assert.Equal(t, cachedAudit.Overview, written.Overview) + assert.Equal(t, cachedAudit.CreatedItems, written.CreatedItems) + assert.Equal(t, cachedAudit.FirewallAnalysis, written.FirewallAnalysis) + assert.Equal(t, cachedAudit.MCPFailures, written.MCPFailures) + require.NotNil(t, written.Comparison) + require.NotNil(t, written.Comparison.Delta) + assert.Equal(t, AuditComparisonIntDelta{Before: 2, After: 5, Changed: true}, written.Comparison.Delta.Turns) + assert.Equal(t, AuditComparisonStringDelta{Before: "read_only", After: "write_capable", Changed: true}, written.Comparison.Delta.Posture) + assert.Equal(t, AuditComparisonIntDelta{Before: 1, After: 7, Changed: true}, written.Comparison.Delta.BlockedRequests) + require.NotNil(t, written.Comparison.Delta.MCPFailure) + assert.Equal(t, []string{"cache-only-mcp"}, written.Comparison.Delta.MCPFailure.After) } func TestBuildLogsDataPreservesCachedRunRecord(t *testing.T) { @@ -747,10 +832,11 @@ func TestBuildLogsDataPreservesCachedRunRecord(t *testing.T) { IntentionalFailure: true, } - processedRun := processedRunFromCachedData(cached) - assert.Empty(t, processedRun.Run.LogsPath) + outputDir := t.TempDir() + processedRun := processedRunFromCachedData(cached, nil, outputDir) + assert.Equal(t, filepath.Join(outputDir, "run-42"), processedRun.Run.LogsPath) - data := buildLogsData([]ProcessedRun{processedRun}, t.TempDir(), nil) + data := buildLogsData([]ProcessedRun{processedRun}, outputDir, nil) require.Equal(t, []RunData{cached}, data.Runs) assert.Equal(t, 1, data.Summary.TotalRuns) diff --git a/pkg/cli/logs_models.go b/pkg/cli/logs_models.go index ea25911917e..361fa9e7b14 100644 --- a/pkg/cli/logs_models.go +++ b/pkg/cli/logs_models.go @@ -136,6 +136,7 @@ type ProcessedRun struct { JobDetails []JobInfoWithDuration SafeOutputs []CreatedItemReport cachedData *RunData + cachedAudit *AuditData } // ReportProvenance holds the shared provenance fields common to all report record types. @@ -317,6 +318,7 @@ type DownloadResult struct { Cached bool // True if loaded from cached summary CachedRun *RunData LogsPath string + cachedAudit *AuditData storageReserved bool } diff --git a/pkg/cli/logs_orchestrator_download.go b/pkg/cli/logs_orchestrator_download.go index a6d442e83d7..64e0d94a7fe 100644 --- a/pkg/cli/logs_orchestrator_download.go +++ b/pkg/cli/logs_orchestrator_download.go @@ -209,7 +209,7 @@ func prepareLogsDownload(ctx context.Context, opts LogsDownloadOptions) (logsDow if opts.cachedJSONLCache != nil { cachedRuns = opts.cachedJSONLCache.runs } - if !cachedJSONLCanSatisfy(artifactFilter, opts.Parse, opts.Audit, opts.Train, opts.ToolGraph) { + if !cachedJSONLCanSatisfy(artifactFilter, opts.Parse, opts.Train, opts.ToolGraph) { cachedRuns = nil } if err := prepareLogsDownloadOutput(ctx, opts); err != nil { @@ -904,7 +904,7 @@ func (c *orderedLogsRunCollector) processReadyResult(index int) { } if result.CachedRun != nil { if c.opts.countLimit.tryAdd() { - c.candidates[index] = processedRunFromCachedData(*result.CachedRun) + c.candidates[index] = processedRunFromCachedData(*result.CachedRun, result.cachedAudit, c.opts.outputDir) c.accepted[index] = true c.acceptedCount++ } diff --git a/pkg/cli/logs_orchestrator_stdin.go b/pkg/cli/logs_orchestrator_stdin.go index fe306644ccb..dbc4e480296 100644 --- a/pkg/cli/logs_orchestrator_stdin.go +++ b/pkg/cli/logs_orchestrator_stdin.go @@ -46,7 +46,7 @@ func DownloadWorkflowLogsFromStdin(ctx context.Context, opts StdinLogsOptions) ( if preparedCachedJSONL.cache != nil { cachedRuns = preparedCachedJSONL.cache.runs } - if !cachedJSONLCanSatisfy(artifactFilter, opts.Parse, opts.Audit, opts.Train, opts.ToolGraph) { + if !cachedJSONLCanSatisfy(artifactFilter, opts.Parse, opts.Train, opts.ToolGraph) { cachedRuns = nil } cachedJSONLWriter := preparedCachedJSONL.writer @@ -218,7 +218,7 @@ func DownloadWorkflowLogsFromStdin(ctx context.Context, opts StdinLogsOptions) ( for _, result := range downloadResults { collectionStats.recordResult(result) if result.CachedRun != nil { - processedRuns = append(processedRuns, processedRunFromCachedData(*result.CachedRun)) + processedRuns = append(processedRuns, processedRunFromCachedData(*result.CachedRun, result.cachedAudit, opts.OutputDir)) continue } if errors.Is(result.Error, errLogsStorageLimitReached) { diff --git a/pkg/cli/logs_run_processor.go b/pkg/cli/logs_run_processor.go index 1536384a0c4..88c2299d43e 100644 --- a/pkg/cli/logs_run_processor.go +++ b/pkg/cli/logs_run_processor.go @@ -289,12 +289,14 @@ func cachedJSONDownloadResult(run WorkflowRun, cachedRuns cachedLogsRuns, filter if !ok { return DownloadResult{}, false } + cachedAudit := cachedRuns[run.DatabaseID].Audit logsOrchestratorLog.Printf("Cache hit for run %d from cached JSONL; skipping artifact download and processing", run.DatabaseID) return DownloadResult{ RunAnalysis: RunAnalysis{Run: run}, Cached: true, CachedRun: &cachedRun, LogsPath: cachedRun.LogsPath, + cachedAudit: cachedAudit, }, true }