Fix legacy daily AIC accounting gaps

actions/setup/js/check_daily_aic_workflow_guardrail.cjs

@@ -225,24 +225,37 @@
   return PRIMARY_GUARDRAIL_ARTIFACT_NAMES.some(name => artifactName === name || artifactName.endsWith(`-${name}`));
 }
 
-function provesLegacyPreHarnessFailure(logText) {
-  return AWF_STARTUP_FAILURE_MARKER.test(logText) && !ENGINE_HARNESS_MARKER.test(logText);
+function inspectLegacyAgentLog(logText) {
+  return {
+    artifactInspected: true,
+    preHarnessFailure: AWF_STARTUP_FAILURE_MARKER.test(logText) && !ENGINE_HARNESS_MARKER.test(logText),
+    sampleReplay: /"driver"\s*:\s*"apply_samples"/.test(logText),
+  };
 }
 
-async function inspectLegacyPreHarnessAgentFailure(artifactClient, artifacts, downloadRoot, token, owner, repo, run, components) {
+async function inspectLegacyAgentArtifact(artifactClient, artifacts, downloadRoot, token, owner, repo, run, components) {
   const job = components.get("agent");
-  if (!job || job.conclusion !== "failure") return false;
-  if (["agent/token_usage.jsonl", "agent_usage.jsonl", "agent_usage.json"].some(file => fs.existsSync(path.join(downloadRoot, file)))) return false;
+  const noEvidence = { artifactInspected: false, preHarnessFailure: false, sampleReplay: false };
+  if (!job) return noEvidence;
+  for (const file of ["agent/token_usage.jsonl", "agent_usage.jsonl", "agent_usage.json"]) {
+    const accountingPath = path.join(downloadRoot, file);
+    if (!fs.existsSync(accountingPath)) continue;
+    try {
+      if (fs.readFileSync(accountingPath, "utf8").trim()) return noEvidence;
+    } catch {
+      return noEvidence;
+    }
+  }
 
   const artifact = artifacts.find(item => item?.name === "agent");
   const createdAt = artifact?.createdAt?.getTime();
   const startedAt = Date.parse(job.started_at);
   const completedAt = Date.parse(job.completed_at);
-  if (!artifact?.id || artifact.expired || !Number.isFinite(createdAt)) return false;
-  if (!Number.isFinite(startedAt)) return false;
-  if (!Number.isFinite(completedAt)) return false;
+  if (!artifact?.id || artifact.expired || !Number.isFinite(createdAt)) return noEvidence;
+  if (!Number.isFinite(startedAt)) return noEvidence;
+  if (!Number.isFinite(completedAt)) return noEvidence;
   if (createdAt < startedAt || createdAt >= completedAt + 1000) {
-    return false;
+    return noEvidence;
   }
 
   const agentRoot = path.join(downloadRoot, "legacy-agent-artifact");
@@ -260,14 +273,14 @@
   try {
     stat = fs.statSync(logPath);
   } catch {
-    return false;
+    return noEvidence;
   }
-  if (!stat.isFile() || stat.size > MAX_LEGACY_AGENT_LOG_BYTES) return false;
+  if (!stat.isFile() || stat.size > MAX_LEGACY_AGENT_LOG_BYTES) return noEvidence;
 
   try {
-    return provesLegacyPreHarnessFailure(fs.readFileSync(logPath, "utf8"));
+    return inspectLegacyAgentLog(fs.readFileSync(logPath, "utf8"));
   } catch {
-    return false;
+    return noEvidence;
   }
 }
 
@@ -364,8 +377,8 @@
       usageJSONLFiles,
     });
     const artifactRoot = download.downloadPath || downloadRoot;
-    const legacyPreHarnessAgentFailure = components ? await inspectLegacyPreHarnessAgentFailure(artifactClient, artifacts, artifactRoot, token, owner, repo, run, components) : false;
-    const aic = components ? sumCoveredComponents(artifactRoot, components, artifact.createdAt.getTime(), artifacts, artifact.name, run.run_attempt, run.id, legacyPreHarnessAgentFailure) : sumAICFromUsageJSONLFiles(usageJSONLFiles);
+    const legacyAgentEvidence = components ? await inspectLegacyAgentArtifact(artifactClient, artifacts, artifactRoot, token, owner, repo, run, components) : null;
+    const aic = components ? sumCoveredComponents(artifactRoot, components, artifact.createdAt.getTime(), artifacts, artifact.name, run.run_attempt, run.id, legacyAgentEvidence) : sumAICFromUsageJSONLFiles(usageJSONLFiles);
     logDailyGuardrail("Computed run AIC from artifact", {
       runId,
       artifactId: artifact.id,
@@ -374,7 +387,7 @@
     });
     return aic;
   } finally {
     fs.rmSync(downloadRoot, { recursive: true, force: true });
   }
 }
 

actions/setup/js/daily_aic_component_coverage.cjs

@@ -94,7 +94,10 @@ function logComponentAIC(runId, name, job, aic, reason, details = {}) {
   );
 }
 
-function sumCoveredComponents(directory, components, artifactCreatedAt, artifacts, usageArtifactName, attempt, runId, legacyPreHarnessAgentFailure = false) {
+/**
+ * @param {{ artifactInspected: boolean, preHarnessFailure: boolean, sampleReplay: boolean } | null} legacyAgentEvidence
+ */
+function sumCoveredComponents(directory, components, artifactCreatedAt, artifacts, usageArtifactName, attempt, runId, legacyAgentEvidence = null) {
   let total = 0;
   for (const [name, job] of components) {
     if (job.conclusion === "skipped") {
@@ -163,15 +166,27 @@ function sumCoveredComponents(directory, components, artifactCreatedAt, artifact
       });
       continue;
     }
-    if (!selected && name === "agent" && job.conclusion === "failure" && legacyPreHarnessAgentFailure) {
+    if (!selected && name === "agent" && job.conclusion === "failure" && legacyAgentEvidence?.preHarnessFailure) {
       logComponentAIC(runId, name, job, 0, "legacy_pre_harness_failure");
       continue;
     }
+    if (!selected && name === "agent" && job.conclusion === "success" && candidateStates[0].state === "empty" && legacyAgentEvidence?.sampleReplay) {
+      logComponentAIC(runId, name, job, 0, "legacy_sample_replay", {
+        source: candidateStates[0].file,
+      });
+      continue;
+    }
     if (!selected) {
       if (provesExecutionNotStarted(directory, name, runId, job.run_attempt)) {
         logComponentAIC(runId, name, job, 0, "execution_not_started");
         continue;
       }
+      if (name === "agent" && job.conclusion === "failure" && candidateStates[0].state === "missing" && !fs.existsSync(path.join(directory, name, "execution.json")) && !legacyAgentEvidence?.artifactInspected) {
+        logComponentAIC(runId, name, job, 0, "failed_before_accounting", {
+          source: candidateStates[0].file,
+        });
+        continue;
+      }
       if (name === "evals" && job.conclusion === "failure") {
         logComponentAIC(runId, name, job, 0, "failed_before_accounting", {
           source: candidateStates[0].file,

actions/setup/js/daily_aic_component_coverage.test.cjs

@@ -310,9 +310,11 @@ it.each([null, 0])("counts missing agent accounting as zero when a failed job ne
   expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"aic":0,"reason":"runner_not_assigned"'));
 });
 
-it("still requires accounting when a failed agent job has no authoritative source", async () => {
+it("counts missing agent accounting as zero when the agent job failed", async () => {
   const f = evaluate({}, [job("agent", { conclusion: "failure" })]);
-  await expect(f.result).rejects.toThrow("Missing accounting for executed agent component");
+  await expect(f.result).resolves.toBe(0);
+  expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"aic":0,"reason":"failed_before_accounting"'));
+  expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"source":"agent/token_usage.jsonl"'));
 });
 
 it("counts legacy pre-harness agent failures as zero from the agent artifact", async () => {
@@ -335,6 +337,17 @@ it("still requires accounting when a legacy agent artifact contains a harness ma
   await expect(f.result).rejects.toThrow("Missing accounting for executed agent component");
 });
 
+it("counts a legacy successful sample replay with empty accounting as zero", async () => {
+  const f = evaluate({ "agent/token_usage.jsonl": "" }, [job("agent")], {
+    agentFiles: {
+      "agent-stdio.log": '{"type":"result","subtype":"success","terminal_reason":"completed","num_turns":1,"driver":"apply_samples"}\n',
+    },
+  });
+  await expect(f.result).resolves.toBe(0);
+  expect(f.client.downloadArtifact).toHaveBeenCalledTimes(2);
+  expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"aic":0,"reason":"legacy_sample_replay"'));
+});
+
 it("still requires accounting when an agent job succeeds", async () => {
   const f = evaluate({ "agent/token_usage.jsonl": "" }, [job("agent")]);
   await expect(f.result).rejects.toThrow("Missing accounting for executed agent component");