fix: retain generated smoke failure artifacts

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ concurrency:
 jobs:
   validate:
     runs-on: ubuntu-latest
-    timeout-minutes: 45
+    timeout-minutes: 60
 
     steps:
       - name: Checkout

scripts/create-croco-app-generated-smoke-matrix.mts

@@ -1,5 +1,13 @@
+import type {
+  SmokeCaseArtifactBundle,
+  SmokeCaseRecoverySummary,
+  SmokeFailureClassification,
+} from "./create-croco-app-generated-smoke-report.mts";
+
 export type SmokeMatrixTier = "spine-blocking" | "ecosystem-advisory";
 
+export const REST_SPA_CONTRACT_SMOKE_CASE_NAME = "rest-spa-contracts";
+
 export type SmokeMatrixStatus = "pending" | "passed" | "failed";
 
 export type AdvisorySmokeMetadata = {
@@ -15,6 +23,15 @@ export type SmokeMatrixCaseDefinition = {
 
 export type SmokeMatrixCaseState = SmokeMatrixCaseDefinition & {
   readonly status: SmokeMatrixStatus;
+  readonly failureEvidence?: SmokeMatrixCaseFailureEvidence;
+};
+
+export type SmokeMatrixCaseFailureEvidence = {
+  readonly error: string;
+  readonly diagnosticCodes: readonly string[];
+  readonly recovery: SmokeCaseRecoverySummary;
+  readonly classification: SmokeFailureClassification;
+  readonly artifactBundle?: SmokeCaseArtifactBundle;
 };
 
 export type SmokeMatrixFailure = {
@@ -164,6 +181,7 @@ export const GENERATED_SMOKE_MATRIX_CASES = [
         "CROCO_GENERATED_SMOKE_CASES=ai-saas-golden-path pnpm create-croco-app:smoke; inspect the AI full demo flow and contracts.",
     },
   },
+  { name: REST_SPA_CONTRACT_SMOKE_CASE_NAME, tier: "spine-blocking" },
 ] as const satisfies readonly SmokeMatrixCaseDefinition[];
 
 const SMOKE_MATRIX_TIERS = ["spine-blocking", "ecosystem-advisory"] as const;
@@ -276,7 +294,7 @@ export function selectGeneratedSmokeMatrixCases<T extends SmokeMatrixCaseDefinit
 
 export function createGeneratedSmokeMatrixTierReport(
   tier: SmokeMatrixTier,
-  selectedCases: readonly Pick<SmokeMatrixCaseState, "name" | "status">[],
+  selectedCases: readonly Pick<SmokeMatrixCaseState, "name" | "status" | "failureEvidence">[],
   options: {
     readonly filteredRun: boolean;
     readonly previousReport?: unknown;
@@ -287,12 +305,19 @@ export function createGeneratedSmokeMatrixTierReport(
   const previous = isGeneratedSmokeMatrixTierReport(options.previousReport, tier)
     ? new Map(options.previousReport.cases.map((smokeCase) => [smokeCase.name, smokeCase]))
     : new Map<string, SmokeMatrixCaseState>();
-  const updates = new Map(selectedCases.map((smokeCase) => [smokeCase.name, smokeCase.status]));
+  const updates = new Map(selectedCases.map((smokeCase) => [smokeCase.name, smokeCase]));
   const cases = GENERATED_SMOKE_MATRIX_CASES.filter((smokeCase) => smokeCase.tier === tier).map(
-    (definition) => ({
-      ...definition,
-      status: updates.get(definition.name) ?? previous.get(definition.name)?.status ?? "pending",
-    }),
+    (definition) => {
+      const update = updates.get(definition.name);
+      const previousCase = previous.get(definition.name);
+      const failureEvidence = update ? update.failureEvidence : previousCase?.failureEvidence;
+
+      return {
+        ...definition,
+        status: update?.status ?? previousCase?.status ?? "pending",
+        ...(failureEvidence ? { failureEvidence } : {}),
+      };
+    },
   );
   const status = options.failure
     ? "failed"
@@ -378,6 +403,12 @@ export function isGeneratedSmokeMatrixTierReport(
     if (!definition || smokeCase.tier !== tier || !isSmokeMatrixStatus(smokeCase.status)) {
       return false;
     }
+    if (
+      smokeCase.failureEvidence !== undefined &&
+      !isSmokeMatrixCaseFailureEvidence(smokeCase.failureEvidence)
+    ) {
+      return false;
+    }
     if (tier === "ecosystem-advisory") {
       if (
         !isAdvisoryMetadata(smokeCase.advisory) ||
@@ -441,6 +472,39 @@ export function renderGeneratedSmokeMatrixReport(
     "",
   );
 
+  const failedCases = report.cases.filter(
+    (
+      smokeCase,
+    ): smokeCase is SmokeMatrixCaseState & {
+      readonly failureEvidence: SmokeMatrixCaseFailureEvidence;
+    } => smokeCase.failureEvidence !== undefined,
+  );
+  if (failedCases.length > 0) {
+    lines.push("## Failed Case Recovery", "");
+    for (const smokeCase of failedCases) {
+      const { artifactBundle, classification, diagnosticCodes, error, recovery } =
+        smokeCase.failureEvidence;
+      lines.push(
+        `### ${smokeCase.name}`,
+        "",
+        `- Classification: ${classification.kind} (${escapeMarkdown(classification.reason)})`,
+        `- Rerun: \`${escapeBackticks(recovery.localRerunCommand)}\``,
+        `- Diagnostics: ${formatSmokeMatrixList(diagnosticCodes)}`,
+        `- Error: ${escapeMarkdown(error)}`,
+      );
+      if (artifactBundle) {
+        lines.push(
+          `- Artifacts: \`${escapeBackticks(artifactBundle.path)}\``,
+          `- Stdout: \`${escapeBackticks(artifactBundle.stdoutPath)}\``,
+          `- Stderr: \`${escapeBackticks(artifactBundle.stderrPath)}\``,
+          `- Output capture: ${artifactBundle.outputTruncated ? "truncated at 64 MiB" : "complete"}`,
+          ...artifactBundle.files.map((file) => `- File: \`${escapeBackticks(file)}\``),
+        );
+      }
+      lines.push("");
+    }
+  }
+
   return lines.join("\n");
 }
 
@@ -545,10 +609,48 @@ function isSmokeMatrixFailure(value: unknown): value is SmokeMatrixFailure {
   );
 }
 
+function isSmokeMatrixCaseFailureEvidence(value: unknown): value is SmokeMatrixCaseFailureEvidence {
+  return (
+    isRecord(value) &&
+    typeof value.error === "string" &&
+    Array.isArray(value.diagnosticCodes) &&
+    value.diagnosticCodes.every((code) => typeof code === "string") &&
+    isRecord(value.recovery) &&
+    typeof value.recovery.localRerunCommand === "string" &&
+    isRecord(value.classification) &&
+    (value.classification.kind === "deterministic" ||
+      value.classification.kind === "suspectedFlaky") &&
+    typeof value.classification.reason === "string" &&
+    (value.artifactBundle === undefined || isSmokeCaseArtifactBundle(value.artifactBundle))
+  );
+}
+
+function isSmokeCaseArtifactBundle(value: unknown): boolean {
+  return (
+    isRecord(value) &&
+    typeof value.path === "string" &&
+    typeof value.stdoutPath === "string" &&
+    typeof value.stderrPath === "string" &&
+    Array.isArray(value.files) &&
+    value.files.every((file) => typeof file === "string") &&
+    typeof value.outputTruncated === "boolean"
+  );
+}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null;
 }
 
 function escapeMarkdown(value: string): string {
   return value.replace(/\|/g, "\\|").replace(/\n/g, " ");
 }
+
+function escapeBackticks(value: string): string {
+  return value.replace(/`/g, "\\`");
+}
+
+function formatSmokeMatrixList(values: readonly string[]): string {
+  return values.length > 0
+    ? values.map((value) => `\`${escapeBackticks(value)}\``).join(", ")
+    : "_none_";
+}

scripts/create-croco-app-generated-smoke-report.mts

@@ -1,5 +1,6 @@
-import { copyFileSync, existsSync, mkdirSync } from "node:fs";
+import { copyFileSync, existsSync, mkdirSync, readdirSync } from "node:fs";
 import { dirname, join, relative } from "node:path";
+import { REST_SPA_CONTRACT_SMOKE_CASE_NAME } from "./create-croco-app-generated-smoke-matrix.mts";
 
 export type GeneratedSmokeArtifact = {
   readonly sourcePath: string;
@@ -28,7 +29,7 @@ export function copyGeneratedSmokeArtifacts(options: {
     return {
       sourcePath,
       reportPath,
-      reportRelativePath: normalizePath(relative(options.generatedSmokeReportDir, reportPath)),
+      reportRelativePath: toPosixPath(relative(options.generatedSmokeReportDir, reportPath)),
     };
   });
 }
@@ -45,8 +46,175 @@ export function renderGeneratedSmokeArtifacts(
     .join(", ");
 }
 
-function normalizePath(path: string): string {
-  return path.replace(/\\/g, "/");
+export type SmokeCaseRecoverySummary = {
+  readonly localRerunCommand: string;
+};
+
+export type SmokeCaseArtifactBundle = {
+  readonly path: string;
+  readonly stdoutPath: string;
+  readonly stderrPath: string;
+  readonly files: readonly string[];
+  readonly outputTruncated: boolean;
+};
+
+export type SmokeFailureClassification = {
+  readonly kind: "deterministic" | "suspectedFlaky";
+  readonly reason: string;
+};
+
+export type SmokeCommandFailureEvidence = {
+  readonly message: string;
+  readonly stdout: string;
+  readonly stderr: string;
+  readonly signal: string | null;
+};
+
+export function createSmokeRecoverySummary(caseName: string): SmokeCaseRecoverySummary {
+  return {
+    localRerunCommand: `pnpm create-croco-app:smoke ${caseName}`,
+  };
+}
+
+export function classifySmokeFailure(input: {
+  readonly message: string;
+  readonly stdout?: string;
+  readonly stderr?: string;
+  readonly signal?: string | null;
+}): SmokeFailureClassification {
+  const output = [input.message, input.stdout, input.stderr, input.signal]
+    .filter(Boolean)
+    .join("\n");
+  const matchedIndicator = [
+    /\bETIMEDOUT\b/i,
+    /\bEAI_AGAIN\b/i,
+    /\bECONNRESET\b/i,
+    /\bsocket hang up\b/i,
+    /\bnetwork timeout\b/i,
+    /\bfetch failed\b/i,
+    /\bERR_SOCKET_CONNECTION_TIMEOUT\b/i,
+    /\bERR_NETWORK\b/i,
+    /\bSIG(?:TERM|KILL)\b/i,
+  ].find((pattern) => pattern.test(output));
+
+  if (matchedIndicator) {
+    return {
+      kind: "suspectedFlaky",
+      reason: `transient failure indicator matched ${matchedIndicator.source}`,
+    };
+  }
+
+  return {
+    kind: "deterministic",
+    reason: "no transient timeout, network, DNS, socket, fetch, or termination indicator detected",
+  };
+}
+
+export function classifySmokeCommandFailure(
+  input: SmokeCommandFailureEvidence,
+): SmokeFailureClassification {
+  return classifySmokeFailure({
+    message: "",
+    stdout: input.stdout,
+    stderr: input.stderr,
+    signal: input.signal,
+  });
+}
+
+export function extractSmokeCommandDiagnosticCodes(
+  input: SmokeCommandFailureEvidence,
+): readonly string[] {
+  return extractSmokeDiagnosticCodes([input.stdout, input.stderr].join("\n"));
+}
+
+export function extractSmokeDiagnosticCodes(output: string): readonly string[] {
+  return [
+    ...new Set(output.match(/\b(?:CROCO_[A-Z0-9_]+|[a-z0-9-]+\/[a-z0-9-]+)\b/g) ?? []),
+  ].sort();
+}
+
+const ignoredArtifactDirectories = new Set([
+  ".git",
+  ".next",
+  ".output",
+  ".turbo",
+  ".wrangler",
+  "coverage",
+  "dist",
+  "node_modules",
+]);
+
+const caseSpecificSmokeFailureArtifactPaths: Readonly<Record<string, readonly string[]>> = {
+  [REST_SPA_CONTRACT_SMOKE_CASE_NAME]: ["apps/api-server/src/controllers/userSchemas.ts"],
+};
+
+export function shouldSkipSmokeArtifactDirectory(name: string): boolean {
+  return ignoredArtifactDirectories.has(name);
+}
+
+export function shouldIncludeSmokeFailureArtifact(
+  relativePath: string,
+  caseName?: string,
+): boolean {
+  const normalizedPath = toPosixPath(relativePath);
+  const segments = normalizedPath.split("/");
+  const fileName = segments.at(-1) ?? "";
+
+  if (segments.some((segment) => shouldSkipSmokeArtifactDirectory(segment))) {
+    return false;
+  }
+
+  if (
+    fileName === "package.json" ||
+    fileName === "pnpm-workspace.yaml" ||
+    fileName === "pnpm-lock.yaml" ||
+    fileName === "turbo.json" ||
+    /^croco(?:[.-].*)?\.json$/.test(fileName) ||
+    fileName === "openapi.json" ||
+    fileName === "strict-openapi-canary.json" ||
+    fileName === "contract-graph.snapshot.json" ||
+    fileName === "contract-graph.coverage.json" ||
+    /^tsconfig(?:\..+)?\.json$/.test(fileName) ||
+    /^(?:babel|next|open-next|postcss|sst|tailwind|vite)\.config\.[cm]?[jt]s$/.test(fileName) ||
+    fileName === "wrangler.toml" ||
+    /^Dockerfile(?:\..+)?$/.test(fileName)
+  ) {
+    return true;
+  }
+
+  return (
+    segments[0] === ".croco" ||
+    (caseName !== undefined &&
+      (caseSpecificSmokeFailureArtifactPaths[caseName] ?? []).includes(normalizedPath))
+  );
+}
+
+export function collectSmokeFailureArtifactFiles(projectDir: string, caseName: string): string[] {
+  return collectSmokeFailureArtifactFilesInDirectory(projectDir, projectDir, caseName);
+}
+
+function collectSmokeFailureArtifactFilesInDirectory(
+  projectDir: string,
+  directory: string,
+  caseName: string,
+): string[] {
+  return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
+    const entryPath = join(directory, entry.name);
+    const relativePath = relative(projectDir, entryPath);
+    if (entry.isDirectory()) {
+      return shouldSkipSmokeArtifactDirectory(entry.name)
+        ? []
+        : collectSmokeFailureArtifactFilesInDirectory(projectDir, entryPath, caseName);
+    }
+
+    return entry.isFile() && shouldIncludeSmokeFailureArtifact(relativePath, caseName)
+      ? [entryPath]
+      : [];
+  });
+}
+
+export function toPosixPath(value: string): string {
+  return value.replace(/\\/g, "/");
 }
 
 function escapeBackticks(value: string): string {