diff --git a/.changeset/327f9106.md b/.changeset/327f9106.md new file mode 100644 index 000000000..7f7f9635f --- /dev/null +++ b/.changeset/327f9106.md @@ -0,0 +1,6 @@ +--- +"@croco/cli": patch +--- + +- fix: preserve verification evidence after output failures +- fix: keep publish-profile CLI tests deterministic diff --git a/packages/cli/src/tests/CliTestContract.spec.ts b/packages/cli/src/tests/CliTestContract.spec.ts new file mode 100644 index 000000000..e6dfd81a6 --- /dev/null +++ b/packages/cli/src/tests/CliTestContract.spec.ts @@ -0,0 +1,13 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +describe("CLI test contract", () => { + it("keeps integration tests out of the unit test task", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../../package.json", import.meta.url), "utf8"), + ) as { readonly scripts?: Readonly> }; + + expect(packageJson.scripts?.test).toBe('vitest run --exclude "src/tests/integration/**"'); + expect(packageJson.scripts?.["test:e2e"]).toBe("vitest run src/tests/integration"); + }); +}); diff --git a/scripts/package-bin-smoke.mts b/scripts/package-bin-smoke.mts index 31dc5afaf..4a60ba707 100644 --- a/scripts/package-bin-smoke.mts +++ b/scripts/package-bin-smoke.mts @@ -3,7 +3,7 @@ import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSyn import { tmpdir } from "node:os"; import { dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { findPackageJsonFiles } from "./package-manifest-contracts.mjs"; +import { effectivePublishManifest, findPackageJsonFiles } from "./package-manifest-contracts.mjs"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -147,7 +147,7 @@ function packageIndexFor(packageJsonFiles: readonly string[]): ReadonlyMap 0; } diff --git a/scripts/package-manifest-contracts.mjs b/scripts/package-manifest-contracts.mjs index 6135dd577..b9fc98884 100644 --- a/scripts/package-manifest-contracts.mjs +++ b/scripts/package-manifest-contracts.mjs @@ -55,6 +55,15 @@ export function fieldMatchesPath(source, rootFieldName, publishFieldPath) { return valuesMatch(rootValue, publishValue); } +export function effectivePublishManifest(sourceManifest) { + const publishManifest = { + ...sourceManifest, + ...sourceManifest.publishConfig, + }; + delete publishManifest.publishConfig; + return publishManifest; +} + function valuesMatch(left, right) { if (Object.is(left, right)) { return true; diff --git a/scripts/release-spine-evidence.mts b/scripts/release-spine-evidence.mts index 0ec6744c9..31ee6a776 100644 --- a/scripts/release-spine-evidence.mts +++ b/scripts/release-spine-evidence.mts @@ -1,7 +1,18 @@ #!/usr/bin/env node import { execFileSync, spawn } from "node:child_process"; -import { cpSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { + closeSync, + cpSync, + existsSync, + mkdirSync, + openSync, + readFileSync, + rmSync, + statSync, + writeFileSync, + writeSync, +} from "node:fs"; import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; import { argv, exit } from "node:process"; import { pathToFileURL } from "node:url"; @@ -122,7 +133,9 @@ export type CommandRunResult = { readonly signal: string | null; readonly status: number | null; readonly stderr: string; + readonly stderrFileComplete?: boolean; readonly stdout: string; + readonly stdoutFileComplete?: boolean; readonly timedOut: boolean; }; @@ -131,7 +144,16 @@ export type CommandRunner = ( context: { readonly cwd: string; readonly env?: NodeJS.ProcessEnv; + readonly maxOutputBufferLength?: number; + readonly stderrPath?: string; + readonly stdoutPath?: string; readonly timeoutMs: number; + readonly writeOutput?: ( + descriptor: number, + output: Uint8Array, + offset: number, + length: number, + ) => number; }, ) => CommandRunResult | Promise; @@ -154,11 +176,16 @@ type RunOptions = Options & { readonly changedFiles?: readonly string[]; readonly clock?: Clock; readonly commands?: readonly EvidenceCommand[]; + readonly getInterruptSignal?: () => NodeJS.Signals | null; + readonly commandOutputWriter?: CommandRunnerContext["writeOutput"]; + readonly maxCommandOutputBufferLength?: number; readonly maxOutputExcerptLength?: number; readonly onCheckpoint?: (report: ReleaseSpineEvidenceReport) => void; readonly runner?: CommandRunner; }; +type CommandRunnerContext = Parameters[1]; + const systemClock: Clock = { nowIso: () => new Date().toISOString(), nowMs: () => Date.now(), @@ -200,6 +227,7 @@ function readChangedFiles( } } let activeCommandProcess: ChildProcess | null = null; +let activeInterruptKillTimer: ReturnType | null = null; export function createReleaseSpineEvidenceManifest(): readonly EvidenceCommand[] { return createVerificationManifest("spine"); @@ -382,19 +410,41 @@ function getErrorCode(error: Error | undefined): string | null { return typeof code === "string" ? code : null; } -function appendBoundedText(current: string, chunk: string): string { +function appendBoundedText( + current: string, + chunk: string, + maxLength = COMMAND_OUTPUT_MAX_BUFFER, +): string { const next = `${current}${chunk}`; - if (next.length <= COMMAND_OUTPUT_MAX_BUFFER) { + if (next.length <= maxLength) { return next; } - return next.slice(-COMMAND_OUTPUT_MAX_BUFFER); + return next.slice(-maxLength); } -function killActiveCommand(signal: NodeJS.Signals): void { - if (activeCommandProcess) { - signalCommandProcessTree(activeCommandProcess, signal); +export function interruptActiveCommand( + signal: NodeJS.Signals, + killGraceMs = COMMAND_TIMEOUT_KILL_GRACE_MS, +): void { + const child = activeCommandProcess; + if (!child) { + return; } + + signalCommandProcessTree(child, signal); + if (signal === "SIGKILL") { + return; + } + if (activeInterruptKillTimer) { + clearTimeout(activeInterruptKillTimer); + } + activeInterruptKillTimer = setTimeout(() => { + if (activeCommandProcess === child) { + signalCommandProcessTree(child, "SIGKILL"); + } + activeInterruptKillTimer = null; + }, killGraceMs); } function signalCommandProcessTree(child: ChildProcess, signal: NodeJS.Signals): void { @@ -422,7 +472,9 @@ export const defaultCommandRunner: CommandRunner = (check, context) => signal: null, status: null, stderr: "", + stderrFileComplete: false, stdout: "", + stdoutFileComplete: false, timedOut: false, }); return; @@ -435,6 +487,67 @@ export const defaultCommandRunner: CommandRunner = (check, context) => let stderr = ""; let stdout = ""; let timedOut = false; + const writeOutput = + context.writeOutput ?? + ((descriptor: number, output: Uint8Array, offset: number, length: number) => + writeSync(descriptor, output, offset, length)); + let stdoutFileComplete = context.stdoutPath === undefined; + let stderrFileComplete = context.stderrPath === undefined; + let stdoutDescriptor: number | null = null; + let stderrDescriptor: number | null = null; + const recordOutputError = (stream: "stderr" | "stdout", error: unknown) => { + if (stream === "stdout") { + stdoutFileComplete = false; + } else { + stderrFileComplete = false; + } + if (errorMessage !== null) { + return; + } + const normalizedError = error instanceof Error ? error : new Error(String(error)); + errorCode = getErrorCode(normalizedError); + errorMessage = `Failed to persist command ${stream}: ${normalizedError.message}`; + }; + const closeOutput = (stream: "stderr" | "stdout", descriptor: number | null) => { + if (descriptor === null) { + return; + } + try { + closeSync(descriptor); + } catch (error) { + recordOutputError(stream, error); + } + }; + try { + stdoutDescriptor = openCommandOutput(context.stdoutPath); + stdoutFileComplete = true; + } catch (error) { + recordOutputError("stdout", error); + } + if (errorMessage === null) { + try { + stderrDescriptor = openCommandOutput(context.stderrPath); + stderrFileComplete = true; + } catch (error) { + recordOutputError("stderr", error); + } + } + if (errorMessage !== null) { + closeOutput("stdout", stdoutDescriptor); + closeOutput("stderr", stderrDescriptor); + resolveResult({ + errorCode, + errorMessage, + signal: null, + status: null, + stderr, + stderrFileComplete, + stdout, + stdoutFileComplete, + timedOut: false, + }); + return; + } const child = spawn(command, args, { cwd: context.cwd, detached: process.platform !== "win32", @@ -463,14 +576,61 @@ export const defaultCommandRunner: CommandRunner = (check, context) => if (activeCommandProcess === child) { activeCommandProcess = null; } - resolveResult(result); + if (activeInterruptKillTimer) { + clearTimeout(activeInterruptKillTimer); + activeInterruptKillTimer = null; + } + closeOutput("stdout", stdoutDescriptor); + closeOutput("stderr", stderrDescriptor); + resolveResult({ + ...result, + errorCode: errorCode ?? result.errorCode, + errorMessage: errorMessage ?? result.errorMessage, + stderrFileComplete, + status: errorMessage ? null : result.status, + stdoutFileComplete, + }); }; child.stdout?.on("data", (chunk: unknown) => { - stdout = appendBoundedText(stdout, toText(chunk)); + const output = toText(chunk); + if (stdoutDescriptor !== null) { + try { + writeAllCommandOutput(stdoutDescriptor, output, writeOutput); + } catch (error) { + recordOutputError("stdout", error); + closeOutput("stdout", stdoutDescriptor); + stdoutDescriptor = null; + signalCommandProcessTree(child, "SIGTERM"); + if (!killTimer) { + killTimer = setTimeout( + () => signalCommandProcessTree(child, "SIGKILL"), + COMMAND_TIMEOUT_KILL_GRACE_MS, + ); + } + } + } + stdout = appendBoundedText(stdout, output, context.maxOutputBufferLength); }); child.stderr?.on("data", (chunk: unknown) => { - stderr = appendBoundedText(stderr, toText(chunk)); + const output = toText(chunk); + if (stderrDescriptor !== null) { + try { + writeAllCommandOutput(stderrDescriptor, output, writeOutput); + } catch (error) { + recordOutputError("stderr", error); + closeOutput("stderr", stderrDescriptor); + stderrDescriptor = null; + signalCommandProcessTree(child, "SIGTERM"); + if (!killTimer) { + killTimer = setTimeout( + () => signalCommandProcessTree(child, "SIGKILL"), + COMMAND_TIMEOUT_KILL_GRACE_MS, + ); + } + } + } + stderr = appendBoundedText(stderr, output, context.maxOutputBufferLength); }); child.once("error", (error) => { errorCode = getErrorCode(error); @@ -498,6 +658,32 @@ export const defaultCommandRunner: CommandRunner = (check, context) => }); }); +function openCommandOutput(path: string | undefined): number | null { + if (!path) { + return null; + } + + mkdirSync(dirname(path), { recursive: true }); + return openSync(path, "w"); +} + +function writeAllCommandOutput( + descriptor: number, + output: string, + writer: NonNullable, +): void { + const buffer = Buffer.from(output, "utf8"); + let offset = 0; + while (offset < buffer.byteLength) { + const remaining = buffer.byteLength - offset; + const written = writer(descriptor, buffer, offset, remaining); + if (!Number.isInteger(written) || written <= 0 || written > remaining) { + throw new Error(`Command output writer returned invalid byte count ${written}.`); + } + offset += written; + } +} + function outputExcerpt(value: string, maxLength: number): string { if (value.length <= maxLength) { return value; @@ -597,6 +783,72 @@ function collectArtifactReferences( return references; } +function persistFailedCommandOutput( + checkId: string, + result: CommandRunResult, + rootDir: string, + outputDir: string, + modifiedAt: string, +): readonly EvidenceArtifactReference[] { + const outputRoot = join(outputDir, RELEASE_ARTIFACT_DIRECTORY, checkId); + const streams: readonly (readonly [ + label: string, + fileName: string, + output: string, + fileComplete: boolean | undefined, + ])[] = [ + ["Command stdout", "stdout.log", result.stdout, result.stdoutFileComplete], + ["Command stderr", "stderr.log", result.stderr, result.stderrFileComplete], + ]; + + return streams.map(([label, fileName, output, fileComplete]) => { + const outputPath = join(outputRoot, fileName); + let writeError: string | null = null; + const usedFallback = fileComplete !== true || !existsSync(outputPath); + let wroteCurrentOutput = !usedFallback; + if (!wroteCurrentOutput) { + try { + mkdirSync(outputRoot, { recursive: true }); + writeFileSync(outputPath, output); + wroteCurrentOutput = true; + } catch (error) { + writeError = error instanceof Error ? error.message : String(error); + } + } + const artifactPath = relativeToRoot(rootDir, outputPath); + const exists = existsSync(outputPath); + + return { + label: usedFallback ? `${label} (bounded fallback; may be truncated)` : label, + path: artifactPath, + required: false, + copiedPath: exists && wroteCurrentOutput ? artifactPath : null, + copyError: writeError, + exists, + fresh: exists && wroteCurrentOutput, + modifiedAt, + sourcePath: artifactPath, + }; + }); +} + +function discardCommandOutput(outputDir: string, checkId: string): string | null { + const outputRoot = join(outputDir, RELEASE_ARTIFACT_DIRECTORY, checkId); + const cleanupErrors: string[] = []; + for (const fileName of ["stdout.log", "stderr.log"]) { + const outputPath = join(outputRoot, fileName); + try { + rmSync(outputPath, { force: true }); + } catch (error) { + cleanupErrors.push(`${fileName}: ${error instanceof Error ? error.message : String(error)}`); + } + } + + return cleanupErrors.length > 0 + ? `Command output cleanup failed: ${cleanupErrors.join("; ")}` + : null; +} + function assertCopiedGeneratedSmokeJourneyBundle(bundleRoot: string): void { const reportJson = JSON.parse(readFileSync(join(bundleRoot, "report.json"), "utf8")); assertGeneratedSmokeJourneyReport(reportJson); @@ -661,6 +913,21 @@ function failureReason(result: CommandRunResult, artifactReason: string | null): return null; } +function rejectedCommandRunResult(error: unknown): CommandRunResult { + const normalizedError = error instanceof Error ? error : new Error(String(error)); + return { + errorCode: getErrorCode(normalizedError), + errorMessage: normalizedError.message, + signal: null, + status: null, + stderr: "", + stderrFileComplete: false, + stdout: "", + stdoutFileComplete: false, + timedOut: false, + }; +} + function markSkippedAfterTimeout( report: ReleaseSpineEvidenceReport, startIndex: number, @@ -758,6 +1025,12 @@ export async function runReleaseSpineEvidence( checkpoint(); for (const [index, check] of commands.entries()) { + const signalBeforeCheck = options.getInterruptSignal?.() ?? null; + if (signalBeforeCheck) { + report = markReportInterrupted(report, signalBeforeCheck, clock.nowIso()); + checkpoint(); + break; + } if (check.applicable === false) { report = updateCheck(report, index, { ...report.checks[index], @@ -811,26 +1084,53 @@ export async function runReleaseSpineEvidence( PACKAGE_QUALITY_SPINE_PROMOTION_STATUS: dashboardStatus("spine-promotion"), } : baseCommandEnv; - const result = await runner(check, { - cwd: rootDir, - env: - check.id === "spine-promotion" - ? { - ...process.env, - SPINE_PROMOTION_COMMIT_SHA: report.provenance.commitSha, - SPINE_PROMOTION_RELEASE_CHECKPOINT: join(outputDir, REPORT_JSON_FILE_NAME), - SPINE_PROMOTION_RUN_ATTEMPT: report.provenance.runAttempt, - SPINE_PROMOTION_RUN_ID: report.provenance.runId, - } - : commandEnv, - timeoutMs: effectiveTimeoutMs, - }); + let result: CommandRunResult; + try { + result = await runner(check, { + cwd: rootDir, + env: + check.id === "spine-promotion" + ? { + ...process.env, + SPINE_PROMOTION_COMMIT_SHA: report.provenance.commitSha, + SPINE_PROMOTION_RELEASE_CHECKPOINT: join(outputDir, REPORT_JSON_FILE_NAME), + SPINE_PROMOTION_RUN_ATTEMPT: report.provenance.runAttempt, + SPINE_PROMOTION_RUN_ID: report.provenance.runId, + } + : commandEnv, + maxOutputBufferLength: options.maxCommandOutputBufferLength, + stderrPath: join(outputDir, RELEASE_ARTIFACT_DIRECTORY, check.id, "stderr.log"), + stdoutPath: join(outputDir, RELEASE_ARTIFACT_DIRECTORY, check.id, "stdout.log"), + timeoutMs: effectiveTimeoutMs, + writeOutput: options.commandOutputWriter, + }); + } catch (error) { + result = rejectedCommandRunResult(error); + } const completedAt = clock.nowIso(); const durationMs = Math.max(0, clock.nowMs() - startedMs); - const artifacts = collectArtifactReferences(check, rootDir, outputDir, startedMs); - const artifactReason = artifactFailureReason(artifacts); - const status = - artifactReason && result.status === 0 && !result.timedOut ? "failed" : resultStatus(result); + const commandArtifacts = collectArtifactReferences(check, rootDir, outputDir, startedMs); + const artifactReason = artifactFailureReason(commandArtifacts); + const interruptionSignal = options.getInterruptSignal?.() ?? null; + let status = interruptionSignal + ? "interrupted" + : artifactReason && result.status === 0 && !result.timedOut + ? "failed" + : resultStatus(result); + let cleanupError: string | null = null; + if (status !== "failed" && status !== "timed_out" && status !== "interrupted") { + cleanupError = discardCommandOutput(outputDir, check.id); + if (cleanupError) { + status = "failed"; + } + } + const artifacts = + status === "failed" || status === "timed_out" || status === "interrupted" + ? [ + ...commandArtifacts, + ...persistFailedCommandOutput(check.id, result, rootDir, outputDir, completedAt), + ] + : commandArtifacts; report = updateCheck(report, index, { ...report.checks[index], @@ -838,16 +1138,23 @@ export async function runReleaseSpineEvidence( completedAt, durationMs, effectiveTimeoutMs, - errorCode: result.errorCode, - errorMessage: result.errorMessage, + errorCode: cleanupError ? "COMMAND_OUTPUT_CLEANUP_FAILED" : result.errorCode, + errorMessage: cleanupError ?? result.errorMessage, exitCode: result.status, - failureReason: failureReason(result, artifactReason), - signal: result.signal, + failureReason: interruptionSignal + ? `Release spine evidence was interrupted by ${interruptionSignal}.` + : (cleanupError ?? failureReason(result, artifactReason)), + signal: interruptionSignal ?? result.signal, status, stderrExcerpt: outputExcerpt(result.stderr, maxOutputExcerptLength), stdoutExcerpt: outputExcerpt(result.stdout, maxOutputExcerptLength), }); checkpoint(); + if (interruptionSignal) { + report = markReportInterrupted(report, interruptionSignal, completedAt); + checkpoint(); + break; + } } report = finishReport(report, clock.nowIso()); @@ -1154,31 +1461,11 @@ async function main(): Promise { changedFiles: readChangedFiles(options.rootDir, options.base, options.head), head: options.head, }); - let latestReport: ReleaseSpineEvidenceReport | null = createInitialReport({ - commands, - generatedAt: systemClock.nowIso(), - outputDir: options.outputDir, - profile, - provenance: { - commitSha: process.env.GITHUB_SHA ?? readCurrentCommitOrUnknown(options.rootDir), - runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? "1", - runId: process.env.GITHUB_RUN_ID ?? systemClock.nowIso(), - }, - rootDir: options.rootDir, - totalTimeoutMs: options.totalTimeoutMs, - }); - - const checkpoint = (report: ReleaseSpineEvidenceReport) => { - latestReport = report; - }; + let interruptSignal: NodeJS.Signals | null = null; const interrupt = (signal: string) => { - killActiveCommand(signal as NodeJS.Signals); - if (latestReport) { - const interrupted = markReportInterrupted(latestReport, signal, systemClock.nowIso()); - writeReleaseSpineEvidenceReport(interrupted, options.outputDir); - } + interruptSignal = signal as NodeJS.Signals; + interruptActiveCommand(interruptSignal); console.error(`release-spine-evidence: interrupted by ${signal}`); - exit(signal === "SIGINT" ? 130 : 143); }; process.once("SIGINT", () => interrupt("SIGINT")); @@ -1187,7 +1474,7 @@ async function main(): Promise { const report = await runReleaseSpineEvidence({ ...options, commands, - onCheckpoint: checkpoint, + getInterruptSignal: () => interruptSignal, }); console.log( @@ -1201,7 +1488,15 @@ async function main(): Promise { console.error(`release-spine-evidence: check failed: ${diagnostic}`); } - exit(report.status === "passed" ? 0 : 1); + exit( + interruptSignal + ? interruptSignal === "SIGINT" + ? 130 + : 143 + : report.status === "passed" + ? 0 + : 1, + ); } if (import.meta.url === pathToFileURL(argv[1] ?? "").href) { diff --git a/scripts/tests/package-manifest-contracts.spec.ts b/scripts/tests/package-manifest-contracts.spec.ts index 230df77e6..e334d8f6d 100644 --- a/scripts/tests/package-manifest-contracts.spec.ts +++ b/scripts/tests/package-manifest-contracts.spec.ts @@ -1,8 +1,21 @@ import { describe, expect, it } from "vitest"; -import { fieldMatchesPath } from "../package-manifest-contracts.mjs"; +import { effectivePublishManifest, fieldMatchesPath } from "../package-manifest-contracts.mjs"; describe("package-manifest-contracts", () => { + it("applies publishConfig overrides to the effective publish manifest", () => { + expect( + effectivePublishManifest({ + bin: { source: "./src/cli.ts" }, + name: "@croco/example", + publishConfig: { bin: { published: "./dist/cli.js" } }, + }), + ).toEqual({ + bin: { published: "./dist/cli.js" }, + name: "@croco/example", + }); + }); + it("compares root and publish fields without object key order sensitivity", () => { const source = { exports: { diff --git a/scripts/tests/release-spine-evidence.spec.ts b/scripts/tests/release-spine-evidence.spec.ts index 802b7015e..1e6ba976a 100644 --- a/scripts/tests/release-spine-evidence.spec.ts +++ b/scripts/tests/release-spine-evidence.spec.ts @@ -6,6 +6,7 @@ import { rmSync, utimesSync, writeFileSync, + writeSync, } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; @@ -20,6 +21,7 @@ import { createReleaseSpineEvidenceManifest, defaultCommandRunner, failedCheckDiagnostics, + interruptActiveCommand, markReportInterrupted, parseArgs, runReleaseSpineEvidence, @@ -349,9 +351,14 @@ describe("release-spine-evidence.mts", () => { it("records failed command output with bounded excerpts", async () => { const repo = createTempRepo(); + const outputDir = join(repo, "ci-reports", "release"); + const outputRoot = join(outputDir, "artifacts", "release-metadata"); + mkdirSync(outputRoot, { recursive: true }); + writeFileSync(join(outputRoot, "stdout.log"), "stale stdout"); + writeFileSync(join(outputRoot, "stderr.log"), "stale stderr"); const report = await runReleaseSpineEvidence({ rootDir: repo, - outputDir: join(repo, "ci-reports", "release"), + outputDir, totalTimeoutMs: 1_000, commands: [createCommand("release-metadata")], maxOutputExcerptLength: 12, @@ -371,6 +378,168 @@ describe("release-spine-evidence.mts", () => { expect(report.checks[0]?.stdoutExcerpt).toContain("diagnostics"); expect(report.checks[0]?.stderrExcerpt).toContain("[truncated"); expect(report.checks[0]?.stderrExcerpt).toContain("failed"); + expect( + readFileSync(join(outputDir, "artifacts", "release-metadata", "stdout.log"), "utf8"), + ).toBe("stdout: release metadata diagnostics"); + expect( + readFileSync(join(outputDir, "artifacts", "release-metadata", "stderr.log"), "utf8"), + ).toBe("stderr: release metadata failed"); + expect(report.checks[0]?.artifacts.map(({ copiedPath }) => copiedPath)).toEqual([ + "ci-reports/release/artifacts/release-metadata/stdout.log", + "ci-reports/release/artifacts/release-metadata/stderr.log", + ]); + expect(readFileSync(join(outputDir, "spine-evidence.md"), "utf8")).toContain( + "artifacts/release-metadata/stdout.log", + ); + }); + + it("contains a rejected runner and continues writing the final report", async () => { + const repo = createTempRepo(); + const outputDir = join(repo, "ci-reports", "release"); + const report = await runReleaseSpineEvidence({ + rootDir: repo, + outputDir, + totalTimeoutMs: 1_000, + commands: [createCommand("rejected"), createCommand("continued")], + runner: (check) => { + if (check.id === "rejected") { + throw new Error("runner rejected"); + } + return okResult("continued"); + }, + }); + + expect(report.status).toBe("failed"); + expect(report.checks.map(({ status }) => status)).toEqual(["failed", "passed"]); + expect(report.checks[0]?.errorMessage).toBe("runner rejected"); + expect(readJson(join(outputDir, "spine-evidence.json"))).toEqual( + report, + ); + }); + + it("preserves complete real command output beyond the in-memory buffer", async () => { + const repo = createTempRepo(); + const outputDir = join(repo, "ci-reports", "release"); + const stdout = "full stdout survives the bounded buffer"; + const stderr = "full stderr survives the bounded buffer"; + const report = await runReleaseSpineEvidence({ + rootDir: repo, + outputDir, + totalTimeoutMs: 10_000, + commands: [ + createCommand("real-failure", { + command: [ + process.execPath, + "-e", + `process.stdout.write(${JSON.stringify(stdout)}); process.stderr.write(${JSON.stringify(stderr)}); process.exit(7)`, + ], + timeoutMs: 5_000, + }), + ], + maxCommandOutputBufferLength: 8, + }); + + expect(report.status).toBe("failed"); + expect(report.checks[0]?.stdoutExcerpt).toBe(stdout.slice(-8)); + expect(report.checks[0]?.stderrExcerpt).toBe(stderr.slice(-8)); + expect(readFileSync(join(outputDir, "artifacts", "real-failure", "stdout.log"), "utf8")).toBe( + stdout, + ); + expect(readFileSync(join(outputDir, "artifacts", "real-failure", "stderr.log"), "utf8")).toBe( + stderr, + ); + expect(report.checks[0]?.artifacts.map(({ copiedPath }) => copiedPath)).toEqual([ + "ci-reports/release/artifacts/real-failure/stdout.log", + "ci-reports/release/artifacts/real-failure/stderr.log", + ]); + }); + + it("removes streamed command output after a successful check", async () => { + const repo = createTempRepo(); + const outputDir = join(repo, "ci-reports", "release"); + const report = await runReleaseSpineEvidence({ + rootDir: repo, + outputDir, + totalTimeoutMs: 10_000, + commands: [ + createCommand("real-success", { + command: [process.execPath, "-e", "console.log('success')"], + timeoutMs: 5_000, + }), + ], + }); + + expect(report.status).toBe("passed"); + expect(existsSync(join(outputDir, "artifacts", "real-success", "stdout.log"))).toBe(false); + expect(existsSync(join(outputDir, "artifacts", "real-success", "stderr.log"))).toBe(false); + }); + + it("records command-output cleanup failures without aborting the final report", async () => { + const repo = createTempRepo(); + const outputDir = join(repo, "ci-reports", "release"); + const report = await runReleaseSpineEvidence({ + rootDir: repo, + outputDir, + totalTimeoutMs: 1_000, + commands: [createCommand("cleanup-failure")], + runner: (_check, context) => { + if (!context.stdoutPath) { + throw new Error("stdout path is required"); + } + mkdirSync(context.stdoutPath, { recursive: true }); + writeFileSync(join(context.stdoutPath, "retained.log"), "cleanup blocked"); + return okResult("ok"); + }, + }); + + expect(report.status).toBe("failed"); + expect(report.checks[0]?.status).toBe("failed"); + expect(report.checks[0]?.errorCode).toBe("COMMAND_OUTPUT_CLEANUP_FAILED"); + expect(report.checks[0]?.failureReason).toContain("stdout.log"); + expect(report.checks[0]?.artifacts.map(({ label }) => label)).toEqual([ + "Command stdout (bounded fallback; may be truncated)", + "Command stderr (bounded fallback; may be truncated)", + ]); + expect(existsSync(join(outputDir, "spine-evidence.json"))).toBe(true); + }); + + it("finalizes real command evidence before reporting an interruption", async () => { + const repo = createTempRepo(); + const outputDir = join(repo, "ci-reports", "release"); + const readyPath = join(repo, "interrupted-command.ready"); + let interruptSignal: NodeJS.Signals | null = null; + const reportPromise = runReleaseSpineEvidence({ + rootDir: repo, + outputDir, + totalTimeoutMs: 15_000, + commands: [ + createCommand("interrupted-command", { + command: [ + process.execPath, + "-e", + `process.on("SIGTERM", () => undefined); process.stdout.write("partial output"); require("node:fs").writeFileSync(${JSON.stringify(readyPath)}, "ready"); setInterval(() => undefined, 10_000)`, + ], + timeoutMs: 10_000, + }), + createCommand("not-started"), + ], + getInterruptSignal: () => interruptSignal, + }); + await waitForPath(readyPath); + interruptSignal = "SIGTERM"; + interruptActiveCommand(interruptSignal, 50); + const report = await reportPromise; + + expect(report.status).toBe("interrupted"); + expect(report.checks.map(({ status }) => status)).toEqual(["interrupted", "interrupted"]); + expect(report.checks[0]?.signal).toBe("SIGTERM"); + expect( + readFileSync(join(outputDir, "artifacts", "interrupted-command", "stdout.log"), "utf8"), + ).toBe("partial output"); + expect(report.checks[0]?.artifacts.map(({ copiedPath }) => copiedPath)).toEqual([ + "ci-reports/release/artifacts/interrupted-command/stdout.log", + "ci-reports/release/artifacts/interrupted-command/stderr.log", + ]); }); it("fails spine and publish execution on the same broken shared command ID", async () => { @@ -511,6 +680,8 @@ describe("release-spine-evidence.mts", () => { it("runs real commands through the default async runner", async () => { const repo = createTempRepo(); + const stdoutPath = join(repo, "command-output", "stdout.log"); + const stderrPath = join(repo, "command-output", "stderr.log"); const result = await defaultCommandRunner( { @@ -522,35 +693,178 @@ describe("release-spine-evidence.mts", () => { }, { cwd: repo, + stderrPath, + stdoutPath, timeoutMs: 1_000, }, ); expect(result.status).toBe(0); expect(result.stdout).toContain("runner ok"); + expect(readFileSync(stdoutPath, "utf8")).toBe("runner ok\n"); + expect(readFileSync(stderrPath, "utf8")).toBe(""); expect(result.timedOut).toBe(false); }); + it("reports command output open failures without rejecting", async () => { + const repo = createTempRepo(); + const blockedOutputRoot = join(repo, "blocked-output"); + writeFileSync(blockedOutputRoot, "not a directory"); + + const result = await defaultCommandRunner( + { + id: "blocked-output", + label: "Blocked output", + category: "quality", + command: [process.execPath, "-e", "console.log('not started')"], + timeoutMs: 1_000, + }, + { + cwd: repo, + stdoutPath: join(blockedOutputRoot, "stdout.log"), + timeoutMs: 1_000, + }, + ); + + expect(result.status).toBeNull(); + expect(result.errorMessage).toContain("Failed to persist command stdout"); + expect(result.stdoutFileComplete).toBe(false); + }); + + it("reports command output write failures without throwing", async () => { + const repo = createTempRepo(); + const outputRoot = join(repo, "command-output"); + const outputError = Object.assign(new Error("disk full"), { code: "ENOSPC" }); + + const result = await defaultCommandRunner( + { + id: "failed-output-write", + label: "Failed output write", + category: "quality", + command: [process.execPath, "-e", "process.stdout.write('diagnostics')"], + timeoutMs: 1_000, + }, + { + cwd: repo, + stderrPath: join(outputRoot, "stderr.log"), + stdoutPath: join(outputRoot, "stdout.log"), + timeoutMs: 1_000, + writeOutput: () => { + throw outputError; + }, + }, + ); + + expect(result.status).toBeNull(); + expect(result.errorCode).toBe("ENOSPC"); + expect(result.errorMessage).toContain("Failed to persist command stdout: disk full"); + expect(result.stdoutFileComplete).toBe(false); + }); + + it("keeps the complete sibling stream when one output stream fails", async () => { + const repo = createTempRepo(); + const outputDir = join(repo, "ci-reports", "release"); + const stdout = "o".repeat(100); + const stderr = "s".repeat(100); + const report = await runReleaseSpineEvidence({ + rootDir: repo, + outputDir, + totalTimeoutMs: 5_000, + commands: [ + createCommand("partial-output-failure", { + command: [ + process.execPath, + "-e", + `process.stderr.write(${JSON.stringify(stderr)}); process.stdout.write(${JSON.stringify(stdout)})`, + ], + timeoutMs: 2_000, + }), + ], + commandOutputWriter: (descriptor, output, offset, length) => { + const text = Buffer.from(output) + .subarray(offset, offset + length) + .toString("utf8"); + if (/^o+$/.test(text)) { + throw Object.assign(new Error("stdout disk full"), { code: "ENOSPC" }); + } + return writeSync(descriptor, output, offset, length); + }, + maxCommandOutputBufferLength: 8, + }); + + expect(report.status).toBe("failed"); + expect( + readFileSync(join(outputDir, "artifacts", "partial-output-failure", "stdout.log"), "utf8"), + ).toBe(stdout.slice(-8)); + expect( + readFileSync(join(outputDir, "artifacts", "partial-output-failure", "stderr.log"), "utf8"), + ).toBe(stderr); + expect(report.checks[0]?.artifacts.map(({ fresh }) => fresh)).toEqual([true, true]); + expect(report.checks[0]?.artifacts.map(({ label }) => label)).toEqual([ + "Command stdout (bounded fallback; may be truncated)", + "Command stderr", + ]); + }); + + it("retries short command-output writes until the full buffer is persisted", async () => { + const repo = createTempRepo(); + const stdoutPath = join(repo, "command-output", "stdout.log"); + let writeCalls = 0; + const result = await defaultCommandRunner( + { + id: "short-output-write", + label: "Short output write", + category: "quality", + command: [process.execPath, "-e", "process.stdout.write('complete diagnostics')"], + timeoutMs: 1_000, + }, + { + cwd: repo, + stdoutPath, + timeoutMs: 1_000, + writeOutput: (descriptor, output, offset) => { + writeCalls++; + return writeSync(descriptor, output, offset, 1); + }, + }, + ); + + expect(result.status).toBe(0); + expect(result.stdoutFileComplete).toBe(true); + expect(writeCalls).toBeGreaterThan(1); + expect(readFileSync(stdoutPath, "utf8")).toBe("complete diagnostics"); + }); + it("times out real commands through the default async runner", async () => { const repo = createTempRepo(); + const stdoutPath = join(repo, "command-output", "stdout.log"); + const stderrPath = join(repo, "command-output", "stderr.log"); const result = await defaultCommandRunner( { id: "slow-node", label: "Slow Node", category: "quality", - command: [process.execPath, "-e", "setTimeout(() => undefined, 10_000)"], - timeoutMs: 50, + command: [ + process.execPath, + "-e", + "process.stdout.write('before timeout'); setTimeout(() => undefined, 10_000)", + ], + timeoutMs: 200, }, { cwd: repo, - timeoutMs: 50, + stderrPath, + stdoutPath, + timeoutMs: 200, }, ); expect(result.status).toBeNull(); expect(result.signal).toBe("SIGTERM"); expect(result.timedOut).toBe(true); + expect(readFileSync(stdoutPath, "utf8")).toBe("before timeout"); + expect(readFileSync(stderrPath, "utf8")).toBe(""); }); it("parses root, output, and timeout options after the pnpm separator", () => { @@ -672,6 +986,7 @@ function createCommand( id: string, options: { readonly artifacts?: readonly EvidenceArtifactExpectation[]; + readonly command?: readonly string[]; readonly timeoutMs?: number; } = {}, ): EvidenceCommand { @@ -679,7 +994,7 @@ function createCommand( id, label: id, category: "quality", - command: ["fake", id], + command: options.command ?? ["fake", id], timeoutMs: options.timeoutMs ?? 100, artifacts: options.artifacts, }; @@ -701,6 +1016,16 @@ function readJson(path: string): T { return JSON.parse(readFileSync(path, "utf-8")) as T; } +async function waitForPath(path: string, timeoutMs = 10_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!existsSync(path)) { + if (Date.now() >= deadline) { + throw new Error(`Timed out waiting for ${path}`); + } + await new Promise((resolvePromise) => setTimeout(resolvePromise, 10)); + } +} + function findCheck(manifest: readonly EvidenceCommand[], id: string): EvidenceCommand { const check = manifest.find((entry) => entry.id === id); if (!check) { diff --git a/scripts/tests/verification-manifest.spec.ts b/scripts/tests/verification-manifest.spec.ts index 2979ec8b0..a1377132e 100644 --- a/scripts/tests/verification-manifest.spec.ts +++ b/scripts/tests/verification-manifest.spec.ts @@ -16,6 +16,7 @@ import { RELEASE_GATE_TEST_PATHS, RELEASE_GATE_WORKFLOW_PATHS, } from "../release-gate-maintenance.mts"; +import { effectivePublishManifest, findPackageJsonFiles } from "../package-manifest-contracts.mjs"; import type { EvidenceCommand } from "../release-spine-evidence.mts"; const ROOT_DIR = resolve(__dirname, "../.."); @@ -269,17 +270,58 @@ describe("verification manifest", () => { ); }); - it("builds every binary package before scoped package binary smoke", () => { - const manifest = createVerificationManifest("publish", { - base: "origin/trunk", - changedFiles: ["packages/cli/src/index.ts"], - head: "HEAD", - }); - const buildCommand = manifest.find(({ id }) => id === "build")?.command; - - expect(buildCommand).toContain("--filter=@croco/cli"); - expect(buildCommand).toContain("--filter=create-croco-app"); - expect(manifest.find(({ id }) => id === "package-bins-smoke")?.applicable).toBe(true); + it("builds every package binary before a change-scoped binary smoke", () => { + const binPackages = findPackageJsonFiles(resolve(ROOT_DIR, "packages")) + .map((packagePath) => { + const sourcePkg = JSON.parse(readFileSync(packagePath, "utf8")) as { + readonly name?: string; + readonly private?: boolean; + readonly publishConfig?: Record; + }; + const publishManifest = effectivePublishManifest(sourcePkg) as { + readonly bin?: unknown; + }; + return { + bin: publishManifest.bin, + name: sourcePkg.name, + packagePath, + private: sourcePkg.private, + }; + }) + .filter( + ( + entry, + ): entry is { + readonly bin: unknown; + readonly name: string; + readonly packagePath: string; + readonly private?: boolean; + } => entry.private !== true && entry.bin !== undefined && typeof entry.name === "string", + ); + const expectedBinFilters = binPackages.map(({ name }) => `--filter=${name}`).sort(); + + for (const { packagePath } of binPackages) { + const manifest = createVerificationManifest("publish", { + base: "origin/trunk", + changedFiles: [`${relative(ROOT_DIR, dirname(packagePath))}/src/index.ts`], + head: "HEAD", + }); + const packageBuildFilters = [ + ...new Set( + manifest + .find(({ id }) => id === "build") + ?.command.filter( + (argument) => + argument.startsWith("--filter=") && + argument !== "--filter=...[origin/trunk]" && + argument !== "--filter=!@croco/docs", + ), + ), + ].sort(); + + expect(packageBuildFilters).toEqual(expectedBinFilters); + expect(manifest.find(({ id }) => id === "package-bins-smoke")?.applicable).toBe(true); + } }); it("keeps the release-gate inventory complete, sorted, and executable from one root alias", () => { diff --git a/scripts/verification-manifest.mts b/scripts/verification-manifest.mts index 533163032..2105ae2d1 100644 --- a/scripts/verification-manifest.mts +++ b/scripts/verification-manifest.mts @@ -55,6 +55,14 @@ const CORE_COVERAGE_PACKAGE_DIRECTORIES = CORE_COVERAGE_PACKAGES.map((packageNam packageName.startsWith("@croco/") ? packageName.slice("@croco/".length) : packageName, ); +const PACKAGE_BIN_BUILD_FILTERS = [ + "@croco/cli", + "create-croco-app", + "@croco/openapi-spec", + "@croco/migration-runner", + "@croco/rpc-codegen", +] as const; + function isApplicableToChangedFiles( context: VerificationContext, predicate: (path: string) => boolean, @@ -88,7 +96,9 @@ function affectsPackageEntrypoints(path: string): boolean { function affectsPackageBins(path: string): boolean { return ( /^(?:package\.json|pnpm-lock\.yaml|pnpm-workspace\.yaml|turbo\.json|\.nvmrc)$/.test(path) || - /^packages\/(?:cli|create-croco-app)\/(?:package\.json|src\/)/.test(path) || + /^packages\/(?:cli|create-croco-app|migration-runner|openapi-spec|rpc-codegen)\/(?:package\.json|src\/)/.test( + path, + ) || path === "scripts/package-bin-smoke.mts" ); } @@ -406,8 +416,10 @@ const spineOnly = (context: VerificationContext): readonly EvidenceCommand[] => changeScoped && scaffoldApplicable ? ["--filter=create-croco-app"] : []; const entrypointsApplicable = isApplicableToChangedFiles(context, affectsPackageEntrypoints); const binsApplicable = isApplicableToChangedFiles(context, affectsPackageBins); - const binBuildArguments = - changeScoped && binsApplicable ? ["--filter=@croco/cli", "--filter=create-croco-app"] : []; + const packageBinBuildArguments = + changeScoped && binsApplicable + ? PACKAGE_BIN_BUILD_FILTERS.map((packageName) => `--filter=${packageName}`) + : []; const cliApplicable = isApplicableToChangedFiles(context, affectsCli); const coreCoverageApplicable = isApplicableToChangedFiles(context, affectsCoreCoverage); const packageGraphApplicable = isApplicableToChangedFiles(context, affectsPackageGraph); @@ -424,7 +436,7 @@ const spineOnly = (context: VerificationContext): readonly EvidenceCommand[] => "build", ...affectedArguments, ...scaffoldBuildArguments, - ...binBuildArguments, + ...packageBinBuildArguments, "--summarize", "--continue=always", ],