feat: emit unified executable test evidence

.changeset/unified-test-evidence.md

@@ -0,0 +1,6 @@
+---
+"@croco/testing": minor
+"@croco/problems-core": patch
+---
+
+Publish a runner-neutral `croco.test-evidence/v1` contract, JSON schema, Vitest and Playwright reporters, fidelity validation, flaky-attempt preservation, redaction, and deterministic JSON/Markdown bundles with explicit missing-artifact failures.

.github/workflows/ci.yml

@@ -171,6 +171,7 @@ jobs:
         id: verification_profile
         shell: bash
         env:
+          CROCO_TEST_EVIDENCE_DIR: ${{ github.workspace }}/ci-reports/test-evidence/records
           VERIFICATION_BASE: ${{ needs.changes.outputs.base }}
           VERIFICATION_PROFILE: ${{ needs.changes.outputs.profile }}
         run: |
@@ -199,6 +200,48 @@ jobs:
             echo "Verification report was not generated." >> "$GITHUB_STEP_SUMMARY"
           fi
 
+      - name: Ensure native Vitest evidence
+        if: always()
+        shell: bash
+        env:
+          CROCO_TEST_EVIDENCE_DIR: ${{ github.workspace }}/ci-reports/test-evidence/records
+        run: |
+          if ! find ci-reports/test-evidence/records -type f -name 'vitest-*.json' -print -quit 2>/dev/null | grep -q .; then
+            pnpm --filter @croco/testing test
+          fi
+
+      - name: Aggregate executable test evidence
+        if: always()
+        shell: bash
+        env:
+          VERIFICATION_PROFILE: ${{ needs.changes.outputs.profile }}
+        run: |
+          report="ci-reports/verification/${VERIFICATION_PROFILE}/spine-evidence.json"
+          if [ "$VERIFICATION_PROFILE" = "spine" ]; then
+            report="ci-reports/release/spine-evidence.json"
+          fi
+          evidence_args=(--input "$report")
+          vitest_record_count=0
+          if [ -d ci-reports/test-evidence/records ]; then
+            while IFS= read -r evidence_record; do
+              evidence_args+=(--input "$evidence_record")
+              if [[ "$evidence_record" == */vitest-*.json ]]; then
+                vitest_record_count=$((vitest_record_count + 1))
+              fi
+            done < <(find ci-reports/test-evidence/records -type f -name '*.json' -print | sort)
+          fi
+          if [ "$vitest_record_count" -eq 0 ]; then
+            evidence_args+=(--input ci-reports/test-evidence/records/required-vitest-record.json)
+          fi
+          evidence_status=0
+          node --experimental-strip-types scripts/test-evidence-bundle.mts "${evidence_args[@]}" || evidence_status=$?
+          if [ -f ci-reports/test-evidence/summary.md ]; then
+            cat ci-reports/test-evidence/summary.md >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "Test evidence summary was not generated." >> "$GITHUB_STEP_SUMMARY"
+          fi
+          exit "$evidence_status"
+
       - name: Upload verification evidence
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -207,6 +250,7 @@ jobs:
           path: |
             ci-reports/verification
             ci-reports/release
+            ci-reports/test-evidence
           include-hidden-files: true
           if-no-files-found: warn
 

docs/problem-code-registry.json

@@ -1,6 +1,6 @@
 {
   "version": "croco.problem-code-registry.v1",
-  "problemCount": 560,
+  "problemCount": 562,
   "problems": [
     {
       "code": "ACCESS_DENIED",
@@ -4382,6 +4382,66 @@
         }
       ]
     },
+    {
+      "code": "CROCO_TEST_EVIDENCE_CONTRACT_INVALID",
+      "category": "ValidationError",
+      "status": 422,
+      "title": "Validation Error",
+      "cookbookPath": "/reference/problem-recovery-cookbook/#croco-test-evidence-contract-invalid",
+      "recovery": {
+        "cause": "The request or generated contract failed schema or semantic validation.",
+        "userAction": "Fix the invalid fields and retry with schema-conformant input.",
+        "operatorAction": "Inspect schema diagnostics, generated contracts, and validation metadata.",
+        "retryability": "not-retryable",
+        "redactionPolicy": "public",
+        "telemetry": {
+          "eventName": "croco.problem.info",
+          "severity": "info",
+          "attributes": ["problem.code", "problem.category", "problem.status"]
+        }
+      },
+      "lifecycle": {
+        "status": "active"
+      },
+      "sources": [
+        {
+          "file": "packages/testing/src/libs/test-evidence.mts",
+          "line": 159,
+          "column": 5,
+          "kind": "problem-constructor"
+        }
+      ]
+    },
+    {
+      "code": "CROCO_TEST_EVIDENCE_FIDELITY_UNSATISFIED",
+      "category": "ValidationError",
+      "status": 422,
+      "title": "Validation Error",
+      "cookbookPath": "/reference/problem-recovery-cookbook/#croco-test-evidence-fidelity-unsatisfied",
+      "recovery": {
+        "cause": "The request or generated contract failed schema or semantic validation.",
+        "userAction": "Fix the invalid fields and retry with schema-conformant input.",
+        "operatorAction": "Inspect schema diagnostics, generated contracts, and validation metadata.",
+        "retryability": "not-retryable",
+        "redactionPolicy": "public",
+        "telemetry": {
+          "eventName": "croco.problem.info",
+          "severity": "info",
+          "attributes": ["problem.code", "problem.category", "problem.status"]
+        }
+      },
+      "lifecycle": {
+        "status": "active"
+      },
+      "sources": [
+        {
+          "file": "packages/testing/src/libs/test-evidence.mts",
+          "line": 173,
+          "column": 5,
+          "kind": "problem-constructor"
+        }
+      ]
+    },
     {
       "code": "dataloader-core/batch-result-length-mismatch",
       "category": "InternalServerError",

package.json

@@ -29,6 +29,7 @@
     "release": "pnpm build && changeset publish",
     "test": "turbo test",
     "test:release-gates": "node --experimental-strip-types scripts/verification-command.mts --id release-gate-tests",
+    "test-evidence:bundle": "node --experimental-strip-types scripts/test-evidence-bundle.mts",
     "test:coverage:core": "pnpm --filter @croco/problems-core --filter @croco/telemetry-api --filter @croco/tenant-core --filter @croco/dataloader-core --filter @croco/ratelimit-core --filter @croco/llm-core --filter @croco/storage-core --filter @croco/architecture-policy build && node --experimental-strip-types scripts/verification-command.mts --id core-coverage",
     "test:coverage:core:warning": "node --experimental-strip-types scripts/verification-command.mts --id core-coverage-warning",
     "typecheck": "turbo typecheck",

packages/docs/src/content/docs/api/problems-core/src/classes/Problem.md

@@ -382,6 +382,8 @@ RFC 7807 Problem Details를 표현하는 기본 추상 에러 클래스입니다
 - [`TelemetryForceFlushUnsupportedProblem`](/api/telemetry-sdk-node/src/classes/telemetryforceflushunsupportedproblem/)
 - [`UnsupportedTelemetrySignalProblem`](/api/telemetry-sdk-node/src/classes/unsupportedtelemetrysignalproblem/)
 - [`TelemetryAutoInstrumentationProblem`](/api/telemetry-sdk-node/src/classes/telemetryautoinstrumentationproblem/)
+- [`TestEvidenceContractError`](/api/testing/src/classes/testevidencecontracterror/)
+- [`TestEvidenceFidelityError`](/api/testing/src/classes/testevidencefidelityerror/)
 - [`TestKernelDisposedProblem`](/api/testing/src/classes/testkerneldisposedproblem/)
 - [`TestKernelDisposalProblem`](/api/testing/src/classes/testkerneldisposalproblem/)
 - [`TestKernelResourceFidelityProblem`](/api/testing/src/classes/testkernelresourcefidelityproblem/)

packages/docs/src/content/docs/api/testing/src/classes/CrocoPlaywrightEvidenceReporter.md

@@ -0,0 +1,52 @@
+---
+editUrl: false
+next: false
+prev: false
+title: "CrocoPlaywrightEvidenceReporter"
+---
+
+## Constructors
+
+### Constructor
+
+> **new CrocoPlaywrightEvidenceReporter**(`options`): `CrocoPlaywrightEvidenceReporter`
+
+#### Parameters
+
+##### options
+
+[`TestEvidenceReporterOptions`](/api/testing/src/type-aliases/testevidencereporteroptions/)
+
+#### Returns
+
+`CrocoPlaywrightEvidenceReporter`
+
+## Methods
+
+### onEnd()
+
+> **onEnd**(): `Promise`\<`void`\>
+
+#### Returns
+
+`Promise`\<`void`\>
+
+***
+
+### onTestEnd()
+
+> **onTestEnd**(`test`, `result`): `void`
+
+#### Parameters
+
+##### test
+
+[`PlaywrightTestCase`](/api/testing/src/type-aliases/playwrighttestcase/)
+
+##### result
+
+[`PlaywrightTestResult`](/api/testing/src/type-aliases/playwrighttestresult/)
+
+#### Returns
+
+`void`

packages/docs/src/content/docs/api/testing/src/classes/CrocoVitestEvidenceReporter.md

@@ -0,0 +1,38 @@
+---
+editUrl: false
+next: false
+prev: false
+title: "CrocoVitestEvidenceReporter"
+---
+
+## Constructors
+
+### Constructor
+
+> **new CrocoVitestEvidenceReporter**(`options`): `CrocoVitestEvidenceReporter`
+
+#### Parameters
+
+##### options
+
+[`TestEvidenceReporterOptions`](/api/testing/src/type-aliases/testevidencereporteroptions/)
+
+#### Returns
+
+`CrocoVitestEvidenceReporter`
+
+## Methods
+
+### onTestCaseResult()
+
+> **onTestCaseResult**(`task`): `Promise`\<`void`\>
+
+#### Parameters
+
+##### task
+
+[`VitestTask`](/api/testing/src/type-aliases/vitesttask/)
+
+#### Returns
+
+`Promise`\<`void`\>