diff --git a/.changeset/deterministic-test-runtime-controls.md b/.changeset/deterministic-test-runtime-controls.md
new file mode 100644
index 000000000..f11d470de
--- /dev/null
+++ b/.changeset/deterministic-test-runtime-controls.md
@@ -0,0 +1,8 @@
+---
+"@croco/ratelimit-core": minor
+"@croco/tasks-core": minor
+"@croco/testing": minor
+"@croco/problems-core": minor
+---
+
+Provide scoped TestKernel virtual time, seeded IDs and random values, environment snapshots, outbound-call denial, replay metadata, and deterministic leak diagnostics. Retry backoff, task timeouts, and in-memory rate-limit stores can now consume injected time and randomness without global timer or random patches.
diff --git a/docs/problem-code-registry.json b/docs/problem-code-registry.json
index 94f1dddfd..9dbdfd553 100644
--- a/docs/problem-code-registry.json
+++ b/docs/problem-code-registry.json
@@ -1,6 +1,6 @@
{
"version": "croco.problem-code-registry.v1",
- "problemCount": 571,
+ "problemCount": 576,
"problems": [
{
"code": "ACCESS_DENIED",
@@ -11966,7 +11966,7 @@
"sources": [
{
"file": "packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts",
- "line": 4,
+ "line": 6,
"column": 3,
"kind": "problem-class"
}
@@ -11996,7 +11996,7 @@
"sources": [
{
"file": "packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts",
- "line": 25,
+ "line": 43,
"column": 5,
"kind": "problem-constructor"
}
@@ -12026,7 +12026,7 @@
"sources": [
{
"file": "packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts",
- "line": 14,
+ "line": 16,
"column": 3,
"kind": "problem-class"
}
@@ -12092,6 +12092,36 @@
}
]
},
+ {
+ "code": "ratelimit/prune-interval",
+ "category": "ValidationError",
+ "status": 422,
+ "title": "Validation Error",
+ "cookbookPath": "/reference/problem-recovery-cookbook/#ratelimit-prune-interval",
+ "recovery": {
+ "cause": "The request or generated contract failed schema or semantic validation.",
+ "userAction": "Fix the invalid fields and retry with schema-conformant input.",
+ "operatorAction": "Inspect schema diagnostics, generated contracts, and validation metadata.",
+ "retryability": "not-retryable",
+ "redactionPolicy": "public",
+ "telemetry": {
+ "eventName": "croco.problem.info",
+ "severity": "info",
+ "attributes": ["problem.code", "problem.category", "problem.status"]
+ }
+ },
+ "lifecycle": {
+ "status": "active"
+ },
+ "sources": [
+ {
+ "file": "packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts",
+ "line": 27,
+ "column": 5,
+ "kind": "problem-constructor"
+ }
+ ]
+ },
{
"code": "repository-core/batch-loader-factory-not-registered",
"category": "InternalServerError",
@@ -14636,7 +14666,7 @@
"sources": [
{
"file": "packages/testing/src/libs/TestKernel.ts",
- "line": 144,
+ "line": 189,
"column": 5,
"kind": "problem-constructor"
}
@@ -14666,7 +14696,67 @@
"sources": [
{
"file": "packages/testing/src/libs/TestKernel.ts",
- "line": 172,
+ "line": 217,
+ "column": 5,
+ "kind": "problem-constructor"
+ }
+ ]
+ },
+ {
+ "code": "testing/test-kernel-leak",
+ "category": "InternalServerError",
+ "status": 500,
+ "title": "Internal Server Error",
+ "cookbookPath": "/reference/problem-recovery-cookbook/#testing-test-kernel-leak",
+ "recovery": {
+ "cause": "Croco or an upstream dependency failed after accepting the request.",
+ "userAction": "Retry later only when the operation is idempotent or the caller owns retry safety.",
+ "operatorAction": "Use traces, logs, and upstream diagnostics to isolate the failing boundary.",
+ "retryability": "conditional",
+ "redactionPolicy": "operator-only",
+ "telemetry": {
+ "eventName": "croco.problem.error",
+ "severity": "error",
+ "attributes": ["problem.code", "problem.category", "problem.status"]
+ }
+ },
+ "lifecycle": {
+ "status": "active"
+ },
+ "sources": [
+ {
+ "file": "packages/testing/src/libs/TestKernel.ts",
+ "line": 227,
+ "column": 5,
+ "kind": "problem-constructor"
+ }
+ ]
+ },
+ {
+ "code": "testing/test-kernel-outbound-call",
+ "category": "InternalServerError",
+ "status": 500,
+ "title": "Internal Server Error",
+ "cookbookPath": "/reference/problem-recovery-cookbook/#testing-test-kernel-outbound-call",
+ "recovery": {
+ "cause": "Croco or an upstream dependency failed after accepting the request.",
+ "userAction": "Retry later only when the operation is idempotent or the caller owns retry safety.",
+ "operatorAction": "Use traces, logs, and upstream diagnostics to isolate the failing boundary.",
+ "retryability": "conditional",
+ "redactionPolicy": "operator-only",
+ "telemetry": {
+ "eventName": "croco.problem.error",
+ "severity": "error",
+ "attributes": ["problem.code", "problem.category", "problem.status"]
+ }
+ },
+ "lifecycle": {
+ "status": "active"
+ },
+ "sources": [
+ {
+ "file": "packages/testing/src/libs/TestRuntime.ts",
+ "line": 49,
"column": 5,
"kind": "problem-constructor"
}
@@ -14696,7 +14786,7 @@
"sources": [
{
"file": "packages/testing/src/libs/TestKernel.ts",
- "line": 182,
+ "line": 238,
"column": 5,
"kind": "problem-constructor"
}
@@ -14726,7 +14816,7 @@
"sources": [
{
"file": "packages/testing/src/libs/TestKernel.ts",
- "line": 198,
+ "line": 254,
"column": 5,
"kind": "problem-constructor"
}
@@ -14756,7 +14846,7 @@
"sources": [
{
"file": "packages/testing/src/libs/TestKernel.ts",
- "line": 211,
+ "line": 267,
"column": 5,
"kind": "problem-constructor"
}
@@ -14786,7 +14876,67 @@
"sources": [
{
"file": "packages/testing/src/libs/TestKernel.ts",
- "line": 131,
+ "line": 176,
+ "column": 5,
+ "kind": "problem-constructor"
+ }
+ ]
+ },
+ {
+ "code": "testing/test-runtime-configuration",
+ "category": "ValidationError",
+ "status": 422,
+ "title": "Validation Error",
+ "cookbookPath": "/reference/problem-recovery-cookbook/#testing-test-runtime-configuration",
+ "recovery": {
+ "cause": "The request or generated contract failed schema or semantic validation.",
+ "userAction": "Fix the invalid fields and retry with schema-conformant input.",
+ "operatorAction": "Inspect schema diagnostics, generated contracts, and validation metadata.",
+ "retryability": "not-retryable",
+ "redactionPolicy": "public",
+ "telemetry": {
+ "eventName": "croco.problem.info",
+ "severity": "info",
+ "attributes": ["problem.code", "problem.category", "problem.status"]
+ }
+ },
+ "lifecycle": {
+ "status": "active"
+ },
+ "sources": [
+ {
+ "file": "packages/testing/src/libs/TestRuntime.ts",
+ "line": 65,
+ "column": 5,
+ "kind": "problem-constructor"
+ }
+ ]
+ },
+ {
+ "code": "testing/test-runtime-drain-limit",
+ "category": "InternalServerError",
+ "status": 500,
+ "title": "Internal Server Error",
+ "cookbookPath": "/reference/problem-recovery-cookbook/#testing-test-runtime-drain-limit",
+ "recovery": {
+ "cause": "Croco or an upstream dependency failed after accepting the request.",
+ "userAction": "Retry later only when the operation is idempotent or the caller owns retry safety.",
+ "operatorAction": "Use traces, logs, and upstream diagnostics to isolate the failing boundary.",
+ "retryability": "conditional",
+ "redactionPolicy": "operator-only",
+ "telemetry": {
+ "eventName": "croco.problem.error",
+ "severity": "error",
+ "attributes": ["problem.code", "problem.category", "problem.status"]
+ }
+ },
+ "lifecycle": {
+ "status": "active"
+ },
+ "sources": [
+ {
+ "file": "packages/testing/src/libs/TestRuntime.ts",
+ "line": 76,
"column": 5,
"kind": "problem-constructor"
}
diff --git a/packages/docs/src/content/docs/en/reference/problem-recovery-cookbook.md b/packages/docs/src/content/docs/en/reference/problem-recovery-cookbook.md
index a22e09f15..b7fd5c74a 100644
--- a/packages/docs/src/content/docs/en/reference/problem-recovery-cookbook.md
+++ b/packages/docs/src/content/docs/en/reference/problem-recovery-cookbook.md
@@ -7,7 +7,7 @@ description: Generated Croco Problem code registry with recovery and telemetry m
> Generated by `pnpm problem-registry:write`. Do not edit this file by hand.
-This cookbook documents 571 public Croco Problem codes. The deterministic JSON registry is generated at `docs/problem-code-registry.json`, and generated client union types are emitted at `packages/problems-core/src/generated/problem-code-registry.ts`.
+This cookbook documents 576 public Croco Problem codes. The deterministic JSON registry is generated at `docs/problem-code-registry.json`, and generated client union types are emitted at `packages/problems-core/src/generated/problem-code-registry.ts`.
## Index
@@ -416,6 +416,7 @@ This cookbook documents 571 public Croco Problem codes. The deterministic JSON r
| [`RATE_LIMIT_WINDOW_ERROR`](#rate-limit-window-error) | BadRequest | 400 | not-retryable | public | active | 1 |
| [`ratelimit-upstash/invalid-policy`](#ratelimit-upstash-invalid-policy) | InternalServerError | 500 | conditional | operator-only | active | 1 |
| [`ratelimit-upstash/missing-config`](#ratelimit-upstash-missing-config) | InternalServerError | 500 | conditional | operator-only | active | 1 |
+| [`ratelimit/prune-interval`](#ratelimit-prune-interval) | ValidationError | 422 | not-retryable | public | active | 1 |
| [`repository-core/batch-loader-factory-not-registered`](#repository-core-batch-loader-factory-not-registered) | InternalServerError | 500 | conditional | operator-only | active | 1 |
| [`repository-core/batch-loader-factory-resolution-failed`](#repository-core-batch-loader-factory-resolution-failed) | InternalServerError | 500 | conditional | operator-only | active | 1 |
| [`repository-core/batch-loader-scope-collision`](#repository-core-batch-loader-scope-collision) | InternalServerError | 500 | conditional | operator-only | active | 1 |
@@ -502,10 +503,14 @@ This cookbook documents 571 public Croco Problem codes. The deterministic JSON r
| [`testing/telemetry-provider-already-installed`](#testing-telemetry-provider-already-installed) | InternalServerError | 500 | conditional | operator-only | active | 1 |
| [`testing/test-kernel-disposal-failed`](#testing-test-kernel-disposal-failed) | InternalServerError | 500 | conditional | operator-only | active | 1 |
| [`testing/test-kernel-disposed`](#testing-test-kernel-disposed) | InternalServerError | 500 | conditional | operator-only | active | 1 |
+| [`testing/test-kernel-leak`](#testing-test-kernel-leak) | InternalServerError | 500 | conditional | operator-only | active | 1 |
+| [`testing/test-kernel-outbound-call`](#testing-test-kernel-outbound-call) | InternalServerError | 500 | conditional | operator-only | active | 1 |
| [`testing/test-kernel-resource-fidelity`](#testing-test-kernel-resource-fidelity) | InternalServerError | 500 | conditional | operator-only | active | 1 |
| [`testing/test-kernel-resource-not-found`](#testing-test-kernel-resource-not-found) | InternalServerError | 500 | conditional | operator-only | active | 1 |
| [`testing/test-kernel-resource-registration`](#testing-test-kernel-resource-registration) | InternalServerError | 500 | conditional | operator-only | active | 1 |
| [`testing/test-kernel-validation-policy`](#testing-test-kernel-validation-policy) | InternalServerError | 500 | conditional | operator-only | active | 1 |
+| [`testing/test-runtime-configuration`](#testing-test-runtime-configuration) | ValidationError | 422 | not-retryable | public | active | 1 |
+| [`testing/test-runtime-drain-limit`](#testing-test-runtime-drain-limit) | InternalServerError | 500 | conditional | operator-only | active | 1 |
| [`testing/transaction-context-not-active`](#testing-transaction-context-not-active) | InternalServerError | 500 | conditional | operator-only | active | 1 |
| [`TOKEN_LIMIT_EXCEEDED`](#token-limit-exceeded) | BadRequest | 400 | not-retryable | public | active | 1 |
| [`TOOL_EXECUTION_ERROR`](#tool-execution-error) | InternalServerError | 500 | conditional | operator-only | active | 1 |
@@ -7767,7 +7772,7 @@ Sources:
Sources:
-- `packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts:4:3` (problem-class)
+- `packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts:6:3` (problem-class)
@@ -7785,7 +7790,7 @@ Sources:
Sources:
-- `packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts:25:5` (problem-constructor)
+- `packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts:43:5` (problem-constructor)
@@ -7803,7 +7808,7 @@ Sources:
Sources:
-- `packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts:14:3` (problem-class)
+- `packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts:16:3` (problem-class)
@@ -7841,6 +7846,24 @@ Sources:
- `packages/ratelimit-upstash/src/libs/problems/RateLimitUpstashProblems.ts:21:3` (problem-class)
+
+
+## `ratelimit/prune-interval`
+
+- Category: `ValidationError`
+- HTTP status: `422` Validation Error
+- Retryability: `not-retryable`
+- Redaction policy: `public`
+- Lifecycle: `active`
+- Cause: The request or generated contract failed schema or semantic validation.
+- User action: Fix the invalid fields and retry with schema-conformant input.
+- Operator action: Inspect schema diagnostics, generated contracts, and validation metadata.
+- Telemetry: `croco.problem.info` (info) with `problem.code`, `problem.category`, `problem.status`
+
+Sources:
+
+- `packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts:27:5` (problem-constructor)
+
## `repository-core/batch-loader-factory-not-registered`
@@ -9369,7 +9392,7 @@ Sources:
Sources:
-- `packages/testing/src/libs/TestKernel.ts:144:5` (problem-constructor)
+- `packages/testing/src/libs/TestKernel.ts:189:5` (problem-constructor)
@@ -9387,7 +9410,43 @@ Sources:
Sources:
-- `packages/testing/src/libs/TestKernel.ts:172:5` (problem-constructor)
+- `packages/testing/src/libs/TestKernel.ts:217:5` (problem-constructor)
+
+
+
+## `testing/test-kernel-leak`
+
+- Category: `InternalServerError`
+- HTTP status: `500` Internal Server Error
+- Retryability: `conditional`
+- Redaction policy: `operator-only`
+- Lifecycle: `active`
+- Cause: Croco or an upstream dependency failed after accepting the request.
+- User action: Retry later only when the operation is idempotent or the caller owns retry safety.
+- Operator action: Use traces, logs, and upstream diagnostics to isolate the failing boundary.
+- Telemetry: `croco.problem.error` (error) with `problem.code`, `problem.category`, `problem.status`
+
+Sources:
+
+- `packages/testing/src/libs/TestKernel.ts:227:5` (problem-constructor)
+
+
+
+## `testing/test-kernel-outbound-call`
+
+- Category: `InternalServerError`
+- HTTP status: `500` Internal Server Error
+- Retryability: `conditional`
+- Redaction policy: `operator-only`
+- Lifecycle: `active`
+- Cause: Croco or an upstream dependency failed after accepting the request.
+- User action: Retry later only when the operation is idempotent or the caller owns retry safety.
+- Operator action: Use traces, logs, and upstream diagnostics to isolate the failing boundary.
+- Telemetry: `croco.problem.error` (error) with `problem.code`, `problem.category`, `problem.status`
+
+Sources:
+
+- `packages/testing/src/libs/TestRuntime.ts:49:5` (problem-constructor)
@@ -9405,7 +9464,7 @@ Sources:
Sources:
-- `packages/testing/src/libs/TestKernel.ts:182:5` (problem-constructor)
+- `packages/testing/src/libs/TestKernel.ts:238:5` (problem-constructor)
@@ -9423,7 +9482,7 @@ Sources:
Sources:
-- `packages/testing/src/libs/TestKernel.ts:198:5` (problem-constructor)
+- `packages/testing/src/libs/TestKernel.ts:254:5` (problem-constructor)
@@ -9441,7 +9500,7 @@ Sources:
Sources:
-- `packages/testing/src/libs/TestKernel.ts:211:5` (problem-constructor)
+- `packages/testing/src/libs/TestKernel.ts:267:5` (problem-constructor)
@@ -9459,7 +9518,43 @@ Sources:
Sources:
-- `packages/testing/src/libs/TestKernel.ts:131:5` (problem-constructor)
+- `packages/testing/src/libs/TestKernel.ts:176:5` (problem-constructor)
+
+
+
+## `testing/test-runtime-configuration`
+
+- Category: `ValidationError`
+- HTTP status: `422` Validation Error
+- Retryability: `not-retryable`
+- Redaction policy: `public`
+- Lifecycle: `active`
+- Cause: The request or generated contract failed schema or semantic validation.
+- User action: Fix the invalid fields and retry with schema-conformant input.
+- Operator action: Inspect schema diagnostics, generated contracts, and validation metadata.
+- Telemetry: `croco.problem.info` (info) with `problem.code`, `problem.category`, `problem.status`
+
+Sources:
+
+- `packages/testing/src/libs/TestRuntime.ts:65:5` (problem-constructor)
+
+
+
+## `testing/test-runtime-drain-limit`
+
+- Category: `InternalServerError`
+- HTTP status: `500` Internal Server Error
+- Retryability: `conditional`
+- Redaction policy: `operator-only`
+- Lifecycle: `active`
+- Cause: Croco or an upstream dependency failed after accepting the request.
+- User action: Retry later only when the operation is idempotent or the caller owns retry safety.
+- Operator action: Use traces, logs, and upstream diagnostics to isolate the failing boundary.
+- Telemetry: `croco.problem.error` (error) with `problem.code`, `problem.category`, `problem.status`
+
+Sources:
+
+- `packages/testing/src/libs/TestRuntime.ts:76:5` (problem-constructor)
diff --git a/packages/problems-core/src/generated/problem-code-registry.ts b/packages/problems-core/src/generated/problem-code-registry.ts
index f354fc14f..da295df32 100644
--- a/packages/problems-core/src/generated/problem-code-registry.ts
+++ b/packages/problems-core/src/generated/problem-code-registry.ts
@@ -3,7 +3,7 @@ import type { ProblemCodeRegistry } from "../libs/ProblemRegistry";
export const CROCO_PROBLEM_CODE_REGISTRY = {
version: "croco.problem-code-registry.v1",
- problemCount: 571,
+ problemCount: 576,
problems: [
{
code: "ACCESS_DENIED",
@@ -12449,7 +12449,7 @@ export const CROCO_PROBLEM_CODE_REGISTRY = {
sources: [
{
file: "packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts",
- line: 4,
+ line: 6,
column: 3,
kind: "problem-class",
},
@@ -12481,7 +12481,7 @@ export const CROCO_PROBLEM_CODE_REGISTRY = {
sources: [
{
file: "packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts",
- line: 25,
+ line: 43,
column: 5,
kind: "problem-constructor",
},
@@ -12512,7 +12512,7 @@ export const CROCO_PROBLEM_CODE_REGISTRY = {
sources: [
{
file: "packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts",
- line: 14,
+ line: 16,
column: 3,
kind: "problem-class",
},
@@ -12582,6 +12582,36 @@ export const CROCO_PROBLEM_CODE_REGISTRY = {
},
],
},
+ {
+ code: "ratelimit/prune-interval",
+ category: "ValidationError",
+ status: 422,
+ title: "Validation Error",
+ cookbookPath: "/reference/problem-recovery-cookbook/#ratelimit-prune-interval",
+ recovery: {
+ cause: "The request or generated contract failed schema or semantic validation.",
+ userAction: "Fix the invalid fields and retry with schema-conformant input.",
+ operatorAction: "Inspect schema diagnostics, generated contracts, and validation metadata.",
+ retryability: "not-retryable",
+ redactionPolicy: "public",
+ telemetry: {
+ eventName: "croco.problem.info",
+ severity: "info",
+ attributes: ["problem.code", "problem.category", "problem.status"],
+ },
+ },
+ lifecycle: {
+ status: "active",
+ },
+ sources: [
+ {
+ file: "packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts",
+ line: 27,
+ column: 5,
+ kind: "problem-constructor",
+ },
+ ],
+ },
{
code: "repository-core/batch-loader-factory-not-registered",
category: "InternalServerError",
@@ -15253,7 +15283,7 @@ export const CROCO_PROBLEM_CODE_REGISTRY = {
sources: [
{
file: "packages/testing/src/libs/TestKernel.ts",
- line: 144,
+ line: 189,
column: 5,
kind: "problem-constructor",
},
@@ -15285,7 +15315,71 @@ export const CROCO_PROBLEM_CODE_REGISTRY = {
sources: [
{
file: "packages/testing/src/libs/TestKernel.ts",
- line: 172,
+ line: 217,
+ column: 5,
+ kind: "problem-constructor",
+ },
+ ],
+ },
+ {
+ code: "testing/test-kernel-leak",
+ category: "InternalServerError",
+ status: 500,
+ title: "Internal Server Error",
+ cookbookPath: "/reference/problem-recovery-cookbook/#testing-test-kernel-leak",
+ recovery: {
+ cause: "Croco or an upstream dependency failed after accepting the request.",
+ userAction:
+ "Retry later only when the operation is idempotent or the caller owns retry safety.",
+ operatorAction:
+ "Use traces, logs, and upstream diagnostics to isolate the failing boundary.",
+ retryability: "conditional",
+ redactionPolicy: "operator-only",
+ telemetry: {
+ eventName: "croco.problem.error",
+ severity: "error",
+ attributes: ["problem.code", "problem.category", "problem.status"],
+ },
+ },
+ lifecycle: {
+ status: "active",
+ },
+ sources: [
+ {
+ file: "packages/testing/src/libs/TestKernel.ts",
+ line: 227,
+ column: 5,
+ kind: "problem-constructor",
+ },
+ ],
+ },
+ {
+ code: "testing/test-kernel-outbound-call",
+ category: "InternalServerError",
+ status: 500,
+ title: "Internal Server Error",
+ cookbookPath: "/reference/problem-recovery-cookbook/#testing-test-kernel-outbound-call",
+ recovery: {
+ cause: "Croco or an upstream dependency failed after accepting the request.",
+ userAction:
+ "Retry later only when the operation is idempotent or the caller owns retry safety.",
+ operatorAction:
+ "Use traces, logs, and upstream diagnostics to isolate the failing boundary.",
+ retryability: "conditional",
+ redactionPolicy: "operator-only",
+ telemetry: {
+ eventName: "croco.problem.error",
+ severity: "error",
+ attributes: ["problem.code", "problem.category", "problem.status"],
+ },
+ },
+ lifecycle: {
+ status: "active",
+ },
+ sources: [
+ {
+ file: "packages/testing/src/libs/TestRuntime.ts",
+ line: 49,
column: 5,
kind: "problem-constructor",
},
@@ -15317,7 +15411,7 @@ export const CROCO_PROBLEM_CODE_REGISTRY = {
sources: [
{
file: "packages/testing/src/libs/TestKernel.ts",
- line: 182,
+ line: 238,
column: 5,
kind: "problem-constructor",
},
@@ -15349,7 +15443,7 @@ export const CROCO_PROBLEM_CODE_REGISTRY = {
sources: [
{
file: "packages/testing/src/libs/TestKernel.ts",
- line: 198,
+ line: 254,
column: 5,
kind: "problem-constructor",
},
@@ -15382,7 +15476,7 @@ export const CROCO_PROBLEM_CODE_REGISTRY = {
sources: [
{
file: "packages/testing/src/libs/TestKernel.ts",
- line: 211,
+ line: 267,
column: 5,
kind: "problem-constructor",
},
@@ -15414,7 +15508,69 @@ export const CROCO_PROBLEM_CODE_REGISTRY = {
sources: [
{
file: "packages/testing/src/libs/TestKernel.ts",
- line: 131,
+ line: 176,
+ column: 5,
+ kind: "problem-constructor",
+ },
+ ],
+ },
+ {
+ code: "testing/test-runtime-configuration",
+ category: "ValidationError",
+ status: 422,
+ title: "Validation Error",
+ cookbookPath: "/reference/problem-recovery-cookbook/#testing-test-runtime-configuration",
+ recovery: {
+ cause: "The request or generated contract failed schema or semantic validation.",
+ userAction: "Fix the invalid fields and retry with schema-conformant input.",
+ operatorAction: "Inspect schema diagnostics, generated contracts, and validation metadata.",
+ retryability: "not-retryable",
+ redactionPolicy: "public",
+ telemetry: {
+ eventName: "croco.problem.info",
+ severity: "info",
+ attributes: ["problem.code", "problem.category", "problem.status"],
+ },
+ },
+ lifecycle: {
+ status: "active",
+ },
+ sources: [
+ {
+ file: "packages/testing/src/libs/TestRuntime.ts",
+ line: 65,
+ column: 5,
+ kind: "problem-constructor",
+ },
+ ],
+ },
+ {
+ code: "testing/test-runtime-drain-limit",
+ category: "InternalServerError",
+ status: 500,
+ title: "Internal Server Error",
+ cookbookPath: "/reference/problem-recovery-cookbook/#testing-test-runtime-drain-limit",
+ recovery: {
+ cause: "Croco or an upstream dependency failed after accepting the request.",
+ userAction:
+ "Retry later only when the operation is idempotent or the caller owns retry safety.",
+ operatorAction:
+ "Use traces, logs, and upstream diagnostics to isolate the failing boundary.",
+ retryability: "conditional",
+ redactionPolicy: "operator-only",
+ telemetry: {
+ eventName: "croco.problem.error",
+ severity: "error",
+ attributes: ["problem.code", "problem.category", "problem.status"],
+ },
+ },
+ lifecycle: {
+ status: "active",
+ },
+ sources: [
+ {
+ file: "packages/testing/src/libs/TestRuntime.ts",
+ line: 76,
column: 5,
kind: "problem-constructor",
},
diff --git a/packages/ratelimit-core/src/index.ts b/packages/ratelimit-core/src/index.ts
index 40d0584c5..7f7eb1db1 100644
--- a/packages/ratelimit-core/src/index.ts
+++ b/packages/ratelimit-core/src/index.ts
@@ -25,7 +25,6 @@ export {
*/
export {
FixedWindowInMemoryStore,
- type InMemoryRateLimitStoreOptions,
SlidingWindowInMemoryStore,
TokenBucketInMemoryStore,
} from "./libs/InMemoryRateLimitStore";
@@ -48,6 +47,7 @@ export {
*/
export {
RateLimitKeyBuilderProblem,
+ RateLimitPruneIntervalProblem,
RateLimitRefundUnsupportedProblem,
RateLimitWindowProblem,
} from "./libs/problems/RateLimitConfigProblems";
@@ -112,3 +112,8 @@ export {
type TokenBucketRefundReceipt,
type TokenBucketPolicy,
} from "./libs/types";
+
+export type {
+ InMemoryRateLimitStoreOptions,
+ RateLimitPruneScheduler,
+} from "./libs/InMemoryRateLimitStore";
diff --git a/packages/ratelimit-core/src/libs/InMemoryRateLimitStore.ts b/packages/ratelimit-core/src/libs/InMemoryRateLimitStore.ts
index e2e365294..cdba632c2 100644
--- a/packages/ratelimit-core/src/libs/InMemoryRateLimitStore.ts
+++ b/packages/ratelimit-core/src/libs/InMemoryRateLimitStore.ts
@@ -8,13 +8,34 @@ import type {
SlidingWindowPolicy,
TokenBucketPolicy,
} from "./types";
-import { RateLimitRefundUnsupportedProblem } from "./problems/RateLimitConfigProblems";
+import {
+ RateLimitPruneIntervalProblem,
+ RateLimitRefundUnsupportedProblem,
+} from "./problems/RateLimitConfigProblems";
export type InMemoryRateLimitStoreOptions = {
- pruneIntervalMs?: number;
+ readonly now?: () => number;
+ readonly pruneIntervalMs?: number;
+ readonly random?: () => number;
+ readonly scheduler?: RateLimitPruneScheduler;
};
+export interface RateLimitPruneScheduler {
+ schedule(callback: () => void | Promise, intervalMs: number): () => void;
+}
+
const DEFAULT_PRUNE_INTERVAL_MS = 60000;
+const MAX_NATIVE_TIMER_DELAY_MS = 2_147_483_647;
+
+const DEFAULT_PRUNE_SCHEDULER: RateLimitPruneScheduler = {
+ schedule(callback, intervalMs) {
+ const timer = setTimeout(() => {
+ void callback();
+ }, intervalMs);
+ timer.unref?.();
+ return () => clearTimeout(timer);
+ },
+};
type MutableRateLimitStats = {
allowed: number;
@@ -27,30 +48,52 @@ function recordRefund(stats: MutableRateLimitStats): void {
stats.total = Math.max(0, stats.total - 1);
}
+function schedulePruning(
+ options: InMemoryRateLimitStoreOptions,
+ callback: () => void | Promise,
+): (() => void) | undefined {
+ const pruneIntervalMs = options.pruneIntervalMs ?? DEFAULT_PRUNE_INTERVAL_MS;
+ if (pruneIntervalMs <= 0) return undefined;
+ if (
+ options.scheduler === undefined &&
+ (!Number.isFinite(pruneIntervalMs) || pruneIntervalMs > MAX_NATIVE_TIMER_DELAY_MS)
+ ) {
+ throw new RateLimitPruneIntervalProblem(pruneIntervalMs);
+ }
+ const scheduler = options.scheduler ?? DEFAULT_PRUNE_SCHEDULER;
+ let cancelScheduledWork: (() => void) | undefined;
+ let closed = false;
+ const scheduleNext = () => {
+ cancelScheduledWork = scheduler.schedule(async () => {
+ await callback();
+ if (!closed) scheduleNext();
+ }, pruneIntervalMs);
+ };
+ scheduleNext();
+ return () => {
+ closed = true;
+ cancelScheduledWork?.();
+ };
+}
+
export class FixedWindowInMemoryStore extends FixedWindowStore {
private readonly windows = new Map<
string,
{ count: number; windowStart: number; windowMs: number }
>();
private readonly globalStats = { allowed: 0, denied: 0, total: 0 };
- private readonly pruneTimer?: ReturnType;
+ private readonly cancelPruning?: () => void;
constructor(options: InMemoryRateLimitStoreOptions = {}) {
- super();
-
- const pruneIntervalMs = options.pruneIntervalMs ?? DEFAULT_PRUNE_INTERVAL_MS;
- if (pruneIntervalMs > 0) {
- this.pruneTimer = setInterval(() => {
- void this.pruneExpired();
- }, pruneIntervalMs);
- this.pruneTimer.unref?.();
- }
+ super(options.now, options.random);
+
+ this.cancelPruning = schedulePruning(options, async () => {
+ await this.pruneExpired();
+ });
}
close(): void {
- if (this.pruneTimer !== undefined) {
- clearInterval(this.pruneTimer);
- }
+ this.cancelPruning?.();
}
destroy(): void {
@@ -89,7 +132,7 @@ export class FixedWindowInMemoryStore extends FixedWindowStore {
const entry = this.windows.get(key);
if (!entry) return null;
- const windowStart = Math.floor(Date.now() / policy.windowMs) * policy.windowMs;
+ const windowStart = Math.floor(this.now() / policy.windowMs) * policy.windowMs;
if (entry.windowStart !== windowStart) {
return null;
}
@@ -127,7 +170,7 @@ export class FixedWindowInMemoryStore extends FixedWindowStore {
}
async pruneExpired(): Promise {
- const now = Date.now();
+ const now = this.now();
let deletedCount = 0;
for (const [key, entry] of this.windows.entries()) {
@@ -154,24 +197,18 @@ export class SlidingWindowInMemoryStore extends SlidingWindowStore {
>();
private readonly _windowMsCache = new Map();
private readonly globalStats = { allowed: 0, denied: 0, total: 0 };
- private readonly pruneTimer?: ReturnType;
+ private readonly cancelPruning?: () => void;
constructor(options: InMemoryRateLimitStoreOptions = {}) {
- super();
-
- const pruneIntervalMs = options.pruneIntervalMs ?? DEFAULT_PRUNE_INTERVAL_MS;
- if (pruneIntervalMs > 0) {
- this.pruneTimer = setInterval(() => {
- void this.pruneExpired();
- }, pruneIntervalMs);
- this.pruneTimer.unref?.();
- }
+ super(options.now, options.random);
+
+ this.cancelPruning = schedulePruning(options, async () => {
+ await this.pruneExpired();
+ });
}
close(): void {
- if (this.pruneTimer !== undefined) {
- clearInterval(this.pruneTimer);
- }
+ this.cancelPruning?.();
}
destroy(): void {
@@ -205,7 +242,7 @@ export class SlidingWindowInMemoryStore extends SlidingWindowStore {
throw new RateLimitRefundUnsupportedProblem();
}
- const now = Date.now();
+ const now = this.now();
const windowStart = now - policy.windowMs;
await this.removeTimestamps(key, windowStart);
@@ -307,7 +344,7 @@ export class SlidingWindowInMemoryStore extends SlidingWindowStore {
}
async pruneExpired(): Promise {
- const now = Date.now();
+ const now = this.now();
let deletedCount = 0;
for (const [key, entry] of this.windows.entries()) {
@@ -338,24 +375,18 @@ export class TokenBucketInMemoryStore extends TokenBucketStore {
{ tokens: number; lastRefill: number; ttlMs: number }
>();
private readonly globalStats = { allowed: 0, denied: 0, total: 0 };
- private readonly pruneTimer?: ReturnType;
+ private readonly cancelPruning?: () => void;
constructor(options: InMemoryRateLimitStoreOptions = {}) {
- super();
-
- const pruneIntervalMs = options.pruneIntervalMs ?? DEFAULT_PRUNE_INTERVAL_MS;
- if (pruneIntervalMs > 0) {
- this.pruneTimer = setInterval(() => {
- void this.pruneExpired();
- }, pruneIntervalMs);
- this.pruneTimer.unref?.();
- }
+ super(options.now, options.random);
+
+ this.cancelPruning = schedulePruning(options, async () => {
+ await this.pruneExpired();
+ });
}
close(): void {
- if (this.pruneTimer !== undefined) {
- clearInterval(this.pruneTimer);
- }
+ this.cancelPruning?.();
}
destroy(): void {
@@ -418,7 +449,7 @@ export class TokenBucketInMemoryStore extends TokenBucketStore {
}
async pruneExpired(): Promise {
- const now = Date.now();
+ const now = this.now();
let deletedCount = 0;
for (const [key, entry] of this.buckets.entries()) {
diff --git a/packages/ratelimit-core/src/libs/RateLimitStore.ts b/packages/ratelimit-core/src/libs/RateLimitStore.ts
index d0b54516a..dcd528c6f 100644
--- a/packages/ratelimit-core/src/libs/RateLimitStore.ts
+++ b/packages/ratelimit-core/src/libs/RateLimitStore.ts
@@ -34,6 +34,13 @@ export type DistributedRateLimitStoreOptions = {
};
export abstract class RateLimitStore {
+ private refundReceiptSequence = 0;
+
+ protected constructor(
+ protected readonly now: () => number = () => Date.now(),
+ protected readonly random: () => number = Math.random,
+ ) {}
+
abstract check(key: string, policy: RateLimitPolicy): Promise;
async refund(
_key: string,
@@ -44,6 +51,12 @@ export abstract class RateLimitStore {
}
abstract getStats(key?: string): Promise;
abstract pruneExpired(): Promise;
+
+ protected createRefundReceiptId(algorithm: string, timestamp: number): string {
+ this.refundReceiptSequence = (this.refundReceiptSequence + 1) % Number.MAX_SAFE_INTEGER;
+ const randomPart = this.random().toString(36).slice(2, 10);
+ return `${algorithm}:${timestamp}:${this.refundReceiptSequence}:${randomPart}`;
+ }
}
export abstract class DistributedRateLimitStore extends RateLimitStore {
@@ -70,7 +83,7 @@ export abstract class FixedWindowStore extends DistributedRateLimitStore {
): Promise;
async checkFixedWindow(key: string, policy: FixedWindowPolicy): Promise {
- const now = Date.now();
+ const now = this.now();
const windowStart = Math.floor(now / policy.windowMs) * policy.windowMs;
const resetAtMs = windowStart + policy.windowMs;
@@ -155,7 +168,7 @@ export abstract class FixedWindowStore extends DistributedRateLimitStore {
): FixedWindowRefundReceipt {
const receipt: FixedWindowRefundReceipt = {
algorithm: "fixed",
- id: createRefundReceiptId("fixed", windowStart),
+ id: this.createRefundReceiptId("fixed", windowStart),
windowStart,
};
const current = this.refundReceipts.get(key);
@@ -218,7 +231,7 @@ export abstract class SlidingWindowStore extends DistributedRateLimitStore {
protected abstract removeTimestamps(key: string, before: number): Promise;
async checkSlidingWindow(key: string, policy: SlidingWindowPolicy): Promise {
- const now = Date.now();
+ const now = this.now();
const windowStart = now - policy.windowMs;
await this.removeTimestamps(key, windowStart);
@@ -229,7 +242,7 @@ export abstract class SlidingWindowStore extends DistributedRateLimitStore {
const refundReceipt: SlidingWindowRefundReceipt | undefined = success
? {
algorithm: "sliding",
- id: createRefundReceiptId("sliding", now),
+ id: this.createRefundReceiptId("sliding", now),
timestamp: now,
}
: undefined;
@@ -266,7 +279,7 @@ export abstract class TokenBucketStore extends DistributedRateLimitStore {
protected abstract setBucket(key: string, entry: TokenBucketEntry, ttlMs: number): Promise;
async checkTokenBucket(key: string, policy: TokenBucketPolicy): Promise {
- const now = Date.now();
+ const now = this.now();
const intervalMs = policy.refillIntervalMs;
let bucket = await this.getBucket(key);
@@ -333,7 +346,7 @@ export abstract class TokenBucketStore extends DistributedRateLimitStore {
throw new RateLimitRefundUnsupportedProblem();
}
- const now = Date.now();
+ const now = this.now();
const intervalMs = policy.refillIntervalMs;
const ttlMs = (policy.capacity * intervalMs) / policy.refillRate;
const resetAtMs = now + intervalMs / policy.refillRate;
@@ -391,7 +404,7 @@ export abstract class TokenBucketStore extends DistributedRateLimitStore {
};
}
- const now = Date.now();
+ const now = this.now();
const tokensToAdd = Math.floor(
((now - bucket.lastRefill) / policy.refillIntervalMs) * policy.refillRate,
);
@@ -415,12 +428,12 @@ export abstract class TokenBucketStore extends DistributedRateLimitStore {
key: string,
expiresAtMs: number,
): TokenBucketRefundReceipt {
- const now = Date.now();
+ const now = this.now();
this.pruneTokenBucketRefundReceipts(key, now);
const receipt: TokenBucketRefundReceipt = {
algorithm: "token-bucket",
- id: createRefundReceiptId("token-bucket", now),
+ id: this.createRefundReceiptId("token-bucket", now),
expiresAtMs,
};
const receipts = this.refundReceipts.get(key) ?? new Map();
@@ -475,11 +488,3 @@ export abstract class TokenBucketStore extends DistributedRateLimitStore {
}
}
}
-
-let refundReceiptSequence = 0;
-
-function createRefundReceiptId(algorithm: string, timestamp: number): string {
- refundReceiptSequence = (refundReceiptSequence + 1) % Number.MAX_SAFE_INTEGER;
- const randomPart = Math.random().toString(36).slice(2, 10);
- return `${algorithm}:${timestamp}:${refundReceiptSequence}:${randomPart}`;
-}
diff --git a/packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts b/packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts
index fc565dc35..34459a6d7 100644
--- a/packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts
+++ b/packages/ratelimit-core/src/libs/problems/RateLimitConfigProblems.ts
@@ -1,5 +1,7 @@
import { Problem, ProblemCategory } from "@croco/problems-core";
+const MAX_NATIVE_TIMER_DELAY_MS = 2_147_483_647;
+
export class RateLimitKeyBuilderProblem extends Problem {
readonly code = "RATE_LIMIT_KEY_BUILDER_ERROR";
readonly category = ProblemCategory.InternalServerError;
@@ -20,6 +22,22 @@ export class RateLimitWindowProblem extends Problem {
}
}
+export class RateLimitPruneIntervalProblem extends Problem {
+ constructor(value: number) {
+ super(
+ "ratelimit/prune-interval",
+ ProblemCategory.ValidationError,
+ `Rate limit prune interval must be a finite delay no greater than ${MAX_NATIVE_TIMER_DELAY_MS}ms; received '${String(value)}'.`,
+ {
+ extensions: {
+ maxDelayMs: MAX_NATIVE_TIMER_DELAY_MS,
+ value: String(value),
+ },
+ },
+ );
+ }
+}
+
export class RateLimitRefundUnsupportedProblem extends Problem {
constructor() {
super(
diff --git a/packages/ratelimit-core/src/tests/InMemoryRateLimitStore.spec.ts b/packages/ratelimit-core/src/tests/InMemoryRateLimitStore.spec.ts
index 725972a6f..8ec766733 100644
--- a/packages/ratelimit-core/src/tests/InMemoryRateLimitStore.spec.ts
+++ b/packages/ratelimit-core/src/tests/InMemoryRateLimitStore.spec.ts
@@ -1,10 +1,12 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { Container } from "@croco/framework-context";
import {
FixedWindowInMemoryStore,
InMemoryRateLimitStore,
SlidingWindowInMemoryStore,
TokenBucketInMemoryStore,
} from "../libs/InMemoryRateLimitStore";
+import { RateLimitPruneIntervalProblem } from "../libs/problems/RateLimitConfigProblems";
import type { FixedWindowPolicy, SlidingWindowPolicy, TokenBucketPolicy } from "../libs/types";
describe("InMemoryRateLimitStore", () => {
@@ -17,6 +19,7 @@ describe("InMemoryRateLimitStore", () => {
};
beforeEach(() => {
+ Container.reset();
store = new InMemoryRateLimitStore({ pruneIntervalMs: 0 });
});
@@ -24,6 +27,15 @@ describe("InMemoryRateLimitStore", () => {
store.close();
});
+ it.each([Number.NaN, Number.POSITIVE_INFINITY, 2_147_483_648])(
+ "rejects unsupported native prune intervals (%s) with a stable Problem",
+ (pruneIntervalMs) => {
+ expect(() => new InMemoryRateLimitStore({ pruneIntervalMs })).toThrow(
+ RateLimitPruneIntervalProblem,
+ );
+ },
+ );
+
it("should allow requests within limit", async () => {
const result1 = await store.check("user:1", policy);
expect(result1.success).toBe(true);
diff --git a/packages/tasks-core/src/index.ts b/packages/tasks-core/src/index.ts
index ee904bfcb..0c0700e8f 100644
--- a/packages/tasks-core/src/index.ts
+++ b/packages/tasks-core/src/index.ts
@@ -13,11 +13,6 @@ export {
TaskRunnerDIFailureProblem,
} from "./libs/problems/TasksProblems";
-/**
- * 레지스트리에 등록된 태스크 엔트리 타입입니다.
- */
-export type { RegisteredTask } from "./libs/TaskRegistry";
-
/**
* 태스크 메타데이터를 수집하고 조회하는 전역 레지스트리입니다.
*/
@@ -38,3 +33,6 @@ export type {
TaskOptions,
TaskReference,
} from "./libs/types";
+
+export type { RegisteredTask } from "./libs/TaskRegistry";
+export type { TaskRunnerRuntime } from "./libs/TaskRunner";
diff --git a/packages/tasks-core/src/libs/TaskRunner.ts b/packages/tasks-core/src/libs/TaskRunner.ts
index ea6d3a1bc..dee10ee48 100644
--- a/packages/tasks-core/src/libs/TaskRunner.ts
+++ b/packages/tasks-core/src/libs/TaskRunner.ts
@@ -14,6 +14,11 @@ type Constructor = new (...args: unknown[]) => T;
const MAX_TIMER_DELAY_MS = 2_147_483_647;
+export interface TaskRunnerRuntime {
+ readonly now?: () => number;
+ readonly schedule?: (callback: () => void, delayMs: number) => () => void;
+}
+
const noopLogger: ILogger = {
debug: () => {},
info: () => {},
@@ -91,11 +96,23 @@ function recordDiagnosticError(error: unknown): void {
}
export class TaskRunner {
+ private readonly now: () => number;
+ private readonly schedule: (callback: () => void, delayMs: number) => () => void;
+
constructor(
private executionManager: ExecutionManager,
private registry: TaskRegistry = TaskRegistry.fromMetadata(),
private logger: ILogger = noopLogger,
- ) {}
+ runtime: TaskRunnerRuntime = {},
+ ) {
+ this.now = runtime.now ?? (() => Date.now());
+ this.schedule =
+ runtime.schedule ??
+ ((callback, delayMs) => {
+ const timeout = setTimeout(callback, delayMs);
+ return () => clearTimeout(timeout);
+ });
+ }
async execute(
taskId: string,
@@ -166,7 +183,7 @@ export class TaskRunner {
? undefined
: new TaskExecutionTimeoutProblem(startedExecution.id, timeoutMs);
let timeoutClaimed = false;
- let timeoutHandle: ReturnType | undefined;
+ let cancelTimeout: (() => void) | undefined;
let triggerTimeout: (() => void) | undefined;
const timeoutPromise =
timeoutProblem === undefined || deadline === undefined
@@ -182,20 +199,23 @@ export class TaskRunner {
);
};
const scheduleTimeout = () => {
- const remaining = deadline - Date.now();
+ const remaining = deadline - this.now();
if (remaining <= 0) {
triggerTimeout?.();
return;
}
- timeoutHandle = setTimeout(scheduleTimeout, Math.min(remaining, MAX_TIMER_DELAY_MS));
+ cancelTimeout = this.schedule(
+ scheduleTimeout,
+ Math.min(remaining, MAX_TIMER_DELAY_MS),
+ );
};
scheduleTimeout();
});
try {
const instance = this.createInstance(target) as Record;
- if (deadline !== undefined && Date.now() >= deadline) {
+ if (deadline !== undefined && this.now() >= deadline) {
triggerTimeout?.();
return await timeoutPromise;
}
@@ -211,12 +231,12 @@ export class TaskRunner {
? Promise.race([handlerPromise, timeoutPromise])
: handlerPromise);
- if (deadline !== undefined && Date.now() >= deadline) {
+ if (deadline !== undefined && this.now() >= deadline) {
triggerTimeout?.();
return await timeoutPromise;
}
- if (timeoutHandle !== undefined) clearTimeout(timeoutHandle);
+ cancelTimeout?.();
try {
await this.executionManager.complete(startedExecution.id, result);
@@ -229,7 +249,7 @@ export class TaskRunner {
}
return result;
} catch (error) {
- if (timeoutHandle !== undefined) clearTimeout(timeoutHandle);
+ cancelTimeout?.();
if (timeoutClaimed && timeoutPromise !== undefined) {
return await timeoutPromise;
}
@@ -238,7 +258,7 @@ export class TaskRunner {
throw error;
}
- if (deadline !== undefined && Date.now() >= deadline && triggerTimeout !== undefined) {
+ if (deadline !== undefined && this.now() >= deadline && triggerTimeout !== undefined) {
triggerTimeout();
return await timeoutPromise;
}
diff --git a/packages/tasks-core/src/tests/TaskRunner.spec.ts b/packages/tasks-core/src/tests/TaskRunner.spec.ts
index 7b5eb3154..040b67b48 100644
--- a/packages/tasks-core/src/tests/TaskRunner.spec.ts
+++ b/packages/tasks-core/src/tests/TaskRunner.spec.ts
@@ -64,7 +64,7 @@ describe("TaskRunner", () => {
reconcileTimedOut: vi.fn().mockResolvedValue({ scanned: 0, timedOut: 0 }),
};
- @Component()
+ @Component({ scope: "singleton" })
class TestTaskHandler {
@Task({ name: "test-task" })
async handle(payload: { data: string }): Promise {
@@ -636,6 +636,59 @@ describe("TaskRunner", () => {
expect(mockExecutionManager.fail).not.toHaveBeenCalled();
});
+ it("should use injected time and scheduling boundaries for deterministic timeouts", async () => {
+ let now = new Date("2026-01-01T00:00:00.000Z").getTime();
+ let scheduledTimeout: (() => void) | undefined;
+ let scheduledDelayMs: number | undefined;
+ let receivedContext: TaskExecutionContext | undefined;
+
+ @Component()
+ class ControlledTimedTaskHandler {
+ @Task({ name: "controlled-timed-task", timeout: 100 })
+ async handle(_payload: unknown, context: TaskExecutionContext): Promise {
+ receivedContext = context;
+ return new Promise((_resolve, reject) => {
+ context.signal.addEventListener("abort", () => reject(context.signal.reason), {
+ once: true,
+ });
+ });
+ }
+ }
+
+ Container.set(ControlledTimedTaskHandler, new ControlledTimedTaskHandler());
+ registry.collectFromMetadata();
+ mockExecutionManager.start = vi.fn().mockResolvedValue(
+ execution({
+ type: "controlled-timed-task",
+ payload: {},
+ status: "running",
+ attempts: 1,
+ startedAt: new Date(now),
+ timeout: 100,
+ }),
+ );
+ const runner = new TaskRunner(mockExecutionManager, registry, undefined, {
+ now: () => now,
+ schedule: (callback, delayMs) => {
+ scheduledTimeout = callback;
+ scheduledDelayMs = delayMs;
+ return () => {
+ scheduledTimeout = undefined;
+ };
+ },
+ });
+
+ const result = runner.execute("controlled-timed-task", {});
+ await vi.waitFor(() => expect(scheduledTimeout).toBeDefined());
+ expect(scheduledDelayMs).toBe(100);
+ now += 100;
+ scheduledTimeout?.();
+
+ await expect(result).rejects.toThrow(TaskExecutionTimeoutProblem);
+ expect(receivedContext?.signal.aborted).toBe(true);
+ expect(mockExecutionManager.timeout).toHaveBeenCalledWith("exec-123");
+ });
+
it("should ignore a handler result that arrives after timeout", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z"));
diff --git a/packages/testing/README.md b/packages/testing/README.md
index 8d1ee953c..9185d9584 100644
--- a/packages/testing/README.md
+++ b/packages/testing/README.md
@@ -12,7 +12,7 @@ const response = await app.get("/users");
Production-parity tests pass the same bootstrap export used by the deployed application:
```typescript
-import { createTestKernel } from "@croco/testing";
+import { createTestKernel, fixedClock, seededIds } from "@croco/testing";
import { createCrocoApp } from "../app";
await using test = await createTestKernel({
@@ -28,42 +28,59 @@ expect(test.fidelity).toEqual({
});
```
+Deterministic scenarios opt into kernel-owned controls instead of patching process globals:
+
+```typescript
+await using test = await createTestKernel({
+ bootstrap: createCrocoApp,
+ clock: fixedClock("2026-01-01T00:00:00Z"),
+ fidelity: "application",
+ ids: seededIds("invitation-retry"),
+ network: "deny",
+ scenarioId: "invitation-retry",
+});
+
+await test.clock.advanceBy("30s");
+test.expectClean();
+```
+
## API
-| Helper | Purpose |
-| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ |
-| `createTestingApp(config)` | Creates an isolated `CrocoApp` with seeded test defaults and HTTP request helpers. |
-| `createTestKernel(config)` | Boots the real application bootstrap inside an isolated runtime scope and reports application or adapter fidelity as structured evidence. |
-| `createTestingHarness(app)` | Wraps an existing `CrocoApp` with the same request and contract helpers. |
-| `createEventTestingHarness(config)` | Creates an isolated in-memory event bus and dispatches decorated handlers. |
-| `createTestingRequestContext(config)` | Builds a deterministic request/runtime context for service tests. |
-| `runWithTestingContext(fn, config)` | Runs code inside Croco `Context` and clears AsyncLocalStorage state when execution completes. |
-| `createTestingTransactionContext(config)` | Provides explicit in-transaction and after-commit hook behavior for tests. |
-| `resetCrocoTestingContext()` | Resets the Croco DI container and seeds test logger/error/health defaults. |
-| `installTestingTelemetryCapture()` | Captures spans in memory without starting an SDK exporter. |
-| `createFailureDrillCatalog()` | Builds deterministic no-credential failure drills for provider timeout, duplicate delivery, outbox, telemetry, tenant, and quota failures. |
-| `runFailureDrills(cases)` | Executes failure drills and rejects runs that lack the expected Problem code, recovery action, telemetry evidence, or audit evidence. |
-| `createOperationalFailureDrillMatrix(cases)` | Validates the exact ordered operational incident matrix without changing the generic six-scenario catalog. |
-| `runOperationalFailureDrills(cases)` | Executes operational fixtures and verifies their Problem or diagnostic outcome, recovery action, and real-boundary provenance. |
-| `assertProblemResponse(response, expected)` | Verifies an RFC 7807 Problem Details response without depending on a test runner. |
-| `assertOpenAPIRoute(controllersOrSpec, expected)` | Verifies generated OpenAPI route metadata and response contracts. |
-| `createRpcTestFetch(app)` | Returns a fetch-compatible function that routes generated RPC clients into the in-memory app. |
-| `createAuthProviderConformanceSuite(config)` | Reusable auth provider cases for token/session auth, webhooks, tenant mapping, readiness, and live-smoke gating. |
-| `createStorageProviderConformanceSuite(config)` | Reusable storage provider contract cases for default no-credential CI. |
-| `createProviderConformanceMatrixSuite(config)` | Validates provider profile manifests for required capabilities, optional unsupported reasons, and method evidence. |
-| `createLlmProviderConformanceSuite(config)` | Reusable LLM provider contract cases for mocked or live provider fixtures. |
-| `createBillingProviderConformanceSuite(config)` | Builds runner-neutral billing gateway and webhook conformance cases for provider packages. |
-| `createUpstashRedisMeteringConformanceSuite(config)` | Reusable Upstash Redis metering cases for config, usage storage, idempotency, upstream errors, and live-smoke gating. |
-| `createUpstashRedisRateLimitConformanceSuite(config)` | Reusable Upstash Redis rate-limit cases for config, errors, refund idempotency, and live-smoke gating. |
-| `createQStashTaskConformanceSuite(config)` | Reusable QStash task publish cases for config, validation, idempotency, upstream errors, and live-smoke gating. |
-| `createQStashBatchConformanceSuite(config)` | Reusable QStash batch chunk cases for terminal chunks, continuation envelopes, upstream errors, and live-smoke gating. |
-| `createQStashTriggerConformanceSuite(config)` | Reusable QStash trigger cases for schedule sync, webhook verification, dispatch, upstream diagnostics, and live-smoke gating. |
-| `createDrizzleProviderConformanceSuite(config)` | Builds reusable Drizzle provider cases for schema, transaction, tenant, and error contracts. |
-| `assertDrizzleProblem(operation, expected)` | Verifies Drizzle provider failures surface stable Croco Problem codes, categories, or status. |
-| `createTestEvidenceRecord(input)` | Builds validated `croco.test-evidence/v1` records and derives flaky outcomes from retained attempts. |
-| `createTestEvidenceBundle(records, artifactExists)` | Deterministically aggregates runner-neutral evidence and reports every missing required attachment. |
-| `CrocoVitestEvidenceReporter` | Adapts Vitest results and retries into the common evidence model without replacing Vitest. |
-| `CrocoPlaywrightEvidenceReporter` | Adapts Playwright attempts, traces, screenshots, and reports without replacing Playwright. |
+| Helper | Purpose |
+| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
+| `createTestingApp(config)` | Creates an isolated `CrocoApp` with seeded test defaults and HTTP request helpers. |
+| `createTestKernel(config)` | Boots the real application bootstrap inside an isolated runtime scope and reports application or adapter fidelity as structured evidence. |
+| `fixedClock(initial)` / `seededIds(seed)` | Creates virtual time and deterministic IDs/random values for TestKernel-controlled retry, timeout, rate-limit, task, and provider fixtures. |
+| `createTestingHarness(app)` | Wraps an existing `CrocoApp` with the same request and contract helpers. |
+| `createEventTestingHarness(config)` | Creates an isolated in-memory event bus and dispatches decorated handlers. |
+| `createTestingRequestContext(config)` | Builds a deterministic request/runtime context for service tests. |
+| `runWithTestingContext(fn, config)` | Runs code inside Croco `Context` and clears AsyncLocalStorage state when execution completes. |
+| `createTestingTransactionContext(config)` | Provides explicit in-transaction and after-commit hook behavior for tests. |
+| `resetCrocoTestingContext()` | Resets the Croco DI container and seeds test logger/error/health defaults. |
+| `installTestingTelemetryCapture()` | Captures spans in memory without starting an SDK exporter. |
+| `createFailureDrillCatalog()` | Builds deterministic no-credential failure drills for provider timeout, duplicate delivery, outbox, telemetry, tenant, and quota failures. |
+| `runFailureDrills(cases)` | Executes failure drills and rejects runs that lack the expected Problem code, recovery action, telemetry evidence, or audit evidence. |
+| `createOperationalFailureDrillMatrix(cases)` | Validates the exact ordered operational incident matrix without changing the generic six-scenario catalog. |
+| `runOperationalFailureDrills(cases)` | Executes operational fixtures and verifies their Problem or diagnostic outcome, recovery action, and real-boundary provenance. |
+| `assertProblemResponse(response, expected)` | Verifies an RFC 7807 Problem Details response without depending on a test runner. |
+| `assertOpenAPIRoute(controllersOrSpec, expected)` | Verifies generated OpenAPI route metadata and response contracts. |
+| `createRpcTestFetch(app)` | Returns a fetch-compatible function that routes generated RPC clients into the in-memory app. |
+| `createAuthProviderConformanceSuite(config)` | Reusable auth provider cases for token/session auth, webhooks, tenant mapping, readiness, and live-smoke gating. |
+| `createStorageProviderConformanceSuite(config)` | Reusable storage provider contract cases for default no-credential CI. |
+| `createProviderConformanceMatrixSuite(config)` | Validates provider profile manifests for required capabilities, optional unsupported reasons, and method evidence. |
+| `createLlmProviderConformanceSuite(config)` | Reusable LLM provider contract cases for mocked or live provider fixtures. |
+| `createBillingProviderConformanceSuite(config)` | Builds runner-neutral billing gateway and webhook conformance cases for provider packages. |
+| `createUpstashRedisMeteringConformanceSuite(config)` | Reusable Upstash Redis metering cases for config, usage storage, idempotency, upstream errors, and live-smoke gating. |
+| `createUpstashRedisRateLimitConformanceSuite(config)` | Reusable Upstash Redis rate-limit cases for config, errors, refund idempotency, and live-smoke gating. |
+| `createQStashTaskConformanceSuite(config)` | Reusable QStash task publish cases for config, validation, idempotency, upstream errors, and live-smoke gating. |
+| `createQStashBatchConformanceSuite(config)` | Reusable QStash batch chunk cases for terminal chunks, continuation envelopes, upstream errors, and live-smoke gating. |
+| `createQStashTriggerConformanceSuite(config)` | Reusable QStash trigger cases for schedule sync, webhook verification, dispatch, upstream diagnostics, and live-smoke gating. |
+| `createDrizzleProviderConformanceSuite(config)` | Builds reusable Drizzle provider cases for schema, transaction, tenant, and error contracts. |
+| `assertDrizzleProblem(operation, expected)` | Verifies Drizzle provider failures surface stable Croco Problem codes, categories, or status. |
+| `createTestEvidenceRecord(input)` | Builds validated `croco.test-evidence/v1` records and derives flaky outcomes from retained attempts. |
+| `createTestEvidenceBundle(records, artifactExists)` | Deterministically aggregates runner-neutral evidence and reports every missing required attachment. |
+| `CrocoVitestEvidenceReporter` | Adapts Vitest results and retries into the common evidence model without replacing Vitest. |
+| `CrocoPlaywrightEvidenceReporter` | Adapts Playwright attempts, traces, screenshots, and reports without replacing Playwright. |
## Isolation Contract
@@ -84,6 +101,32 @@ cleanup once; cleanup failures reject with `TestKernelDisposalProblem`. A bootst
resources before it can return an app can call `context.onCleanup()` so failure-path cleanup is also
guaranteed.
+## Deterministic Runtime Controls
+
+Each `TestKernel` owns a `clock`, `ids`, `random`, `environment`, `network`, and `replay` record.
+The bootstrap callback receives the same controls, so production code can consume existing explicit
+dependency seams instead of requiring a test-only branch. For example, pass `context.retry` to
+retry backoff dependencies, `() => context.clock.now` to an
+`InMemoryIdempotencyStore`, and `() => context.clock.now.getTime()` plus `context.random.next` to
+an in-memory rate-limit store. Task timeout runners accept the same clock through
+`now: () => context.clock.now.getTime()` and schedule timeouts with
+`(callback, delayMs) => context.clock.schedule(callback, delayMs, "task-timeout")`.
+`TestClock.schedule()` is for Croco-owned scheduler boundaries; `advanceBy()` and `drain()` run
+only that queued work deterministically.
+
+Environment overrides are immutable per-kernel snapshots. They do not mutate `process.env`, which
+makes concurrent kernels safe and leaves the ambient environment unchanged. `network: "deny"` is
+the default and rejects calls made through `context.network.fetch()` with a stable host and recovery
+diagnostic. It intentionally does not intercept arbitrary `globalThis.fetch` calls. Use explicit
+runtime dependencies for provider calls rather than global monkey patches.
+
+`test.expectClean()` reports pending virtual scheduled work, explicit `test.track()` operations, and
+pending after-commit hooks with category and source evidence. Use `test.waitUntil()`,
+`test.trackEventHandler()`, `test.trackSpan()`, or `test.trackResource()` at the matching adapter
+boundary so those outstanding operations are included too. `test.replay` records the scenario ID,
+seed, and virtual time needed to reproduce a failing deterministic scenario. User-owned timers,
+promises, and network calls outside these explicit boundaries remain outside the virtual scheduler.
+
`createTestingApp`, `createEventTestingHarness`, and `resetCrocoTestingContext` reset the root Croco DI
container, install a silent logger, replace the health/error defaults, and seed an inactive
`TestingTransactionContext` unless `transactionContext: false` is passed. Request helpers execute
diff --git a/packages/testing/package.json b/packages/testing/package.json
index be729e26d..bda0dd612 100644
--- a/packages/testing/package.json
+++ b/packages/testing/package.json
@@ -68,6 +68,7 @@
},
"devDependencies": {
"@croco/protocols-rest": "workspace:*",
+ "@croco/retry-core": "workspace:*",
"tsup": "8.5.1",
"vitest": "4.0.16"
}
diff --git a/packages/testing/src/index.ts b/packages/testing/src/index.ts
index ab1bd2826..5d906ba0a 100644
--- a/packages/testing/src/index.ts
+++ b/packages/testing/src/index.ts
@@ -83,29 +83,25 @@ export {
TestKernelDisposedProblem,
TestKernelDisposalProblem,
TestKernelHttp,
+ TestKernelLeakProblem,
TestKernelResourceFidelityProblem,
TestKernelResourceNotFoundProblem,
TestKernelResourceRegistrationProblem,
TestKernelValidationProblem,
- type StartedTestResource,
- type TestKernelBootFidelity,
- type TestKernelBootstrapContext,
- type TestKernelBootstrapResult,
- type TestKernelEvidence,
- type TestKernelFidelity,
- type TestKernelOptions,
- type TestKernelResourceEvidence,
- type TestKernelResourceObligation,
- type TestKernelRuntime,
- type TestKernelValidationFidelity,
- type TestResource,
- type TestResourceDiagnostic,
- type TestResourceDiagnosticStage,
- type TestResourceFidelity,
- type TestResourceIsolation,
- type TestResourceMode,
- type TestResourceStartContext,
} from "./libs/TestKernel";
+export {
+ fixedClock,
+ seededIds,
+ TestClock,
+ TestRuntimeDrainProblem,
+ TestEnvironment,
+ TestIdSource,
+ TestKernelOutboundCallProblem,
+ TestNetwork,
+ TestRandomSource,
+ TestRuntime,
+ TestRuntimeConfigurationProblem,
+} from "./libs/TestRuntime";
export {
installTestingTelemetryCapture,
TestingTelemetryCapture,
@@ -287,3 +283,34 @@ export {
type OperationalFailureDrillScenario,
type OperationalFailureDrillScenarioId,
} from "./libs/operational-failure-drill";
+
+export type {
+ StartedTestResource,
+ TestKernelBootFidelity,
+ TestKernelBootstrapContext,
+ TestKernelBootstrapResult,
+ TestKernelEvidence,
+ TestKernelFidelity,
+ TestKernelLeak,
+ TestKernelOptions,
+ TestKernelResourceEvidence,
+ TestKernelResourceObligation,
+ TestKernelRuntime,
+ TestKernelTrackedWork,
+ TestKernelValidationFidelity,
+ TestResource,
+ TestResourceDiagnostic,
+ TestResourceDiagnosticStage,
+ TestResourceFidelity,
+ TestResourceIsolation,
+ TestResourceMode,
+ TestResourceStartContext,
+} from "./libs/TestKernel";
+export type {
+ TestDuration,
+ TestEnvironmentOverrides,
+ TestReplayMetadata,
+ TestRetryDependencies,
+ TestRuntimeOptions,
+ TestScheduledWork,
+} from "./libs/TestRuntime";
diff --git a/packages/testing/src/libs/TestKernel.ts b/packages/testing/src/libs/TestKernel.ts
index 8a335da45..98a4e73ff 100644
--- a/packages/testing/src/libs/TestKernel.ts
+++ b/packages/testing/src/libs/TestKernel.ts
@@ -20,6 +20,18 @@ import {
type TestingRequestOptions,
type TestingTransactionContext,
} from "./testing";
+import { TestIdSource, TestRuntime } from "./TestRuntime";
+import type {
+ TestClock,
+ TestEnvironment,
+ TestNetwork,
+ TestRandomSource,
+ TestEnvironmentOverrides,
+ TestReplayMetadata,
+ TestRetryDependencies,
+ TestRuntimeOptions,
+ TestScheduledWork,
+} from "./TestRuntime";
export type TestKernelBootstrapResult =
| CrocoApp
@@ -45,9 +57,36 @@ export type TestKernelEvidence = {
readonly status: number;
};
+export type TestKernelLeak = {
+ readonly category:
+ | "after-commit"
+ | "event-handler"
+ | "operation"
+ | "operation-failure"
+ | "resource"
+ | "scheduled-work"
+ | "span"
+ | "wait-until";
+ readonly failure?: {
+ readonly code?: string;
+ readonly message: string;
+ readonly name: string;
+ };
+ readonly source: string;
+};
+
+export type TestKernelTrackedWork = Pick;
+
export type TestKernelBootstrapContext = {
+ readonly clock: TestClock;
+ readonly environment: TestEnvironment;
readonly fidelity: TestKernelBootFidelity;
+ readonly ids: TestIdSource;
+ readonly network: TestNetwork;
readonly onCleanup: (cleanup: () => Promise | void) => void;
+ readonly random: TestRandomSource;
+ readonly replay: TestReplayMetadata;
+ readonly retry: TestRetryDependencies;
readonly runtime: TestKernelRuntime;
};
@@ -105,9 +144,15 @@ type TestKernelCommonOptions = {
context: TestKernelBootstrapContext,
) => Promise | TestKernelBootstrapResult;
readonly dispose?: (app: CrocoApp) => Promise | void;
+ readonly clock?: TestRuntimeOptions["clock"];
+ readonly environment?: TestEnvironmentOverrides;
+ readonly ids?: TestRuntimeOptions["ids"];
+ readonly network?: TestRuntimeOptions["network"];
readonly obligations?: readonly TestKernelResourceObligation[];
readonly resources?: readonly TestResource[];
+ readonly scenarioId?: string;
readonly testId?: string;
+ readonly transactionContext?: TestingTransactionContext;
readonly validation?: Partial;
readonly workerId?: string;
};
@@ -177,6 +222,17 @@ export class TestKernelDisposedProblem extends Problem {
}
}
+export class TestKernelLeakProblem extends Problem {
+ constructor(leaks: readonly TestKernelLeak[], replay: TestReplayMetadata) {
+ super(
+ "testing/test-kernel-leak",
+ ProblemCategory.InternalServerError,
+ `TestKernel detected ${leaks.length} pending work item(s). Drain the TestClock, await the operation, or flush after-commit hooks before cleanup.`,
+ { extensions: { leaks, replay } },
+ );
+ }
+}
+
export class TestKernelResourceFidelityProblem extends Problem {
constructor(obligation: TestKernelResourceObligation, fidelity: TestResourceFidelity) {
super(
@@ -254,16 +310,24 @@ export class TestKernelHttp {
}
export class TestKernel implements AsyncDisposable {
+ readonly clock: TestClock;
+ readonly environment: TestEnvironment;
readonly http: TestKernelHttp;
+ readonly ids: TestIdSource;
+ readonly network: TestNetwork;
+ readonly random: TestRandomSource;
+ readonly retry: TestRetryDependencies;
readonly transactionContext: TestingTransactionContext;
private disposal: Promise | undefined;
private disposed = false;
private readonly evidenceBuffer: TestKernelEvidence[] = [];
- private readonly inFlight = new Set>();
+ private readonly inFlight = new Map, TestKernelTrackedWork>();
+ private readonly trackedFailures: Array<{ error: Error; work: TestKernelTrackedWork }> = [];
constructor(
readonly app: CrocoApp,
readonly fidelity: TestKernelFidelity,
+ private readonly controls: TestRuntime,
private readonly scope: ContainerScope,
transactionContext: TestingTransactionContext,
private readonly baseUrl: string,
@@ -273,7 +337,13 @@ export class TestKernel implements AsyncDisposable {
private readonly resourceConnections: ReadonlyMap, unknown>,
private readonly resourceEvidenceBuffer: readonly TestKernelResourceEvidence[],
) {
+ this.clock = controls.clock;
+ this.environment = controls.environment;
this.http = new TestKernelHttp(this);
+ this.ids = controls.ids;
+ this.network = controls.network;
+ this.random = controls.random;
+ this.retry = controls.retry;
this.transactionContext = transactionContext;
}
@@ -281,6 +351,10 @@ export class TestKernel implements AsyncDisposable {
return [...this.evidenceBuffer];
}
+ get replay(): TestReplayMetadata {
+ return this.controls.replay;
+ }
+
get resourceEvidence(): readonly TestKernelResourceEvidence[] {
return [...this.resourceEvidenceBuffer];
}
@@ -290,7 +364,9 @@ export class TestKernel implements AsyncDisposable {
run(fn: () => Promise | T): Promise | T {
this.assertActive();
const result = this.scope.run(fn);
- return result instanceof Promise ? this.track(result) : result;
+ return result instanceof Promise
+ ? this.track(result, { category: "operation", source: "run" })
+ : result;
}
get(token: TokenIdentifier): T {
@@ -308,7 +384,38 @@ export class TestKernel implements AsyncDisposable {
request(path: string | URL | Request, options: TestingRequestOptions = {}): Promise {
this.assertActive();
- return this.track(this.dispatchRequest(path, options));
+ return this.track(this.dispatchRequest(path, options), {
+ category: "operation",
+ source: "request",
+ });
+ }
+
+ track(operation: Promise, work: TestKernelTrackedWork): Promise {
+ this.assertActive();
+ return this.trackInFlight(operation, work);
+ }
+
+ waitUntil(operation: Promise, source = "wait-until"): Promise {
+ return this.track(operation, { category: "wait-until", source });
+ }
+
+ trackEventHandler(operation: Promise, source: string): Promise {
+ return this.track(operation, { category: "event-handler", source });
+ }
+
+ trackResource(operation: Promise, source: string): Promise {
+ return this.track(operation, { category: "resource", source });
+ }
+
+ trackSpan(operation: Promise, source: string): Promise {
+ return this.track(operation, { category: "span", source });
+ }
+
+ expectClean(): void {
+ const leaks = this.collectLeaks();
+ if (leaks.length > 0) {
+ throw new TestKernelLeakProblem(leaks, this.replay);
+ }
}
private async dispatchRequest(
@@ -318,7 +425,7 @@ export class TestKernel implements AsyncDisposable {
const request = toRequest(path, options, this.baseUrl);
const response = await this.scope.run(async () =>
this.lambdaHandler
- ? dispatchLambdaRequest(this.lambdaHandler, request)
+ ? dispatchLambdaRequest(this.lambdaHandler, request, this)
: this.nodeHandler
? this.nodeHandler(request)
: this.app.fetch(request, { platform: "node" }),
@@ -345,9 +452,13 @@ export class TestKernel implements AsyncDisposable {
}
private async disposeOnce(): Promise {
- await Promise.allSettled(this.inFlight);
const failures = await runCleanupSequence(this.scope, this.cleanupOperations);
+ const leaks = this.collectLeaks();
+ if (leaks.length > 0) {
+ failures.unshift(new TestKernelLeakProblem(leaks, this.replay));
+ }
+
if (failures.length > 0) {
throw new TestKernelDisposalProblem(failures);
}
@@ -363,11 +474,37 @@ export class TestKernel implements AsyncDisposable {
}
}
- private track(operation: Promise): Promise {
- this.inFlight.add(operation);
+ private collectLeaks(): TestKernelLeak[] {
+ const operationLeaks = [...this.inFlight.values()];
+ const operationFailures = this.trackedFailures.map(({ error, work }) => ({
+ category: "operation-failure" as const,
+ failure: {
+ ...(error instanceof Problem ? { code: error.code } : {}),
+ message: error.message,
+ name: error.name,
+ },
+ source: work.source,
+ }));
+ const scheduledLeaks = this.clock.pendingWork.map((work: TestScheduledWork) => ({
+ category: "scheduled-work" as const,
+ source: work.source,
+ }));
+ const afterCommitCount = this.transactionContext.getPendingAfterCommitHookCount();
+ const afterCommitLeaks = Array.from({ length: afterCommitCount }, () => ({
+ category: "after-commit" as const,
+ source: "transaction-context",
+ }));
+ return [...operationLeaks, ...operationFailures, ...scheduledLeaks, ...afterCommitLeaks];
+ }
+
+ private trackInFlight(operation: Promise, work: TestKernelTrackedWork): Promise {
+ this.inFlight.set(operation, work);
void operation.then(
() => this.inFlight.delete(operation),
- () => this.inFlight.delete(operation),
+ (error: unknown) => {
+ this.inFlight.delete(operation);
+ this.trackedFailures.push({ error: toError(error), work });
+ },
);
return operation;
}
@@ -376,19 +513,40 @@ export class TestKernel implements AsyncDisposable {
export async function createTestKernel(options: TestKernelOptions): Promise {
const scope = Container.createScope();
const runtime = options.fidelity === "adapter" ? (options.adapter ?? "node") : "node";
+ const runtimeOptions: TestRuntimeOptions = {
+ ...(options.clock === undefined ? {} : { clock: options.clock }),
+ ...(options.environment === undefined ? {} : { environment: options.environment }),
+ ...(options.ids === undefined
+ ? {}
+ : {
+ ids: options.ids instanceof TestIdSource ? options.ids.fork() : options.ids,
+ }),
+ ...(options.network === undefined ? {} : { network: options.network }),
+ ...(options.scenarioId === undefined ? {} : { scenarioId: options.scenarioId }),
+ };
+ const controls = new TestRuntime(runtimeOptions);
const registeredCleanups: Array<() => Promise | void> = [];
const resourceCleanups: Array<() => Promise | void> = [];
- const testId = options.testId ?? `test-${crypto.randomUUID()}`;
+ const testId = options.testId ?? controls.ids.next("test");
const workerId =
options.workerId ??
process.env["VITEST_POOL_ID"] ??
process.env["CI_NODE_INDEX"] ??
`process-${process.pid}`;
const fidelityContext: TestKernelBootstrapContext = {
+ clock: controls.clock,
+ environment: controls.environment,
fidelity: options.fidelity,
+ ids: controls.ids,
+ network: controls.network,
onCleanup(cleanup) {
registeredCleanups.push(cleanup);
},
+ random: controls.random,
+ get replay() {
+ return controls.replay;
+ },
+ retry: controls.retry,
runtime,
};
let app: CrocoApp | undefined;
@@ -408,7 +566,7 @@ export async function createTestKernel(options: TestKernelOptions): Promise {
EventBusConfig.setInstance(new EventBusConfig());
ShutdownManager.getInstance();
- const transactionContext = createTestingTransactionContext();
+ const transactionContext = options.transactionContext ?? createTestingTransactionContext();
const resources = options.resources ?? [];
const resourceIds = new Set();
@@ -489,6 +647,7 @@ export async function createTestKernel(options: TestKernelOptions): Promise {
+async function dispatchLambdaRequest(
+ handler: LambdaHandler,
+ request: Request,
+ controls: Pick,
+): Promise {
const url = new URL(request.url);
const headers = Object.fromEntries(request.headers.entries());
const body =
@@ -589,16 +752,16 @@ async function dispatchLambdaRequest(handler: LambdaHandler, request: Request):
sourceIp: "127.0.0.1",
userAgent: headers["user-agent"] ?? "croco-test-kernel",
},
- requestId: `test-kernel-${crypto.randomUUID()}`,
+ requestId: controls.ids.next("lambda-request"),
routeKey: `${request.method} ${url.pathname}`,
stage: "$default",
- time: new Date(0).toUTCString(),
- timeEpoch: 0,
+ time: controls.clock.now.toUTCString(),
+ timeEpoch: controls.clock.now.getTime(),
},
isBase64Encoded: false,
...(body === undefined ? {} : { body }),
};
- const lambdaResponse = await handler(event, createLambdaContext());
+ const lambdaResponse = await handler(event, createLambdaContext(controls.ids));
const responseHeaders = new Headers(lambdaResponse.headers);
for (const cookie of lambdaResponse.cookies ?? []) {
responseHeaders.append("set-cookie", cookie);
@@ -615,14 +778,14 @@ async function dispatchLambdaRequest(handler: LambdaHandler, request: Request):
});
}
-function createLambdaContext(): LambdaContext {
+function createLambdaContext(ids: TestIdSource): LambdaContext {
return {
callbackWaitsForEmptyEventLoop: false,
functionName: "croco-test-kernel",
functionVersion: "$LATEST",
invokedFunctionArn: "arn:aws:lambda:local:0:function:croco-test-kernel",
memoryLimitInMB: "128",
- awsRequestId: crypto.randomUUID(),
+ awsRequestId: ids.next("lambda"),
logGroupName: "/aws/lambda/croco-test-kernel",
logStreamName: "test",
getRemainingTimeInMillis: () => 30_000,
diff --git a/packages/testing/src/libs/TestRuntime.ts b/packages/testing/src/libs/TestRuntime.ts
new file mode 100644
index 000000000..21cb98fbc
--- /dev/null
+++ b/packages/testing/src/libs/TestRuntime.ts
@@ -0,0 +1,364 @@
+import { Problem, ProblemCategory } from "@croco/problems-core";
+import type { BackoffDependencies } from "@croco/retry-core";
+
+export type TestDuration = number | `${number}${"ms" | "s" | "m"}`;
+
+export type TestScheduledWork = {
+ readonly dueAt: string;
+ readonly id: string;
+ readonly source: string;
+};
+
+export type TestReplayMetadata = {
+ readonly scenarioId: string;
+ readonly seed: string;
+ readonly virtualTime: string;
+};
+
+export type TestRetryDependencies = Required;
+
+export type TestEnvironmentOverrides = Readonly>;
+
+export type TestRuntimeOptions = {
+ readonly clock?: Date | string | TestClock;
+ readonly environment?: TestEnvironmentOverrides;
+ readonly ids?: string | TestIdSource;
+ readonly network?: "allow" | "deny";
+ readonly scenarioId?: string;
+};
+
+type ScheduledCallback = () => void | Promise;
+
+type ScheduledEntry = {
+ readonly callback: ScheduledCallback;
+ readonly dueAtMs: number;
+ readonly id: string;
+ readonly sequence: number;
+ readonly source: string;
+};
+
+const MAX_DRAIN_CALLBACKS = 10_000;
+const MAX_DATE_TIME_MS = 8_640_000_000_000_000;
+
+type DrainBudget = {
+ callbackCount: number;
+};
+
+export class TestKernelOutboundCallProblem extends Problem {
+ constructor(host: string) {
+ super(
+ "testing/test-kernel-outbound-call",
+ ProblemCategory.InternalServerError,
+ `TestKernel blocked an outbound call to '${host}'. Register a provider fake or set network: 'allow' when this call is intentional.`,
+ {
+ extensions: {
+ host,
+ recovery: "Register a provider fake or explicitly allow this outbound call.",
+ },
+ },
+ );
+ }
+}
+
+export class TestRuntimeConfigurationProblem extends Problem {
+ constructor(field: "clock" | "duration", value: string, expectation: string) {
+ super(
+ "testing/test-runtime-configuration",
+ ProblemCategory.ValidationError,
+ `Test runtime '${field}' must be ${expectation}; received '${value}'.`,
+ { extensions: { expectation, field, value } },
+ );
+ }
+}
+
+export class TestRuntimeDrainProblem extends Problem {
+ constructor(limit: number) {
+ super(
+ "testing/test-runtime-drain-limit",
+ ProblemCategory.InternalServerError,
+ `Test runtime drain exceeded ${limit} scheduled callbacks without reaching an idle state.`,
+ {
+ extensions: {
+ limit,
+ recovery:
+ "Cancel the rescheduling callback or schedule its next run after the current virtual time.",
+ },
+ },
+ );
+ }
+}
+
+export class TestClock {
+ private currentTimeMs: number;
+ private scheduledSequence = 0;
+ private readonly scheduled = new Map();
+
+ constructor(initial: Date | string = "2026-01-01T00:00:00.000Z") {
+ const time = new Date(initial).getTime();
+ if (!isValidDateTime(time)) {
+ throw new TestRuntimeConfigurationProblem("clock", String(initial), "a valid date");
+ }
+ this.currentTimeMs = time;
+ }
+
+ get now(): Date {
+ return new Date(this.currentTimeMs);
+ }
+
+ get pendingWork(): readonly TestScheduledWork[] {
+ return [...this.scheduled.values()]
+ .sort((left, right) => left.dueAtMs - right.dueAtMs || left.sequence - right.sequence)
+ .map(({ dueAtMs, id, source }) => ({ dueAt: new Date(dueAtMs).toISOString(), id, source }));
+ }
+
+ schedule(
+ callback: ScheduledCallback,
+ delay: TestDuration,
+ source = "scheduled-work",
+ ): () => void {
+ const dueAtMs = nextDateTime(this.currentTimeMs, parseDuration(delay));
+ const sequence = ++this.scheduledSequence;
+ const id = `scheduled-${sequence}`;
+ this.scheduled.set(id, {
+ callback,
+ dueAtMs,
+ id,
+ sequence,
+ source,
+ });
+ return () => this.scheduled.delete(id);
+ }
+
+ sleep(delay: TestDuration, source = "sleep"): Promise {
+ return new Promise((resolve) => {
+ this.schedule(resolve, delay, source);
+ });
+ }
+
+ async advanceBy(duration: TestDuration): Promise {
+ const targetTime = nextDateTime(this.currentTimeMs, parseDuration(duration));
+ await this.drainUntil(targetTime);
+ this.currentTimeMs = targetTime;
+ }
+
+ async drain(): Promise {
+ const budget: DrainBudget = { callbackCount: 0 };
+ while (this.scheduled.size > 0) {
+ const next = this.nextScheduled();
+ if (!next) return;
+ await this.drainUntil(next.dueAtMs, budget);
+ }
+ }
+
+ private async drainUntil(
+ targetTime: number,
+ budget: DrainBudget = { callbackCount: 0 },
+ ): Promise {
+ while (true) {
+ const next = this.nextScheduled();
+ if (!next || next.dueAtMs > targetTime) return;
+
+ budget.callbackCount += 1;
+ if (budget.callbackCount > MAX_DRAIN_CALLBACKS) {
+ throw new TestRuntimeDrainProblem(MAX_DRAIN_CALLBACKS);
+ }
+
+ this.scheduled.delete(next.id);
+ this.currentTimeMs = next.dueAtMs;
+ await next.callback();
+ }
+ }
+
+ private nextScheduled(): ScheduledEntry | undefined {
+ return [...this.scheduled.values()].sort(
+ (left, right) => left.dueAtMs - right.dueAtMs || left.sequence - right.sequence,
+ )[0];
+ }
+}
+
+export class TestRandomSource {
+ private state: number;
+
+ constructor(readonly seed: string) {
+ this.state = hashSeed(seed) || 1;
+ }
+
+ next(): number {
+ this.state ^= this.state << 13;
+ this.state ^= this.state >>> 17;
+ this.state ^= this.state << 5;
+ return (this.state >>> 0) / 0x1_0000_0000;
+ }
+
+ fork(): TestRandomSource {
+ const fork = new TestRandomSource(this.seed);
+ fork.state = this.state;
+ return fork;
+ }
+}
+
+export class TestIdSource {
+ private sequence = 0;
+ private randomSource: TestRandomSource;
+
+ constructor(readonly seed: string) {
+ this.randomSource = new TestRandomSource(seed);
+ }
+
+ get random(): TestRandomSource {
+ return this.randomSource;
+ }
+
+ fork(): TestIdSource {
+ const fork = new TestIdSource(this.seed);
+ fork.sequence = this.sequence;
+ fork.randomSource = this.random.fork();
+ return fork;
+ }
+
+ next(prefix = "test"): string {
+ this.sequence += 1;
+ const entropy = Math.floor(this.random.next() * 0x1_0000_0000)
+ .toString(36)
+ .padStart(7, "0");
+ return `${prefix}-${this.seed}-${this.sequence}-${entropy}`;
+ }
+}
+
+export class TestEnvironment {
+ private readonly values: Readonly>;
+
+ constructor(overrides: TestEnvironmentOverrides = {}) {
+ this.values = Object.freeze({ ...process.env, ...overrides });
+ }
+
+ get(name: string): string | undefined {
+ return this.values[name];
+ }
+
+ toObject(): Readonly> {
+ return { ...this.values };
+ }
+}
+
+export class TestNetwork {
+ constructor(private readonly mode: "allow" | "deny") {}
+
+ async fetch(input: string | URL | Request, init?: RequestInit): Promise {
+ const url = input instanceof Request ? input.url : input.toString();
+ if (this.mode === "deny") {
+ let host: string;
+ try {
+ host = new URL(url).host;
+ } catch {
+ throw new TestKernelOutboundCallProblem(url);
+ }
+ throw new TestKernelOutboundCallProblem(host);
+ }
+ return fetch(input, init);
+ }
+}
+
+export class TestRuntime {
+ readonly clock: TestClock;
+ readonly environment: TestEnvironment;
+ readonly ids: TestIdSource;
+ readonly network: TestNetwork;
+ readonly random: TestRandomSource;
+ readonly scenarioId: string;
+
+ constructor(options: TestRuntimeOptions = {}) {
+ this.ids =
+ typeof options.ids === "string"
+ ? new TestIdSource(options.ids)
+ : options.ids
+ ? options.ids
+ : new TestIdSource(crypto.randomUUID());
+ this.clock =
+ options.clock instanceof TestClock
+ ? new TestClock(options.clock.now)
+ : new TestClock(options.clock);
+ this.environment = new TestEnvironment(options.environment);
+ this.network = new TestNetwork(options.network ?? "deny");
+ this.random = this.ids.random;
+ this.scenarioId = options.scenarioId ?? this.ids.next("scenario");
+ }
+
+ get replay(): TestReplayMetadata {
+ return Object.freeze({
+ scenarioId: this.scenarioId,
+ seed: this.ids.seed,
+ virtualTime: this.clock.now.toISOString(),
+ });
+ }
+
+ get retry(): TestRetryDependencies {
+ return {
+ random: () => this.random.next(),
+ sleep: (delayMs: number) => this.clock.sleep(delayMs, "retry:backoff"),
+ };
+ }
+}
+
+export function fixedClock(initial: Date | string): TestClock {
+ return new TestClock(initial);
+}
+
+export function seededIds(seed: string): TestIdSource {
+ return new TestIdSource(seed);
+}
+
+function parseDuration(value: TestDuration): number {
+ if (typeof value === "number") {
+ if (!Number.isSafeInteger(value) || value < 0) {
+ throw new TestRuntimeConfigurationProblem(
+ "duration",
+ String(value),
+ "a non-negative safe integer",
+ );
+ }
+ return value;
+ }
+
+ const match = /^(\d+)(ms|s|m)$/.exec(value);
+ if (!match) {
+ throw new TestRuntimeConfigurationProblem("duration", String(value), "an ms, s, or m duration");
+ }
+ const amount = Number(match[1]);
+ const unit = match[2];
+ const multiplier = unit === "m" ? 60_000 : unit === "s" ? 1_000 : 1;
+ const duration = amount * multiplier;
+ if (!Number.isSafeInteger(duration)) {
+ throw new TestRuntimeConfigurationProblem(
+ "duration",
+ String(value),
+ "a non-negative safe integer duration",
+ );
+ }
+ return duration;
+}
+
+function nextDateTime(currentTimeMs: number, duration: number): number {
+ const nextTimeMs = currentTimeMs + duration;
+ if (!isValidDateTime(nextTimeMs)) {
+ throw new TestRuntimeConfigurationProblem(
+ "duration",
+ String(duration),
+ "a duration that keeps virtual time within the valid Date range",
+ );
+ }
+ return nextTimeMs;
+}
+
+function isValidDateTime(value: number): boolean {
+ return Number.isSafeInteger(value) && Math.abs(value) <= MAX_DATE_TIME_MS;
+}
+
+function hashSeed(seed: string): number {
+ let hash = 2_166_136_261;
+ for (const character of seed) {
+ hash ^= character.charCodeAt(0);
+ hash = Math.imul(hash, 16_777_619);
+ }
+ return hash >>> 0;
+}
diff --git a/packages/testing/src/tests/TestKernel.spec.ts b/packages/testing/src/tests/TestKernel.spec.ts
index 885d45d0c..6d79424eb 100644
--- a/packages/testing/src/tests/TestKernel.spec.ts
+++ b/packages/testing/src/tests/TestKernel.spec.ts
@@ -10,15 +10,19 @@ import {
import { beforeEach, describe, expect, it } from "vitest";
import {
createTestKernel,
- type TestKernel,
+ createTestingTransactionContext,
+ fixedClock,
+ seededIds,
TestKernelDisposalProblem,
TestKernelDisposedProblem,
+ TestKernelLeakProblem,
type TestKernelOptions,
TestKernelResourceFidelityProblem,
TestKernelResourceNotFoundProblem,
TestKernelValidationProblem,
type TestResource,
} from "../index";
+import type { TestKernel } from "../index";
class KernelValueService {
constructor(readonly value: string) {}
@@ -587,8 +591,8 @@ describe("TestKernel", () => {
const secondDisposal = kernel.dispose();
expect(firstDisposal).toBe(secondDisposal);
- await expect(firstDisposal).rejects.toBeInstanceOf(TestKernelDisposalProblem);
- await expect(secondDisposal).rejects.toBeInstanceOf(TestKernelDisposalProblem);
+ await expect(firstDisposal).rejects.toThrow(TestKernelDisposalProblem);
+ await expect(secondDisposal).rejects.toThrow(TestKernelDisposalProblem);
expect(cleanupCalls).toBe(1);
expect(fallbackCleanupCalls).toBe(1);
});
@@ -610,11 +614,7 @@ describe("TestKernel", () => {
expect(() => kernelRef.run(() => undefined)).toThrow(TestKernelDisposedProblem);
});
- it("waits for in-flight kernel work before cleanup", async () => {
- let resumeOperation!: () => void;
- const resume = new Promise((resolve) => {
- resumeOperation = resolve;
- });
+ it("reports unresolved tracked work without hanging cleanup", async () => {
let cleanupCalls = 0;
const kernel = await createTestKernel({
bootstrap: () => bootstrapProductionApp("in-flight"),
@@ -623,19 +623,40 @@ describe("TestKernel", () => {
},
fidelity: "application",
});
- const operation = kernel.run(async () => {
- await resume;
- return Container.get(KernelValueService).value;
+
+ kernel.waitUntil(new Promise(() => undefined), "response.flush");
+
+ await expect(kernel.dispose()).rejects.toThrow(TestKernelDisposalProblem);
+ expect(cleanupCalls).toBe(1);
+ });
+
+ it("preserves rejected tracked work as leak evidence", async () => {
+ const kernel = await createTestKernel({
+ bootstrap: () => bootstrapProductionApp("tracked-rejection"),
+ fidelity: "application",
});
- const disposal = kernel.dispose();
+ kernel.waitUntil(Promise.reject(new Error("response flush failed")), "response.flush");
await Promise.resolve();
- expect(cleanupCalls).toBe(0);
- resumeOperation();
- await expect(operation).resolves.toBe("in-flight");
- await disposal;
- expect(cleanupCalls).toBe(1);
+ let error: unknown;
+ try {
+ kernel.expectClean();
+ } catch (caught) {
+ error = caught;
+ }
+ expect(error).toMatchObject({
+ extensions: {
+ leaks: expect.arrayContaining([
+ {
+ category: "operation-failure",
+ failure: { message: "response flush failed", name: "Error" },
+ source: "response.flush",
+ },
+ ]),
+ },
+ });
+ await expect(kernel.dispose()).rejects.toThrow(TestKernelDisposalProblem);
});
it("isolates and executes production shutdown hooks for each kernel", async () => {
@@ -688,6 +709,148 @@ describe("TestKernel", () => {
fidelity: "application",
});
- await expect(kernel.dispose()).rejects.toBeInstanceOf(TestKernelDisposalProblem);
+ await expect(kernel.dispose()).rejects.toThrow(TestKernelDisposalProblem);
+ });
+
+ it("injects deterministic runtime controls and reports replay metadata", async () => {
+ let bootstrapReplay:
+ | {
+ readonly scenarioId: string;
+ readonly seed: string;
+ readonly virtualTime: string;
+ }
+ | undefined;
+ const kernel = await createTestKernel({
+ bootstrap: async (context) => {
+ await context.clock.advanceBy("1m");
+ bootstrapReplay = context.replay;
+ expect(context.environment.get("CROCO_TEST_KERNEL_SCOPE")).toBe("first");
+ expect(context.ids.next("bootstrap")).toContain("retry-scenario");
+ return bootstrapProductionApp("controls");
+ },
+ clock: "2026-02-03T04:05:06.000Z",
+ environment: { CROCO_TEST_KERNEL_SCOPE: "first" },
+ fidelity: "application",
+ ids: "retry-scenario",
+ scenarioId: "retry-timeout",
+ });
+
+ expect(kernel.replay).toBeDefined();
+ expect(bootstrapReplay).toBeDefined();
+ expect(bootstrapReplay).toEqual({
+ scenarioId: "retry-timeout",
+ seed: "retry-scenario",
+ virtualTime: "2026-02-03T04:06:06.000Z",
+ });
+ expect(kernel.replay).toEqual(bootstrapReplay);
+ expect(process.env["CROCO_TEST_KERNEL_SCOPE"]).not.toBe("first");
+
+ await kernel.dispose();
+ });
+
+ it("clones caller-provided controls so concurrent kernels do not share time or ID state", async () => {
+ const sharedClock = fixedClock("2026-01-01T00:00:00.000Z");
+ const sharedIds = seededIds("concurrent-kernels");
+ sharedIds.next("already-used");
+ const expectedIds = sharedIds.fork();
+ expectedIds.next("scenario");
+ expectedIds.next("test");
+ const [first, second] = await Promise.all(
+ ["first", "second"].map((value) =>
+ createTestKernel({
+ bootstrap: () => bootstrapProductionApp(value),
+ clock: sharedClock,
+ fidelity: "application",
+ ids: sharedIds,
+ }),
+ ),
+ );
+
+ await first.clock.advanceBy("30s");
+ const firstExtraId = first.ids.next("extra");
+ const expectedExtraId = expectedIds.next("extra");
+
+ expect(first.clock.now.toISOString()).toBe("2026-01-01T00:00:30.000Z");
+ expect(second.clock.now.toISOString()).toBe("2026-01-01T00:00:00.000Z");
+ expect(firstExtraId).toBe(expectedExtraId);
+ expect(second.ids.next("extra")).toBe(firstExtraId);
+
+ await Promise.all([first.dispose(), second.dispose()]);
+ });
+
+ it("reports pending scheduled work and after-commit hooks with stable leak evidence", async () => {
+ const kernel = await createTestKernel({
+ bootstrap: () => bootstrapProductionApp("leaks"),
+ fidelity: "application",
+ transactionContext: createTestingTransactionContext({ inTransaction: true }),
+ });
+
+ kernel.clock.schedule(() => undefined, "30s", "retry:payment");
+ await kernel.transactionContext.runInTransaction(() => {
+ kernel.transactionContext.onAfterCommit(() => undefined);
+ });
+
+ expect(() => kernel.expectClean()).toThrow(TestKernelLeakProblem);
+ try {
+ kernel.expectClean();
+ } catch (error) {
+ expect(error).toMatchObject({
+ code: "testing/test-kernel-leak",
+ extensions: {
+ leaks: expect.arrayContaining([
+ { category: "scheduled-work", source: "retry:payment" },
+ { category: "after-commit", source: "transaction-context" },
+ ]),
+ },
+ });
+ }
+
+ await kernel.clock.advanceBy("30s");
+ let advancedLeak: unknown;
+ try {
+ kernel.expectClean();
+ } catch (error) {
+ advancedLeak = error;
+ }
+ expect(advancedLeak).toMatchObject({
+ extensions: {
+ replay: {
+ scenarioId: expect.any(String),
+ seed: expect.any(String),
+ virtualTime: "2026-01-01T00:00:30.000Z",
+ },
+ },
+ });
+ await kernel.transactionContext.flushAfterCommitHooks();
+ kernel.expectClean();
+ await kernel.dispose();
+ });
+
+ it("reports tracked adapter work by its runtime boundary", async () => {
+ const kernel = await createTestKernel({
+ bootstrap: () => bootstrapProductionApp("tracked-boundaries"),
+ fidelity: "application",
+ });
+
+ kernel.trackEventHandler(new Promise(() => undefined), "events:payment.created");
+ kernel.trackSpan(new Promise(() => undefined), "telemetry:payment.create");
+ kernel.trackResource(new Promise(() => undefined), "postgres:connection");
+
+ let error: unknown;
+ try {
+ kernel.expectClean();
+ } catch (caught) {
+ error = caught;
+ }
+ expect(error).toMatchObject({
+ extensions: {
+ leaks: expect.arrayContaining([
+ { category: "event-handler", source: "events:payment.created" },
+ { category: "span", source: "telemetry:payment.create" },
+ { category: "resource", source: "postgres:connection" },
+ ]),
+ },
+ });
+ await expect(kernel.dispose()).rejects.toThrow(TestKernelDisposalProblem);
});
});
diff --git a/packages/testing/src/tests/TestRuntime.spec.ts b/packages/testing/src/tests/TestRuntime.spec.ts
new file mode 100644
index 000000000..35738f685
--- /dev/null
+++ b/packages/testing/src/tests/TestRuntime.spec.ts
@@ -0,0 +1,204 @@
+import { beforeEach, describe, expect, it } from "vitest";
+import { createSlidingWindowPolicy, SlidingWindowInMemoryStore } from "@croco/ratelimit-core";
+import { ExponentialBackoff } from "@croco/retry-core";
+import {
+ fixedClock,
+ resetCrocoTestingContext,
+ seededIds,
+ TestKernelOutboundCallProblem,
+ TestRuntime,
+ TestRuntimeConfigurationProblem,
+ TestRuntimeDrainProblem,
+} from "../index";
+
+describe("TestRuntime", () => {
+ beforeEach(() => {
+ resetCrocoTestingContext();
+ });
+
+ it("advances virtual time and drains only Croco-owned scheduled work in order", async () => {
+ const clock = fixedClock("2026-01-01T00:00:00.000Z");
+ const events: string[] = [];
+ clock.schedule(
+ () => {
+ events.push("late");
+ },
+ "30s",
+ "retry:late",
+ );
+ clock.schedule(
+ () => {
+ events.push("early");
+ clock.schedule(
+ () => {
+ events.push("nested");
+ },
+ "5s",
+ "retry:nested",
+ );
+ },
+ "10s",
+ "retry:early",
+ );
+
+ await clock.advanceBy("15s");
+
+ expect(events).toEqual(["early", "nested"]);
+ expect(clock.now.toISOString()).toBe("2026-01-01T00:00:15.000Z");
+ expect(clock.pendingWork).toEqual([
+ {
+ dueAt: "2026-01-01T00:00:30.000Z",
+ id: "scheduled-1",
+ source: "retry:late",
+ },
+ ]);
+ });
+
+ it("preserves insertion order for work scheduled at the same virtual time", async () => {
+ const clock = fixedClock("2026-01-01T00:00:00.000Z");
+ const events: string[] = [];
+
+ for (let index = 0; index < 12; index += 1) {
+ clock.schedule(
+ () => {
+ events.push(String(index));
+ },
+ 0,
+ "same-time",
+ );
+ }
+
+ await clock.drain();
+
+ expect(events).toEqual(Array.from({ length: 12 }, (_, index) => String(index)));
+ });
+
+ it("reports non-terminating same-time rescheduling with a stable Problem", async () => {
+ const clock = fixedClock("2026-01-01T00:00:00.000Z");
+ const reschedule = () => {
+ clock.schedule(reschedule, 0, "same-time-loop");
+ };
+ clock.schedule(reschedule, 0, "same-time-loop");
+
+ await expect(clock.drain()).rejects.toThrow(TestRuntimeDrainProblem);
+ });
+
+ it("reports non-terminating positive-delay rescheduling with a stable Problem", async () => {
+ const clock = fixedClock("2026-01-01T00:00:00.000Z");
+ const reschedule = () => {
+ clock.schedule(reschedule, 1, "positive-delay-loop");
+ };
+ clock.schedule(reschedule, 1, "positive-delay-loop");
+
+ await expect(clock.drain()).rejects.toThrow(TestRuntimeDrainProblem);
+ });
+
+ it("replays seeded ids and random values without touching process state", () => {
+ const first = seededIds("invitation-retry");
+ const second = seededIds("invitation-retry");
+ const advancedIds = seededIds("invitation-retry");
+ advancedIds.next("already-used");
+ const expectedIds = seededIds("invitation-retry");
+ expectedIds.next("already-used");
+ const environmentBefore = process.env["CROCO_TEST_RUNTIME_FIXTURE"];
+ const runtime = new TestRuntime({
+ environment: { CROCO_TEST_RUNTIME_FIXTURE: "scoped" },
+ ids: "invitation-retry",
+ scenarioId: "retry-replay",
+ });
+
+ expect(first.next("invitation")).toBe(second.next("invitation"));
+ expect(first.random.next()).toBe(second.random.next());
+ expect(runtime.environment.get("CROCO_TEST_RUNTIME_FIXTURE")).toBe("scoped");
+ expect(process.env["CROCO_TEST_RUNTIME_FIXTURE"]).toBe(environmentBefore);
+ expect(runtime.replay).toEqual({
+ scenarioId: "retry-replay",
+ seed: "invitation-retry",
+ virtualTime: "2026-01-01T00:00:00.000Z",
+ });
+
+ const runtimeWithAdvancedIds = new TestRuntime({ ids: advancedIds });
+
+ expect(runtimeWithAdvancedIds.scenarioId).toBe(expectedIds.next("scenario"));
+ expect(runtimeWithAdvancedIds.ids).toBe(advancedIds);
+ });
+
+ it("forks seeded ID sources without resetting their sequence or entropy", () => {
+ const ids = seededIds("forked-ids");
+ ids.next("already-used");
+ const fork = ids.fork();
+
+ expect(fork.next("next")).toBe(ids.next("next"));
+ });
+
+ it("reports invalid time controls with a stable Problem code", async () => {
+ expect(() => fixedClock("not-a-date")).toThrow(TestRuntimeConfigurationProblem);
+ expect(() => fixedClock("2026-01-01T00:00:00.000Z").schedule(() => undefined, -1)).toThrow(
+ TestRuntimeConfigurationProblem,
+ );
+ expect(() => fixedClock(new Date(8_640_000_000_000_000)).schedule(() => undefined, 1)).toThrow(
+ TestRuntimeConfigurationProblem,
+ );
+ await expect(
+ fixedClock("2026-01-01T00:00:00.000Z").advanceBy(Number.MAX_SAFE_INTEGER),
+ ).rejects.toThrow(TestRuntimeConfigurationProblem);
+ });
+
+ it("rejects outbound calls by default with a provider-facing diagnostic", async () => {
+ const runtime = new TestRuntime({ network: "deny" });
+
+ await expect(
+ runtime.network.fetch("https://provider.example.test/v1/send"),
+ ).rejects.toMatchObject({
+ code: "testing/test-kernel-outbound-call",
+ extensions: {
+ host: "provider.example.test",
+ recovery: "Register a provider fake or explicitly allow this outbound call.",
+ },
+ });
+ await expect(runtime.network.fetch("not a url")).rejects.toThrow(TestKernelOutboundCallProblem);
+ });
+
+ it("supplies virtual time and seeded entropy to rate-limit boundaries without global timers", async () => {
+ const runtime = new TestRuntime({ ids: "rate-limit", network: "deny" });
+ const store = new SlidingWindowInMemoryStore({
+ now: () => runtime.clock.now.getTime(),
+ pruneIntervalMs: 30_000,
+ random: () => runtime.random.next(),
+ scheduler: {
+ schedule: (callback, delayMs) =>
+ runtime.clock.schedule(callback, delayMs, "rate-limit:prune"),
+ },
+ });
+ const policy = createSlidingWindowPolicy("test-runtime", 1, 30_000);
+
+ await expect(store.check("tenant:one", policy)).resolves.toMatchObject({
+ remaining: 0,
+ success: true,
+ });
+ await expect(store.check("tenant:one", policy)).resolves.toMatchObject({ success: false });
+
+ await runtime.clock.advanceBy(30_001);
+
+ expect(runtime.clock.pendingWork).toMatchObject([{ source: "rate-limit:prune" }]);
+
+ await expect(store.check("tenant:one", policy)).resolves.toMatchObject({
+ remaining: 0,
+ success: true,
+ });
+ store.close();
+ expect(runtime.clock.pendingWork).toEqual([]);
+ });
+
+ it("drains retry-core backoff through kernel-owned virtual time", async () => {
+ const runtime = new TestRuntime({ ids: "retry-replay" });
+ const retry = new ExponentialBackoff({ delay: 30_000, jitter: false }, runtime.retry);
+ const wait = retry.wait(0);
+
+ expect(runtime.clock.pendingWork).toMatchObject([{ source: "retry:backoff" }]);
+
+ await runtime.clock.advanceBy("30s");
+
+ await expect(wait).resolves.toBeUndefined();
+ });
+});
diff --git a/packages/testing/tsconfig.json b/packages/testing/tsconfig.json
index 01aa2f479..d593c2efd 100644
--- a/packages/testing/tsconfig.json
+++ b/packages/testing/tsconfig.json
@@ -16,6 +16,7 @@
"@croco/protocols-core": ["../protocols-core/src/index.ts"],
"@croco/protocols-rest": ["../protocols-rest/src/index.ts"],
"@croco/ratelimit-core": ["../ratelimit-core/src/index.ts"],
+ "@croco/retry-core": ["../retry-core/src/index.ts"],
"@croco/storage-core": ["../storage-core/src/index.ts"],
"@croco/telemetry-api": ["../telemetry-api/src/index.ts"],
"@croco/testing/drizzle": ["./src/drizzle.ts"],
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 1ad78c94a..eee5e5be5 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -3333,6 +3333,9 @@ importers:
'@croco/protocols-rest':
specifier: workspace:*
version: link:../protocols-rest
+ '@croco/retry-core':
+ specifier: workspace:*
+ version: link:../retry-core
tsup:
specifier: 8.5.1
version: 8.5.1(patch_hash=ce9dbc714c187cea78868f1e68c1a0e5097ddbceb7e976a564d94f2291b5bcb9)(@swc/core@1.15.43)(jiti@2.6.1)(postcss@8.5.18)(tsx@4.21.0)(typescript@6.0.3)(yaml@2.9.0)
diff --git a/public-api-surface.snapshot.json b/public-api-surface.snapshot.json
index 52d18abe2..0a7145b38 100644
--- a/public-api-surface.snapshot.json
+++ b/public-api-surface.snapshot.json
@@ -22208,6 +22208,12 @@
"source": "./libs/problems/RateLimitConfigProblems",
"declarationKind": "class"
},
+ {
+ "name": "RateLimitPruneIntervalProblem",
+ "exportKind": "named",
+ "source": "./libs/problems/RateLimitConfigProblems",
+ "declarationKind": "class"
+ },
{
"name": "RateLimitRefundUnsupportedProblem",
"exportKind": "named",
@@ -22358,6 +22364,11 @@
"exportKind": "named",
"source": "./libs/types"
},
+ {
+ "name": "RateLimitPruneScheduler",
+ "exportKind": "named",
+ "source": "./libs/InMemoryRateLimitStore"
+ },
{
"name": "RateLimitRefundReceipt",
"exportKind": "named",
@@ -24137,6 +24148,11 @@
"name": "TaskReference",
"exportKind": "named",
"source": "./libs/types"
+ },
+ {
+ "name": "TaskRunnerRuntime",
+ "exportKind": "named",
+ "source": "./libs/TaskRunner"
}
]
}
@@ -25495,6 +25511,12 @@
"source": "./libs/failure-drill",
"declarationKind": "const"
},
+ {
+ "name": "fixedClock",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime",
+ "declarationKind": "function"
+ },
{
"name": "installTestingTelemetryCapture",
"exportKind": "named",
@@ -25585,6 +25607,12 @@
"source": "./libs/testing",
"declarationKind": "function"
},
+ {
+ "name": "seededIds",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime",
+ "declarationKind": "function"
+ },
{
"name": "serializeOperationalFailureDrillReport",
"exportKind": "named",
@@ -25603,6 +25631,18 @@
"source": "./libs/test-evidence.mjs",
"declarationKind": "const"
},
+ {
+ "name": "TestClock",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime",
+ "declarationKind": "class"
+ },
+ {
+ "name": "TestEnvironment",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime",
+ "declarationKind": "class"
+ },
{
"name": "TestEvidenceContractError",
"exportKind": "named",
@@ -25621,6 +25661,12 @@
"source": "./libs/test-evidence.mjs",
"declarationKind": "function"
},
+ {
+ "name": "TestIdSource",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime",
+ "declarationKind": "class"
+ },
{
"name": "TestingTelemetryCapture",
"exportKind": "named",
@@ -25657,6 +25703,18 @@
"source": "./libs/TestKernel",
"declarationKind": "class"
},
+ {
+ "name": "TestKernelLeakProblem",
+ "exportKind": "named",
+ "source": "./libs/TestKernel",
+ "declarationKind": "class"
+ },
+ {
+ "name": "TestKernelOutboundCallProblem",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime",
+ "declarationKind": "class"
+ },
{
"name": "TestKernelResourceFidelityProblem",
"exportKind": "named",
@@ -25680,6 +25738,36 @@
"exportKind": "named",
"source": "./libs/TestKernel",
"declarationKind": "class"
+ },
+ {
+ "name": "TestNetwork",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime",
+ "declarationKind": "class"
+ },
+ {
+ "name": "TestRandomSource",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime",
+ "declarationKind": "class"
+ },
+ {
+ "name": "TestRuntime",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime",
+ "declarationKind": "class"
+ },
+ {
+ "name": "TestRuntimeConfigurationProblem",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime",
+ "declarationKind": "class"
+ },
+ {
+ "name": "TestRuntimeDrainProblem",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime",
+ "declarationKind": "class"
}
],
"typeExports": [
@@ -26303,6 +26391,16 @@
"exportKind": "named",
"source": "./libs/provider-conformance"
},
+ {
+ "name": "TestDuration",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime"
+ },
+ {
+ "name": "TestEnvironmentOverrides",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime"
+ },
{
"name": "TestEvidenceArtifactProbe",
"exportKind": "named",
@@ -26473,6 +26571,11 @@
"exportKind": "named",
"source": "./libs/TestKernel"
},
+ {
+ "name": "TestKernelLeak",
+ "exportKind": "named",
+ "source": "./libs/TestKernel"
+ },
{
"name": "TestKernelOptions",
"exportKind": "named",
@@ -26493,6 +26596,11 @@
"exportKind": "named",
"source": "./libs/TestKernel"
},
+ {
+ "name": "TestKernelTrackedWork",
+ "exportKind": "named",
+ "source": "./libs/TestKernel"
+ },
{
"name": "TestKernelValidationFidelity",
"exportKind": "named",
@@ -26503,6 +26611,11 @@
"exportKind": "named",
"source": "./libs/testing"
},
+ {
+ "name": "TestReplayMetadata",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime"
+ },
{
"name": "TestResource",
"exportKind": "named",
@@ -26538,6 +26651,21 @@
"exportKind": "named",
"source": "./libs/TestKernel"
},
+ {
+ "name": "TestRetryDependencies",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime"
+ },
+ {
+ "name": "TestRuntimeOptions",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime"
+ },
+ {
+ "name": "TestScheduledWork",
+ "exportKind": "named",
+ "source": "./libs/TestRuntime"
+ },
{
"name": "UnavailableUsageBillingCapabilityConformanceOptions",
"exportKind": "named",