diff --git a/.changeset/dynamic-workflow-source.md b/.changeset/dynamic-workflow-source.md new file mode 100644 index 0000000000..3c3493f0e0 --- /dev/null +++ b/.changeset/dynamic-workflow-source.md @@ -0,0 +1,12 @@ +--- +'@workflow/core': minor +'@workflow/world': minor +'@workflow/world-vercel': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +'@workflow/web-shared': minor +'@workflow/cli': minor +'workflow': minor +--- + +Add experimental dynamic workflows: `start()` accepts workflow source as a string, compiles and stores it with the run through the run-payload serialization pipeline, and replays from that stored code. Steps are exposed to the source through an explicit `experimental_dynamic.steps` map, which is not a security boundary: dynamic source runs with the deployment's full privileges. Off by default; a deployment opts in with `WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1`, and dynamic runs can only start on the current deployment. diff --git a/.github/actions/report-vercel-e2e/action.yml b/.github/actions/report-vercel-e2e/action.yml index 8c0d95abd5..7e521e99ee 100644 --- a/.github/actions/report-vercel-e2e/action.yml +++ b/.github/actions/report-vercel-e2e/action.yml @@ -72,6 +72,7 @@ runs: e2e-${{ inputs.results-name }}.json e2e-${{ inputs.results-name }}.flaky.json e2e-metadata-${{ inputs.app }}-vercel.json + e2e-dynamic-runs-${{ inputs.app }}-vercel.json e2e-failures-${{ inputs.app }}-vercel.json e2e-infra-${{ inputs.app }}-vercel.json e2e-diagnostics-${{ inputs.app }}-vercel.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 56e7281d82..e18ff65624 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -844,7 +844,7 @@ jobs: run: | export DEV_SERVER_LOG_PATH="$GITHUB_WORKSPACE/dev-server-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.log" rm -f "$DEV_SERVER_LOG_PATH" - (cd "$WORKBENCH_APP_PATH" && pnpm dev 2>&1 | tee "$DEV_SERVER_LOG_PATH") & + (cd "$WORKBENCH_APP_PATH" && WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1 pnpm dev 2>&1 | tee "$DEV_SERVER_LOG_PATH") & echo "starting tests in 10 seconds" && sleep 10 pnpm vitest run packages/core/e2e/dev.test.ts; sleep 10 pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.json @@ -857,6 +857,8 @@ jobs: WORKFLOW_DEV_HMR_LOGS: "1" NEXT_CANARY: ${{ matrix.app.canary && '1' || '' }} WORKFLOW_VM: ${{ matrix.app.vm || '' }} + # The server above opts in, so a dynamic E2E opt-in refusal fails. + WORKFLOW_E2E_EXPECT_DYNAMIC_WORKFLOWS: "1" - name: Generate E2E summary if: always() @@ -946,7 +948,7 @@ jobs: - name: Run E2E Tests run: | export PROD_SERVER_LOG_PATH="$GITHUB_WORKSPACE/prod-server-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.log" - (cd "$WORKBENCH_APP_PATH" && pnpm start 2>&1 | tee "$PROD_SERVER_LOG_PATH") & + (cd "$WORKBENCH_APP_PATH" && WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1 pnpm start 2>&1 | tee "$PROD_SERVER_LOG_PATH") & echo "starting tests in 10 seconds" && sleep 10 pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.json env: @@ -956,6 +958,8 @@ jobs: DEPLOYMENT_URL: "http://localhost:${{ matrix.app.name == 'sveltekit' && '4173' || (matrix.app.name == 'astro' && '4321' || '3000') }}" NEXT_CANARY: ${{ matrix.app.canary && '1' || '' }} WORKFLOW_VM: ${{ matrix.app.vm || '' }} + # The server above opts in, so a dynamic E2E opt-in refusal fails. + WORKFLOW_E2E_EXPECT_DYNAMIC_WORKFLOWS: "1" - name: Generate E2E summary if: always() @@ -1055,7 +1059,7 @@ jobs: - name: Run E2E Tests run: | export PROD_SERVER_LOG_PATH="$GITHUB_WORKSPACE/prod-server-postgres-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.log" - (cd "$WORKBENCH_APP_PATH" && pnpm start 2>&1 | tee "$PROD_SERVER_LOG_PATH") & + (cd "$WORKBENCH_APP_PATH" && WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1 pnpm start 2>&1 | tee "$PROD_SERVER_LOG_PATH") & echo "starting tests in 10 seconds" && sleep 10 pnpm run test:e2e --reporter=verbose --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.artifactSuffix }}.json env: @@ -1065,6 +1069,8 @@ jobs: DEPLOYMENT_URL: "http://localhost:${{ matrix.app.name == 'sveltekit' && '4173' || (matrix.app.name == 'astro' && '4321' || '3000') }}" NEXT_CANARY: ${{ matrix.app.canary && '1' || '' }} WORKFLOW_VM: ${{ matrix.app.vm || '' }} + # The server above opts in, so a dynamic E2E opt-in refusal fails. + WORKFLOW_E2E_EXPECT_DYNAMIC_WORKFLOWS: "1" - name: Generate E2E summary if: always() diff --git a/docs/content/docs/v5/advanced/dynamic-workflows.mdx b/docs/content/docs/v5/advanced/dynamic-workflows.mdx new file mode 100644 index 0000000000..d69b432eae --- /dev/null +++ b/docs/content/docs/v5/advanced/dynamic-workflows.mdx @@ -0,0 +1,224 @@ +--- +title: Dynamic Workflows +description: Start a workflow run from source code that was not part of your build. +type: conceptual +summary: Pass workflow source to start() to run orchestration whose shape is only known after deployment. +prerequisites: + - /docs/foundations/starting-workflows + - /docs/how-it-works/code-transform +related: + - /docs/api-reference/workflow-api/start + - /docs/how-it-works/encryption + - /docs/configuration/runtime-tuning +--- + + +Dynamic workflows are **experimental** and **off by default**. The API may change without a major version bump. A deployment must opt in with `WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1`, dynamic runs can only start on the current deployment, and the World must support dynamic-source storage. See [Enabling dynamic workflows](#enabling-dynamic-workflows) and [World support](#world-support). + + + +Dynamic source runs with the **full privileges of your deployment's functions**. It can read every environment variable, use the network and the filesystem, and call any step in the deployment. `experimental_dynamic.steps` is not a security boundary. Only pass source you would merge into your codebase. See [Security](#security). + + +Normally a workflow function is compiled into your build: the [code transform](/docs/how-it-works/code-transform) rewrites every `"use workflow"` function, the build bundles them, and `start()` names one by importing it. + +A dynamic workflow skips that. You hand `start()` a string of JavaScript, and it runs — no build, no deploy: + +```ts +import { start } from 'workflow/api'; +import { fetchUser, sendEmail } from './steps'; + +const run = await start( + ` +async function workflow(input) { + "use workflow"; + + const user = await steps.fetchUser(input.userId); + await steps.sendEmail(user.email); + + return { ok: true }; +} +`, + [{ userId: 'user_123' }], + { + experimental_dynamic: { + steps: { fetchUser, sendEmail }, + }, + } +); + +console.log(await run.status); // 'running' +``` + +Only the *orchestration* is dynamic. Every step the source calls was deployed with your app, and `experimental_dynamic.steps` names the ones it calls by alias. That map does not stop source from reaching other steps; see [Security](#security). There is no way to define a new step from source. + +## When to use this + +Reach for dynamic workflows when the **shape** of the orchestration is only known after you deploy, and the source comes from code you trust as much as your own: + +- **Orchestration your application assembles** from reviewed templates, over a fixed set of deployed steps. +- **Experiments** — try a new composition of existing steps without shipping a build. + +Dynamic workflows are not a way to run code written by your end users or generated by a model from their input. That source would run with your deployment's privileges; see [Security](#security). + +If your workflows are known at build time, use a normal workflow function. It has better types, better errors, no source validation, and no size limits. + +## Enabling dynamic workflows + +Dynamic workflows are off unless the deployment sets: + +```bash +WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1 +``` + +Only `1` or `true` (case-insensitive) enables them; any other value, or no value, leaves them off. The runtime reads the variable where workflows execute, when it needs it, so set it on the deployment or dev server rather than at build time. It controls three things: + +- **Starting.** `start()` with source throws before it contacts the World, creates a run, or enqueues anything unless this process has opted in. +- **Delivery.** When a dynamic run reaches a deployment that has not opted in, the runtime does not execute its stored code. It fails the run with a `RUNTIME_ERROR` rather than retrying it. +- **Health check.** A deployment advertises dynamic support in its [health check](/docs/api-reference/workflow-runtime/health-check) only when it has opted in. + +### Same deployment only + +A dynamic run must execute on the deployment that started it. `start()` rejects a dynamic start whose target differs from the current deployment. That includes an explicit `deploymentId` for another deployment, `deploymentId: 'latest'` when it resolves to a different deployment, and any concrete target when the current deployment cannot be determined. The rejection happens before any capability probe, key lookup, upload, run creation, or queue message. + +## What the source can use + +Dynamic source has no imports. Instead, the generated code predefines a small runtime surface: + +| Binding | What it is | +| --- | --- | +| `steps` | Frozen object of the aliases you passed in `experimental_dynamic.steps`. Calling one dispatches that registered step. | +| `sleep` | The [durable sleep](/docs/api-reference/workflow/sleep) primitive. | +| `createHook` | The [hook](/docs/foundations/hooks) primitive, for waiting on an external signal. | + +The source also runs inside the normal deterministic workflow VM, so the usual [workflow globals](/docs/api-reference/workflow-globals) — `Date`, `Math.random`, `crypto`, `URL`, `TextEncoder`, `structuredClone`, and the rest — are available with the same determinism guarantees as a static workflow. + +Dynamic source exposes only the small set of primitives injected by its generated wrapper. `createWebhook()` also needs the static workflow module's URL and metadata helper, and `getWritable()` needs its workflow-stream helper, so neither is currently injected into dynamic source. Use `createHook()` with server-side `resumeHook()`, and perform streaming through registered steps or a statically compiled workflow. + +Here is a longer example using a timer and a hook to wait for an approval: + +```ts +import { start } from 'workflow/api'; +import { sendEmail } from './steps'; + +const run = await start( + ` +async function workflow(input) { + "use workflow"; + + await sleep("15m"); + + const approval = createHook({ token: input.approvalToken }); + const result = await Promise.race([ + approval, + sleep("1d").then(() => ({ approved: false, timedOut: true })), + ]); + + if (result.approved) { + await steps.sendEmail(input.email); + } + + return result; +} +`, + [{ + userId: 'user_123', + email: 'ada@example.com', + approvalToken: 'approval-req_01J...', + }], + { + experimental_dynamic: { + steps: { sendEmail }, + }, + } +); +``` + +Supply a unique, deterministic approval token from the caller. The workflow must recreate the same token during replay, while the external service needs that token to call `resumeHook()`; do not use a tenant or user ID alone when concurrent runs can overlap. + +## Rules for the source + +`start()` validates the source before it writes anything, so a definition that could never run fails at the call site rather than on a queue delivery: + +- It must declare `async function workflow(...)`. Pass `experimental_dynamic.exportName` to use a different name; export names may contain letters, digits, and `_`, and cannot start with a digit. +- The function's first statement must be the `"use workflow"` directive. +- No `import` or `export`. Reach steps through `steps`, not through modules. +- JavaScript only — no TypeScript syntax, no npm dependencies, no bundling. +- No inline `"use step"` functions. Steps come from `experimental_dynamic.steps`. +- At most 128 KB of source. +- On Vercel, the run's execution context is limited to 2,048 bytes of JSON, and the `dynamicWorkflow` metadata below counts against it. That leaves room for roughly 30 step aliases, depending on how long the aliases and step IDs are. A start that exceeds it fails before anything is written. + +Everything a static workflow must obey still applies: the body has to be [deterministic](/docs/foundations/workflows-and-steps), and any side effect belongs in a step. + +## Workflow IDs + +You do not choose the workflow ID. It is derived from the source and its step bindings: + +``` +workflow//dynamic/// +``` + +Two consequences worth knowing: + +- **The same definition always gets the same ID.** Runs of one generated workflow group together in [observability](/docs/observability) and share a queue topic, even across processes. +- **A caller cannot claim an ID.** Because the hash covers the source *and* the step bindings, arbitrary source cannot be made to run under a static workflow's name — or under another definition's. + +Changing the source, or pointing an alias at a different step, produces a different workflow. + +## How the code is stored + +A dynamic run's workflow function is not in your deployment's bundle, so the run carries its own compiled workflow code — and replaying the run means replaying *that* code, not whatever your deployment contains now. + +That code uses the same serialization path as workflow inputs. It is compressed when the run protocol supports compression and compression is worthwhile, and encrypted when the World supplies run key material (see [Encryption](/docs/how-it-works/encryption)). Vercel's supported configuration provides encrypted storage; the Local and Postgres Worlds store it in plaintext. Retention and deletion apply whether the stored bytes are plaintext or ciphertext. + +When a run has key material, or was started with encryption, a delivery only executes code stored in the run's symmetric `encr` envelope. It refuses plaintext and sealed (`encp`) payloads and fails the run. Encryption keeps the code confidential; it does not prove who wrote it. See [Security](#security). + +On Vercel, durable workflow code storage is ref-backed on the run. The definition's size changes only how those bytes reach the backend: + +- **Small definitions** (the overwhelming majority) ride inline in the `run_created` request frame. The backend materializes those bytes into the run's ref-backed storage, with no upload request from `start()`. +- **Larger definitions** are uploaded first, and `run_created` carries the resulting reference. This costs one extra request at `start()`. + +Both paths are transparent — there is nothing to configure. Here, “inline” describes request transport, not a second durable storage shape. + +Alongside the serialized code, the run records small plaintext metadata on `executionContext.dynamicWorkflow`: the source hash, the export name, and the alias-to-step-ID map. That is what lets a run be identified as dynamic without decoding the source. It is plaintext even when the code is encrypted, so anyone who can read the run can see which step IDs it was given and the aliases they were given under. + +## World support + +Dynamic workflows need a World that can store the run's workflow code. + +| World | Support | +| --- | --- | +| [Vercel](/worlds/vercel) | Encrypted, ref-backed storage (small definitions transported inline; large definitions uploaded first). Requires a backend that advertises dynamic-source storage. | +| [Local](/worlds/local) | Stored in plaintext on the run record in the local filesystem store. | +| [Postgres](/worlds/postgres) | Stored in plaintext on the run row. | +| Others | Whatever the World explicitly attests through the versioned dynamic-workflow storage capability. | + +After the opt-in and same-deployment checks, `start()` checks the backend's dynamic-workflow storage capability. A backend that does not advertise it, including one that predates the capability endpoint, fails the start. On Vercel, `start()` then validates the final execution context against the 2,048-byte limit. All of this happens before serializing or uploading code, creating an event, or publishing a queue message. + +## Security + + +Dynamic source is **trusted application code** with the full privileges of your deployment's functions. The workflow VM is a determinism sandbox, not a security sandbox. Code in it can reach the host process: it can read every environment variable, use the network and the filesystem, and call any step registered in the deployment with any arguments. + + +- **`steps` is not a boundary.** The `steps` object contains only the aliases you passed and is frozen, so ordinary code that calls `steps.somethingElse()` fails the run instead of dispatching a step it was not given. Code that is trying to reach other steps, or the host, can. +- **Only start source you would merge.** Do not build source from end-user input, and do not run model output generated from untrusted input. Either one gives whoever controls that input your deployment's privileges. +- **Opting in is a deployment decision.** A deployment that sets `WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS` executes stored code for any dynamic run it receives. Anyone who can start a workflow on it can run code with its privileges. +- **Encryption gives confidentiality only.** Where the World encrypts stored code, it cannot be read at rest without the run's key, and a delivery refuses code that is not encrypted with that key. Anyone who can obtain the run's key can still write valid code, so encryption does not replace the opt-in. +- **Plaintext Worlds turn storage write access into code execution.** The Local and Postgres Worlds store the code in plaintext. On an opted-in deployment, anyone who can write to the Postgres database or the local data directory can make every worker execute code of their choosing. +- **The step map is readable.** `executionContext.dynamicWorkflow.steps` stores the alias-to-step-ID map in plaintext, so anyone with read access to the run sees the step IDs the source was given. + +Treat dynamic source the way you would treat code in a pull request: written or reviewed by someone you trust with the deployment. + +## Limitations + +- Experimental — the API may change without a major version bump. +- Off unless the deployment sets `WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1`. +- Same-deployment starts only. +- JavaScript only. No TypeScript syntax, npm dependencies, or bundling. +- Steps must already be registered in the deployment; no runtime step registration. +- No inline `"use step"` functions, `createWebhook`, or `getWritable`. +- No caller-provided workflow IDs. +- Parser-based validation checks JavaScript syntax and the required source/wrapper shape without executing it. It does not validate behavior, determinism, or intent. +- On Vercel, roughly 30 step aliases fit the 2,048-byte execution-context limit. +- Requires a World with dynamic-source storage. diff --git a/docs/content/docs/v5/api-reference/workflow-api/start.mdx b/docs/content/docs/v5/api-reference/workflow-api/start.mdx index 9eff28c0e6..51f1736b12 100644 --- a/docs/content/docs/v5/api-reference/workflow-api/start.mdx +++ b/docs/content/docs/v5/api-reference/workflow-api/start.mdx @@ -7,6 +7,7 @@ prerequisites: - /docs/foundations/starting-workflows related: - /docs/foundations/idempotency + - /docs/advanced/dynamic-workflows --- Start/enqueue a new workflow run. @@ -153,3 +154,50 @@ The returned `Run` object is fully functional inside a workflow. Each property a `returnValue` polls the child run every second and holds the polling step's worker slot open for as long as the child takes to finish. For long-running children, spawn without awaiting `returnValue` and have the child resume a [hook](/docs/foundations/hooks) when it completes. See the [`startAndWait()` pattern](/cookbook/advanced/child-workflows). + +### Dynamic Workflow Source + + +Experimental and off by default. The API may change without a major version bump. The deployment must set `WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1`, and dynamic runs can only start on the current deployment. + + +`start()` also accepts a string of workflow **source** instead of an imported function, for orchestration whose shape is only known after you deploy. The source is compiled and stored with the run through the same serialization path as other run payloads, including encryption when the World supplies run key material. + +Dynamic source runs with the full privileges of your deployment's functions: it can read environment variables, use the network and filesystem, and call any step in the deployment. Only pass source you trust as much as your own code. + +`experimental_dynamic.steps` maps the aliases the source may call to steps that are already registered in your deployment. It keeps ordinary source from calling a step by a name it was not given, but it is not a security boundary. There is no way to define a new step from source. + +```typescript +import { start } from "workflow/api"; +import { fetchUser, sendEmail } from "./steps"; + +const run = await start( + ` +async function workflow(input) { + "use workflow"; + + const user = await steps.fetchUser(input.userId); + await steps.sendEmail(user.email); + + return { ok: true }; +} +`, + [{ userId: "user_123" }], + { + experimental_dynamic: { // [!code highlight] + steps: { fetchUser, sendEmail }, // [!code highlight] + }, // [!code highlight] + } +); +``` + +| Option | Type | Description | +| --- | --- | --- | +| `experimental_dynamic.steps` | `Record` | Required. Registered steps the source may call, keyed by the alias it calls them under. | +| `experimental_dynamic.exportName` | `string` | Name of the async workflow function in the source. Use letters, digits, and `_`, not starting with a digit. Defaults to `"workflow"`. | + +The return type is `Run`: the source's shape is only known to whatever generated it, so there is nothing to infer. The workflow ID is derived from the source and its step bindings — it cannot be supplied. + +A dynamic start throws a `WorkflowRuntimeError`, before anything is written, when the deployment has not opted in, when `deploymentId` targets another deployment (including `'latest'` resolving to one), or when the World's backend does not support dynamic-source storage. + +See [Dynamic Workflows](/docs/advanced/dynamic-workflows) for the opt-in, the source rules and limits, the predefined runtime bindings (`steps`, `sleep`, `createHook`), how the code is stored, and the security model. diff --git a/docs/content/docs/v5/api-reference/workflow-runtime/health-check.mdx b/docs/content/docs/v5/api-reference/workflow-runtime/health-check.mdx index a52ab6f165..bfad0a72a6 100644 --- a/docs/content/docs/v5/api-reference/workflow-runtime/health-check.mdx +++ b/docs/content/docs/v5/api-reference/workflow-runtime/health-check.mdx @@ -48,3 +48,4 @@ Returns a `Promise`: | `latencyMs` | `number \| undefined` | Round-trip latency when the check succeeded | | `specVersion` | `number \| undefined` | Workflow spec version of the responding deployment | | `workflowCoreVersion` | `string \| undefined` | `@workflow/core` version of the responding deployment | +| `dynamicWorkflowVersion` | `number \| undefined` | Dynamic-workflow runtime version of the responding deployment. Present only when it has opted in with `WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS`; see [Dynamic Workflows](/docs/advanced/dynamic-workflows) | diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 85bb2195d8..ac68d1fd27 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -237,6 +237,15 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - A bundle whose module scope consumes randomness, reads the clock, or replaces a serialization intrinsic cannot be snapshotted safely. The runtime detects these cases when preparing the snapshot and falls back to per-invocation evaluation. - Set `0` or `false` to always evaluate the bundle per invocation. +## Dynamic workflows + +### `WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS` + +- Default: disabled +- Set `1` or `true` (case-insensitive) to let this deployment start [dynamic workflows](/docs/advanced/dynamic-workflows), execute their stored code on delivery, and advertise dynamic support in its health check. Any other value leaves it disabled. +- Dynamic source runs with the full privileges of the deployment's functions. Enable it only on deployments whose dynamic source you trust as much as your own code. +- When disabled, `start()` with source throws before writing anything, and a delivered dynamic run fails instead of executing. + ## Compression and tracing ### `WORKFLOW_DISABLE_COMPRESSION` diff --git a/docs/content/docs/v5/meta.json b/docs/content/docs/v5/meta.json index 030a3db919..9a53222e17 100644 --- a/docs/content/docs/v5/meta.json +++ b/docs/content/docs/v5/meta.json @@ -5,6 +5,7 @@ "getting-started", "foundations", "how-it-works", + "advanced/dynamic-workflows", "observability", "ai", "testing", diff --git a/package.json b/package.json index af4edd5790..a55f7f9b91 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "test": "turbo test", "clean": "turbo clean", "typecheck": "turbo typecheck", - "test:e2e": "vitest run packages/core/e2e/e2e.test.ts packages/core/e2e/e2e-agent.test.ts", + "test:e2e": "vitest run packages/core/e2e/e2e.test.ts packages/core/e2e/e2e-agent.test.ts packages/core/e2e/e2e-dynamic-workflow.test.ts", "test:e2e:event-log-race-repro": "vitest run packages/core/e2e/event-log-race-repro.test.ts", "test:e2e:event-log-race-repro:local": "bash scripts/event-log-race-repro-local.sh", "test:e2e:nextjs-webpack:staged": "node scripts/test-staged-nextjs-webpack.mjs", diff --git a/packages/cli/src/lib/inspect/hydration.ts b/packages/cli/src/lib/inspect/hydration.ts index 1ed58bf450..21907797af 100644 --- a/packages/cli/src/lib/inspect/hydration.ts +++ b/packages/cli/src/lib/inspect/hydration.ts @@ -352,6 +352,14 @@ async function maybeDecryptFields< result.input = await maybeDecrypt(result.input, k); result.output = await maybeDecrypt(result.output, k); (result as any).error = await maybeDecrypt((result as any).error, k); + // A dynamic run's own workflow code (WorkflowRun), stored through the + // same pipeline as its input. + if ((result as any).dynamicWorkflowCode !== undefined) { + (result as any).dynamicWorkflowCode = await maybeDecrypt( + (result as any).dynamicWorkflowCode, + k + ); + } // Decrypt metadata field (Hook) result.metadata = await maybeDecrypt(result.metadata, k); @@ -406,6 +414,11 @@ function replaceEncryptedAndExpiredWithRef(resource: T): T { for (const key of ['input', 'output', 'metadata', 'error']) { result[key] = toDisplayRef(result[key]); } + // Run-only, so touched only when present: static runs never carry it and + // should not grow an `undefined` field in the inspect output. + if ('dynamicWorkflowCode' in result) { + result.dynamicWorkflowCode = toDisplayRef(result.dynamicWorkflowCode); + } if (result.eventData && typeof result.eventData === 'object') { const ed = { ...(result.eventData as Record) }; diff --git a/packages/core/e2e/e2e-dynamic-workflow.test.ts b/packages/core/e2e/e2e-dynamic-workflow.test.ts new file mode 100644 index 0000000000..dcce4e0529 --- /dev/null +++ b/packages/core/e2e/e2e-dynamic-workflow.test.ts @@ -0,0 +1,409 @@ +/** + * E2E tests for dynamic workflows: runs started from workflow source rather + * than from a workflow function in the deployment's build-time manifest. + * + * The workflows under test are **generated inside the deployment**, by the + * fixtures in `workflows/99_e2e.ts` (`dynamicWorkflowFromApp` and friends). + * That is deliberate, and it is most of the reason this is worth testing end + * to end at all: + * + * - `experimental_dynamic.steps` is given the *imported* `add` step function, so the + * `.stepId` the build-time transform stamped on it is what binds the source + * to a registered step. This runner cannot do that — it holds no handle on + * the function — and would have to fall back to an explicit `{ stepId }`. + * - The source is assembled at runtime by app code, which is how dynamic + * source reaches `start()` in an application. + * - The deployed handler compiles, stores, reads back and evaluates the code + * in its own process, on every delivery. + * + * So each test starts a *static* fixture, which starts a *dynamic* child, and + * then asserts on the child. Runs against every world the matrix covers — + * Vercel, local dev/prod, and Postgres — with no per-world branching. + * + * The deployment must opt in with WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1; + * without it each fixture's `start()` refuses and the test skips. A lane that + * opts its server in also sets WORKFLOW_E2E_EXPECT_DYNAMIC_WORKFLOWS=1 on the + * runner, and there that refusal fails the test instead. + * + * Run locally: + * 1. cd workbench/nextjs-turbopack && WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1 pnpm dev + * 2. DEPLOYMENT_URL=http://localhost:3000 APP_NAME=nextjs-turbopack \ + * pnpm vitest run packages/core/e2e/e2e-dynamic-workflow.test.ts + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { getCurrentTest } from '@vitest/runner'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import type { Run, start as rawStart } from '../src/runtime'; +import { getRun, getWorld } from '../src/runtime'; +import { + getCollectedRunIds, + getWorkflowMetadata, + isJsApp, + requireFixture, + setupRunTracking, + setupWorld, + startTracked, + trackRun, + writeInfraSidecar, +} from './utils'; + +const deploymentUrl = process.env.DEPLOYMENT_URL; +if (!deploymentUrl) { + throw new Error('`DEPLOYMENT_URL` environment variable is not set'); +} + +async function start( + ...args: Parameters> +): Promise> { + return startTracked(...args); +} + +/** Same fixture lookup + conformance gate `e2e.test.ts` uses. */ +const e2e = (fn: string) => { + requireFixture(fn); + return getWorkflowMetadata(deploymentUrl, 'workflows/99_e2e.ts', fn); +}; + +/** What the parent fixture returns: the dynamic child it started. */ +interface DynamicChild { + parentInput?: number; + childRunId: string; +} + +/** + * Set on lanes whose server runs with WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1, + * so a "not opted in" refusal there is a failure rather than a skip. + */ +const expectDynamicWorkflows = + process.env.WORKFLOW_E2E_EXPECT_DYNAMIC_WORKFLOWS === '1'; + +/** + * The messages `start()` throws, inside the fixture's step, when this + * deployment cannot run dynamic workflows. They reach the runner wrapped in + * the parent's failure. + */ +const UNSUPPORTED_DEPLOYMENT: readonly { + pattern: RegExp; + reason: string; + /** Fail instead of skipping when the lane expects dynamic workflows. */ + optIn?: true; +}[] = [ + { + // The deployment has not set WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS. + pattern: /Dynamic workflows are disabled on this deployment/, + reason: 'this deployment has not opted in to dynamic workflows', + optIn: true, + }, + { + // The backend does not advertise dynamic-source storage (including a + // backend without the capabilities route, which reads as none). + pattern: /Dynamic workflows require backend storage capability version/, + reason: + "this deployment's Workflow backend does not advertise dynamic-source storage", + }, + { + // The backend accepted the run but did not persist its workflow code. + pattern: /did not store its dynamic workflow code/, + reason: + "this deployment's Workflow backend has no dynamic-source storage yet", + }, +]; + +/** + * The run id `start()` names in that error. + * + * The dynamic run *is* created before the check runs — `start()` writes + * `run_created` and enqueues the run, and only then reads back what the + * backend kept — so on this path a real dynamic run exists in the world, and + * a short one can even complete (turbo executes it from the queue message). + * Only its *replayability* is missing. Pulling the id out means the sidecar + * still reports it, which is the whole reason to want a run id: to go look at + * one. + * + * Anchored to the exact wording of that message, not to any `wrun_` in the + * string: the error reaches the runner wrapped in the parent fixture's own + * failure, which names the *parent*, and a loose match takes that instead. + */ +const CREATED_RUN_ID = + /Workflow run (wrun_[0-9A-Za-z]+) was created, but this deployment's Workflow backend did not store its dynamic workflow code/; + +/** + * Skip the running test when the deployment cannot run dynamic workflows: it + * has not opted in (unless the lane expects it to have), or its backend has no + * dynamic-source storage. + * + * A real skip rather than a failure, because the gap is the deployment's + * configuration or the backend's, and the suite cannot close it: these go + * live, unchanged, once the deployment opts in and the server side ships. + * Not `expect().toThrow()` either — a passing assertion would be claiming + * coverage the run never got. Same mechanism as the conformance gate. + */ +function skipIfUnsupportedDeployment(error: unknown): never { + const unsupported = + error instanceof Error + ? UNSUPPORTED_DEPLOYMENT.find(({ pattern }) => + pattern.test(error.message) + ) + : undefined; + if ( + error instanceof Error && + unsupported && + !(unsupported.optIn && expectDynamicWorkflows) + ) { + const createdRunId = CREATED_RUN_ID.exec(error.message)?.[1]; + if (createdRunId) { + // Track it before skipping: the run was created and is inspectable, + // even though nothing can replay it. Labelled, because the sidecar + // otherwise shows two bare ids per test — the parent fixture's and this + // one — with nothing saying which is the dynamic run. + trackRun(getRun(createdRunId), { + testName: `${getCurrentTest()?.name ?? 'dynamic workflow'} [dynamic run]`, + }); + } + getCurrentTest()?.context.skip( + unsupported.reason + + (createdRunId ? ` (created, unreplayable run: ${createdRunId})` : '') + ); + } + throw error; +} + +/** Start a parent fixture and read the dynamic child it reports. */ +async function startParent( + fixture: string, + args: unknown[] +): Promise { + const parent = await start(await e2e(fixture), args); + try { + return (await parent.returnValue) as DynamicChild; + } catch (error) { + skipIfUnsupportedDeployment(error); + } +} + +/** + * Read a run's persisted record. + * + * The World rather than `getRun()`: `getRun()` returns a handle of + * promise-returning getters over a small public surface, and these tests + * assert on storage — `executionContext` and the stored code bytes. + * `resolveData: 'all'` is what keeps the payload fields from being stripped. + */ +async function readRunRecord(runId: string) { + const world = await getWorld(); + return world.runs.get(runId, { resolveData: 'all' }); +} + +/** + * Await a child run the deployment started, and return it with its record. + * + * The child's ID comes back from the parent, so the runner never held a `Run` + * for it — `getRun` adopts one, and tracking it gets it into the diagnostics + * dump and the run-ID sidecar alongside directly-started runs. + */ +async function awaitChildRun(childRunId: string) { + const child = trackRun(getRun(childRunId), { + testName: `${getCurrentTest()?.name ?? 'dynamic workflow'} [dynamic run]`, + }); + const output = await child.returnValue; + return { output, record: await readRunRecord(childRunId) }; +} + +/** As above, for a child expected to fail. */ +async function awaitChildRunFailure(childRunId: string, timeoutMs = 60_000) { + trackRun(getRun(childRunId), { + testName: `${getCurrentTest()?.name ?? 'dynamic workflow'} [dynamic run]`, + }); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const record = await readRunRecord(childRunId); + if (record.status === 'failed' || record.status === 'cancelled') { + return record; + } + if (record.status === 'completed') { + throw new Error( + `child run ${childRunId} completed; expected it to fail because its ` + + 'source called a step it was not given' + ); + } + await sleep(500); + } + throw new Error(`child run ${childRunId} did not fail within ${timeoutMs}ms`); +} + +/** + * Write out the run IDs this file created, so a Vercel run can be opened in + * the dashboard afterwards. + * + * `e2e.test.ts` does the same for its own runs, but each e2e file runs in its + * own vitest worker with its own copy of the collector — so without this the + * dynamic runs are tracked in memory and then thrown away, which is exactly + * what someone asks for when they want to see what a dynamic run looks like in + * production. Distinct filename because that sidecar is per-app and the last + * writer would otherwise clobber it. + */ +function writeDynamicRunSidecar() { + if (!process.env.WORKFLOW_VERCEL_ENV) return; + const appName = process.env.APP_NAME || 'unknown'; + fs.writeFileSync( + path.resolve(process.cwd(), `e2e-dynamic-runs-${appName}-vercel.json`), + JSON.stringify( + { + runIds: getCollectedRunIds(), + vercel: { + projectSlug: process.env.WORKFLOW_VERCEL_PROJECT_SLUG, + environment: process.env.WORKFLOW_VERCEL_ENV, + teamSlug: 'vercel-labs', + }, + }, + null, + 2 + ) + ); +} + +afterAll(() => { + writeDynamicRunSidecar(); + writeInfraSidecar(); +}); + +beforeAll(() => { + setupWorld(deploymentUrl); +}); + +beforeEach((ctx) => { + setupRunTracking(ctx.task.name); +}); + +/** + * Dynamic source is JavaScript evaluated in the JS workflow VM, so it is + * JS-implementation-specific by construction. A non-JS SDK would not be + * running the same thing, so it skips rather than carrying this as a gap. + */ +const describeJs = isJsApp() ? describe : describe.skip; + +describeJs('dynamic workflows e2e', { timeout: 120_000 }, () => { + it('runs app-generated source against a registered step', async () => { + const result = await startParent('dynamicWorkflowFromApp', [20]); + + expect(result.parentInput).toBe(20); + expect(result.childRunId).toMatch(/^wrun_/); + + const child = await awaitChildRun(result.childRunId); + // A workflow the deployment never bundled, executing against the `add` + // step it did — bound by the `.stepId` on the imported function. + expect(child.output).toEqual({ total: 41 }); + + // The generated id, derived from the source and its step bindings. + expect(child.record.workflowName).toMatch( + /^workflow\/\/dynamic\/[0-9a-f]{32}\/\/workflow$/ + ); + // Plaintext on purpose: it is what identifies a run as dynamic, and + // exposes the step allowlist it was compiled against, without decrypting + // the code. + expect(child.record.executionContext?.dynamicWorkflow).toMatchObject({ + version: 1, + exportName: 'workflow', + sourceHash: expect.stringMatching(/^[0-9a-f]{64}$/), + }); + }); + + it('stores the generated code with the child run', async () => { + const result = await startParent('dynamicWorkflowFromApp', [1]); + const child = await awaitChildRun(result.childRunId); + + const code = (child.record as { dynamicWorkflowCode?: unknown }) + .dynamicWorkflowCode; + if (code === undefined) { + // Same "no dynamic-source storage" fact, reached a step later: + // `start()`'s fail-fast check reads the created run off its own + // response, and a resilient start has no response to read — so a short + // dynamic run can still finish (turbo runs it from the queue message + // inside one invocation) while the backend stored nothing. + getCurrentTest()?.context.skip( + "this deployment's Workflow backend has no dynamic-source storage yet: the run completed from the queue message without its code being persisted" + ); + } + expect(code).toBeInstanceOf(Uint8Array); + const codeBytes = code as Uint8Array; + expect(codeBytes.byteLength).toBeGreaterThan(0); + + // Opaque at rest. Where the world encrypts, the source must not be + // readable off the stored bytes — that is the property that distinguishes + // ref-backed storage from the prototype's plaintext metadata field. + const encryptionEnabled = Boolean( + ( + child.record.executionContext?.features as + | { encryption?: boolean } + | undefined + )?.encryption + ); + if (encryptionEnabled) { + expect(new TextDecoder().decode(codeBytes)).not.toContain('use workflow'); + } + }); + + it('replays a suspended child by reading its stored code back', async () => { + // The delivery that resumes the child holds no in-memory copy of the code + // and no run input on the message, so it has to read the stored code back + // and decrypt it. This is what caught `world-local` dropping the code on + // a run's first status transition. + const result = await startParent('dynamicWorkflowFromAppWithSleep', [10]); + const child = await awaitChildRun(result.childRunId); + + expect(child.output).toEqual({ total: 21 }); + }); + + it('fails the child when its source calls a step it was not given', async () => { + // `steps` is frozen and holds only the aliases the app passed, so this is + // a run failure rather than an unauthorized step dispatch. + const result = await startParent('dynamicWorkflowDisallowedStep', [3]); + const record = await awaitChildRunFailure(result.childRunId); + + expect(record.status).toBe('failed'); + }); + + it('derives the same workflow id for the same generated source', async () => { + // Two runs of the fixture generate identical source, so they must land on + // one durable id — that is what makes runs of a generated workflow group + // together in observability and share a queue topic. + const first = await startParent('dynamicWorkflowFromApp', [4]); + const second = await startParent('dynamicWorkflowFromApp', [4]); + + const [a, b] = await Promise.all([ + awaitChildRun(first.childRunId), + awaitChildRun(second.childRunId), + ]); + + expect(a.output).toEqual({ total: 9 }); + expect(b.output).toEqual({ total: 9 }); + expect(a.record.workflowName).toBe(b.record.workflowName); + expect(a.record.runId).not.toBe(b.record.runId); + }); + + // Client-side validation is the one part of this that genuinely belongs at + // the runner level: it happens before any write, so no deployment is + // involved and there is no run to observe. It also runs before the opt-in + // check, so the runner needs no WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS. + it('rejects source that cannot be a workflow before creating a run', async () => { + const steps = { add: { stepId: 'step//./workflows/99_e2e//add' } }; + + await expect( + start('const notAWorkflow = 1;', [], { experimental_dynamic: { steps } }) + ).rejects.toThrow(/must declare `async function workflow/); + + await expect( + start('async function workflow() { return 1; }', [], { + experimental_dynamic: { steps }, + }) + ).rejects.toThrow(/"use workflow" directive/); + + await expect( + start('async function workflow() { "use workflow"; }', [], { + experimental_dynamic: { steps: {} }, + }) + ).rejects.toThrow(/at least one registered step/); + }); +}); diff --git a/packages/core/package.json b/packages/core/package.json index cc6ab81caf..f072fd6199 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -105,6 +105,7 @@ "@workflow/world": "workspace:*", "@workflow/world-local": "workspace:*", "@workflow/world-vercel": "workspace:*", + "acorn": "8.15.0", "debug": "4.4.3", "devalue": "5.9.2", "ms": "2.1.3", diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 349c4e59cc..0ccd2157a3 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -52,6 +52,7 @@ import { getStepFunction } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { COMPUTE_INSTANCE_ID } from './runtime/compute-instance.js'; import { + DYNAMIC_WORKFLOWS_ENV, getMaxEventsOverride, getMaxQueueDeliveries, getOpenWaitClockSkewMs, @@ -59,6 +60,7 @@ import { getPreconditionMaxReinvocations, getPreconditionReinvokeDelaySeconds, getReplayDivergenceMaxRetries, + isDynamicWorkflowsEnabled, isInlineOwnershipEnabled, isTurboEnabled, isVmRetentionEnabled, @@ -69,6 +71,11 @@ import { guardDeploymentAffinity, type ReenqueueArgs, } from './runtime/deployment-guard.js'; +import { + type DynamicWorkflowMetadata, + dynamicWorkflowName, + readDynamicWorkflowMetadata, +} from './runtime/dynamic-workflow.js'; import { absorbSkippedSlotReport, appendEventLog, @@ -132,7 +139,11 @@ import { import { useQuickJSVm } from './runtime/vm-mode.js'; import { getWaitContinuationDispatch } from './runtime/wait-continuation.js'; import { getWorld } from './runtime/world.js'; -import { dehydrateRunError, type PayloadKey } from './serialization.js'; +import { + dehydrateRunError, + hydrateDynamicWorkflowCode, + type PayloadKey, +} from './serialization.js'; import { remapErrorStack } from './source-map.js'; import * as Attribute from './telemetry/semantic-conventions.js'; import { @@ -154,6 +165,7 @@ import { } from './types.js'; import { buildWorkflowSuspensionMessage } from './util.js'; import { + compileDynamicWorkflowBundle, compileWorkflowBundle, replayWorkflow, resumeWorkflow, @@ -198,6 +210,9 @@ export { wakeUpRun, } from './runtime/runs.js'; export { + type DynamicStartOptions, + type DynamicWorkflowOptions, + type DynamicWorkflowStepReference, type StartOptions, type StartOptionsBase, type StartOptionsWithDeploymentId, @@ -678,6 +693,108 @@ async function getMaxInlineDurationMs( return ms('2m'); } +/** The workflow code a delivery replays, and the marker when it is dynamic. */ +interface ResolvedWorkflowCode { + code: string; + dynamicWorkflow?: DynamicWorkflowMetadata; +} + +/** + * Pick the workflow code this run replays. + * + * For a static run this is the deployment's bundle, returned unchanged after + * one plaintext property read. The dynamic branch exists for runs started from + * source: their workflow function was never in the bundle, so the code came + * with the run, and replaying it means evaluating that exact code rather than + * whatever the deployment now contains. + * + * Stored code is only executed when all of these hold, and each failure is a + * `WorkflowRuntimeError` so the caller fails the run instead of redelivering a + * message whose verdict cannot change: + * + * - this deployment has opted in to dynamic workflows; + * - the run's `workflowName` is the id its `dynamicWorkflow` marker derives + * (a marker on a static workflow's run does not redirect it to stored code); + * - the code exists and hydrates, which requires the `encr` envelope whenever + * the run has key material or was started with encryption. + * + * Two sources for the code, in order of what the invocation already holds: + * + * 1. On the run snapshot. The normal case: `run_created` and the queue + * message both carry the bytes, so a read is already unnecessary by the + * time replay begins. + * 2. Read back from the run. Needed when the definition was too large to + * send inline and lives behind a ref, or the snapshot was read with + * `resolveData: 'none'`, which omits the code. The run has to exist first, + * hence the barrier. + * + * World and key-lookup errors propagate unchanged, so transient failures are + * still redelivered. + * + * @param getEncryptionKey - Resolved only on the dynamic branch. The key is + * lazy for a reason: some deliveries never need it, and forcing it here + * would put a key fetch on every static run's critical path. + * @param awaitRunReady - Orders the fallback read after the write that + * creates the run; a no-op once the run is durable. + */ +async function resolveWorkflowCodeForRun( + staticWorkflowCode: string, + workflowRun: WorkflowRun, + getEncryptionKey: () => Promise, + world: World, + awaitRunReady: () => Promise +): Promise { + const dynamicWorkflow = readDynamicWorkflowMetadata( + workflowRun.executionContext + ); + if (!dynamicWorkflow) return { code: staticWorkflowCode }; + + const runLabel = `Workflow run "${workflowRun.runId}" is a dynamic workflow run (source ${dynamicWorkflow.sourceHash.slice(0, 12)})`; + if (!isDynamicWorkflowsEnabled()) { + throw new WorkflowRuntimeError( + `${runLabel}, but this deployment has not enabled dynamic workflows, so its stored code was not executed. Set ${DYNAMIC_WORKFLOWS_ENV}=1 on the deployment to enable them.` + ); + } + const expectedWorkflowName = dynamicWorkflowName(dynamicWorkflow); + if (workflowRun.workflowName !== expectedWorkflowName) { + throw new WorkflowRuntimeError( + `${runLabel}, but its workflow name ${JSON.stringify(workflowRun.workflowName)} does not match the dynamic id ${JSON.stringify(expectedWorkflowName)} its marker derives, so its stored code was not executed.` + ); + } + + let stored = workflowRun.dynamicWorkflowCode; + if (stored === undefined) { + await awaitRunReady(); + stored = (await world.runs.get(workflowRun.runId, { resolveData: 'all' })) + .dynamicWorkflowCode; + } + + if (stored === undefined) { + throw new WorkflowRuntimeError( + `${runLabel}, but its stored workflow code is missing, so it cannot be replayed. ` + + 'This means the code was never persisted or its storage has expired.' + ); + } + + const encryptionKey = await getEncryptionKey(); + const features = workflowRun.executionContext?.features as + | { encryption?: unknown } + | undefined; + try { + return { + code: await hydrateDynamicWorkflowCode(stored, encryptionKey, { + encryptionRequired: features?.encryption === true, + }), + dynamicWorkflow, + }; + } catch (cause) { + throw new WorkflowRuntimeError( + `${runLabel}, but its stored workflow code could not be decoded, so it was not executed: ${cause instanceof Error ? cause.message : String(cause)}`, + { cause } + ); + } +} + /** * Creates a single route which handles workflow execution requests, * executing steps inline when possible to reduce function invocations @@ -1015,6 +1132,17 @@ export function workflowEntrypoint( > ) => { if (!workflow || useQuickJSVm(workflow)) return; + // A dynamic run does not replay the deployment's + // bundle, and its own code is not available yet (it is + // encrypted, and resolving it needs the run's key). Skip + // the warm-up rather than cache scripts the replay must + // not use: `replayWorkflow` compiles the resolved code + // itself when no compiled scripts are handed to it. + if ( + readDynamicWorkflowMetadata(workflow.executionContext) + ) { + return; + } if (compiledWorkflowName !== workflow.workflowName) { compiledWorkflowName = workflow.workflowName; compiledWorkflowScripts = compileWorkflowBundle( @@ -2517,6 +2645,13 @@ export function workflowEntrypoint( attributes: runInput.attributes, allowReservedAttributes: runInput.allowReservedAttributes, + // Resilient start: this event is what creates + // the run when `run_created` never landed, so a + // dynamic run's code has to come with it or the + // run is created unable to replay. + dynamicWorkflowCode: runInput.dynamicWorkflowCode, + dynamicWorkflowCodeRef: + runInput.dynamicWorkflowCodeRef, }, } : {}), @@ -2589,6 +2724,19 @@ export function workflowEntrypoint( specVersion: runInput.specVersion, executionContext: runInput.executionContext, input: runInput.input, + // A dynamic run's code rides the queue message for + // exactly this: turbo synthesizes the run snapshot + // instead of waiting for the `run_started` response, + // and without the code there would be nothing to + // execute on the first delivery. Absent when the + // definition was too large to send inline, in which + // case `resolveWorkflowCodeForRun` reads it back from + // the run instead. + ...(runInput.dynamicWorkflowCode + ? { + dynamicWorkflowCode: runInput.dynamicWorkflowCode, + } + : {}), // Seed attributes from start() ride along in `runInput` // (they live in `run_created`'s eventData, not separate // `attr_set` events), so the synthesized snapshot carries @@ -2919,6 +3067,55 @@ export function workflowEntrypoint( } // end else (re-ensure needed) } + // The code this run replays. For every static run that is + // the deployment's bundle, unchanged after one plaintext + // property read. A dynamic run brings its own — resolved + // once here, since the code is fixed for the run's lifetime + // and each replay iteration below would otherwise redo the + // decrypt. + // + // A refusal to execute stored code (opt-in off, a marker + // that does not match the run, missing or undecodable code) + // is a WorkflowRuntimeError and fails the run: every + // redelivery would reach the same verdict. + let resolvedWorkflowCode: ResolvedWorkflowCode; + try { + resolvedWorkflowCode = await resolveWorkflowCodeForRun( + workflowCode, + workflowRun, + () => encryptionKey.value, + world, + awaitRunReady + ); + } catch (err) { + await awaitRunReady(); + if (!(await recordWorkflowSetupFailure(err))) { + throw err; + } + return; + } + const effectiveWorkflowCode = resolvedWorkflowCode.code; + const dynamicWorkflowMetadata = + resolvedWorkflowCode.dynamicWorkflow; + if (dynamicWorkflowMetadata) { + span?.setAttributes({ + ...Attribute.WorkflowDynamic(true), + ...Attribute.WorkflowDynamicSourceHash( + dynamicWorkflowMetadata.sourceHash + ), + }); + runLogger.info('Executing stored dynamic workflow code', { + sourceHash: dynamicWorkflowMetadata.sourceHash, + }); + } + const dynamicWorkflowScripts = + dynamicWorkflowMetadata && !useQuickJSVm(workflowRun) + ? compileDynamicWorkflowBundle( + effectiveWorkflowCode, + workflowRun.workflowName + ) + : undefined; + // The live VM parked at the previous boundary, when the // retention decision kept it. null → this iteration cold- // replays. Invocation-scoped: dies with this delivery. @@ -3072,7 +3269,7 @@ export function workflowEntrypoint( './runtime/quickjs-entrypoint.js' ); const quickjsResult = await runWorkflowWithQuickJS({ - workflowCode, + workflowCode: effectiveWorkflowCode, workflowName, workflowRun, preloadedEvents: @@ -3411,17 +3608,25 @@ export function workflowEntrypoint( if (workflowResult.type === 'replay') { retainedSession = null; const compiled = startWorkflowCompile(workflowRun); + // Static bundles use the process-wide cache; dynamic + // source is compiled once per invocation without + // entering that shared cache. assert( - compiled, + compiled || dynamicWorkflowScripts, 'Node workflow replay requires compiled scripts' ); workflowResult = await replayWorkflow({ - workflowCode, + workflowCode: effectiveWorkflowCode, workflowRun, events: eventLog.events, encryptionKey: await encryptionKey.value, replayPayloadCache, - compiledWorkflowScripts: await compiled, + ...((compiled ?? dynamicWorkflowScripts) + ? { + compiledWorkflowScripts: await (compiled ?? + dynamicWorkflowScripts), + } + : {}), // Turbo: the end-of-run drain inside workflow // execution commits fire-and-forget `*_created` // events before the terminal `awaitRunReady()` below. @@ -5350,7 +5555,7 @@ export function workflowEntrypoint( errorStack = remapErrorStack( errorStack, filename, - workflowCode + effectiveWorkflowCode ); } diff --git a/packages/core/src/runtime/constants.ts b/packages/core/src/runtime/constants.ts index 5f1e57ab5a..8496f37532 100644 --- a/packages/core/src/runtime/constants.ts +++ b/packages/core/src/runtime/constants.ts @@ -419,6 +419,27 @@ export function isVmRetentionEnabled(): boolean { return !(raw === '0' || raw.toLowerCase() === 'false'); } +/** Environment variable that opts a deployment into dynamic workflows. */ +export const DYNAMIC_WORKFLOWS_ENV = 'WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS'; + +/** + * Whether this deployment executes dynamic workflows (default OFF). + * + * Dynamic source runs with the full privileges of the deployment's functions, + * so a deployment must opt in before it starts a dynamic run, advertises + * dynamic support in its health check, or executes stored dynamic code on a + * delivery. Only `1` or `true` (case-insensitive) enables it; any other value + * leaves it off. + * + * Reads `process.env.WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS` on every call so + * the deployment's runtime environment decides, not the build. + */ +export function isDynamicWorkflowsEnabled(): boolean { + const raw = process.env[DYNAMIC_WORKFLOWS_ENV]; + if (raw === undefined) return false; + return raw === '1' || raw.toLowerCase() === 'true'; +} + /** * Whether inline step ownership is enabled (default ON). When on, the lazy * `step_started` that creates an inline step records the owning queue diff --git a/packages/core/src/runtime/dynamic-start-types.test.ts b/packages/core/src/runtime/dynamic-start-types.test.ts new file mode 100644 index 0000000000..d0715ccf5d --- /dev/null +++ b/packages/core/src/runtime/dynamic-start-types.test.ts @@ -0,0 +1,82 @@ +/** + * Type-level checks that the dynamic-source overloads resolve the way the + * docs show them, and that adding them did not shadow the static ones. + * + * Every assertion lives inside a function that is never called: these are + * compile-time checks, and actually invoking `start()` here would attempt + * real world writes. + */ +import { describe, expect, expectTypeOf, it } from 'vitest'; +import type { Run } from './run.js'; +import { start } from './start.js'; + +const SOURCE = `async function workflow() { "use workflow"; }`; +const steps = { fetchUser: { stepId: 'step//./src/steps//fetchUser' } }; + +describe('start() overload resolution', () => { + it('returns Run for dynamic source, with or without args', () => { + function _check() { + expectTypeOf( + start(SOURCE, [{ userId: 'u_1' }], { experimental_dynamic: { steps } }) + ).toEqualTypeOf>>(); + expectTypeOf( + start(SOURCE, { experimental_dynamic: { steps } }) + ).toEqualTypeOf>>(); + } + expect(typeof _check).toBe('function'); + }); + + it('accepts experimental_dynamic.exportName alongside the shared start options', () => { + function _check() { + expectTypeOf( + start(SOURCE, [], { + experimental_dynamic: { steps, exportName: 'orchestrate' }, + attributes: { tenant: 'acme' }, + }) + ).toEqualTypeOf>>(); + } + expect(typeof _check).toBe('function'); + }); + + it('still infers the result type of a static workflow function', () => { + function _check() { + const workflow = Object.assign(async () => 42, { + workflowId: 'workflow//./wf//run', + }); + expectTypeOf(start(workflow, [])).toEqualTypeOf>>(); + } + expect(typeof _check).toBe('function'); + }); + + it('accepts an imported step function in experimental_dynamic.steps', () => { + function _check() { + // The documented call: real step imports, whose `.stepId` the + // build-time transform stamps at runtime and never adds to their type. + // A `{ stepId: string }`-only parameter type rejected this and accepted + // only the escape hatch below — which is exactly what app-side e2e + // fixtures caught, since the runner only ever used the escape hatch. + const add = async (a: number, b: number) => a + b; + expectTypeOf( + start(SOURCE, [], { experimental_dynamic: { steps: { add } } }) + ).toEqualTypeOf>>(); + expectTypeOf( + start(SOURCE, [], { + experimental_dynamic: { + steps: { add: { stepId: 'step//./steps//add' } }, + }, + }) + ).toEqualTypeOf>>(); + } + expect(typeof _check).toBe('function'); + }); + + it('requires `experimental_dynamic` when the first argument is source', () => { + function _check() { + // @ts-expect-error - source without experimental options is not valid + start(SOURCE, []); + // @ts-expect-error - the unreleased old spelling is not public API + start(SOURCE, [], { dynamic: { steps } }); + } + expect(typeof _check).toBe('function'); + }); +}); diff --git a/packages/core/src/runtime/dynamic-workflow-delivery.test.ts b/packages/core/src/runtime/dynamic-workflow-delivery.test.ts new file mode 100644 index 0000000000..aa9f9f066a --- /dev/null +++ b/packages/core/src/runtime/dynamic-workflow-delivery.test.ts @@ -0,0 +1,399 @@ +import { context, trace as otelTrace } from '@opentelemetry/api'; +import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks'; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from '@opentelemetry/sdk-trace-base'; +import { RUN_ERROR_CODES, WorkflowWorldError } from '@workflow/errors'; +import { + type Event, + SPEC_VERSION_CURRENT, + type WorkflowRun, +} from '@workflow/world'; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from 'vitest'; +import { importKey } from '../encryption.js'; +import { runtimeLogger } from '../logger.js'; +import { workflowEntrypoint } from '../runtime.js'; +import { deriveRunKeyPair } from '../sealed-box.js'; +import { + dehydrateDynamicWorkflowCode, + dehydrateWorkflowArguments, + sealTo, +} from '../serialization.js'; +import { DYNAMIC_WORKFLOWS_ENV } from './constants.js'; +import { + compileDynamicWorkflow, + type DynamicWorkflowMetadata, +} from './dynamic-workflow.js'; +import { setWorld } from './world.js'; + +vi.mock('@vercel/functions', () => ({ + waitUntil: vi.fn((p: Promise) => { + p.catch(() => {}); + }), +})); + +const exporter = new InMemorySpanExporter(); +const provider = new BasicTracerProvider(); +const contextManager = new AsyncLocalStorageContextManager(); + +beforeAll(() => { + provider.addSpanProcessor(new SimpleSpanProcessor(exporter)); + contextManager.enable(); + context.setGlobalContextManager(contextManager); + otelTrace.setGlobalTracerProvider(provider); +}); + +afterAll(async () => { + await provider.shutdown(); + context.disable(); + otelTrace.disable(); +}); + +const RUN_ID = 'wrun_01JDYNAMICDELIVERY000000000'; +const KEY_MATERIAL = new Uint8Array(32).fill(0x42); +const STATIC_WORKFLOW_CODE = `async function staticWorkflow() { return 'static'; } +;globalThis.__private_workflows = new Map(); +globalThis.__private_workflows.set('workflow//./src/static//staticWorkflow', staticWorkflow);`; + +async function compileReturning(value: number) { + return compileDynamicWorkflow( + `async function workflow() { "use workflow"; return ${value}; }`, + { steps: { noop: { stepId: 'step//./test//noop' } } } + ); +} + +/** + * Deliver one queue message for a run and record what the handler wrote. + * + * `storedCode` is the run's `dynamicWorkflowCode` on the snapshot the + * `run_started` response returns; `readBackCode` is what `runs.get` returns + * when the delivery has to read it from the run. + */ +async function deliver(options: { + workflowName: string; + executionContext: Record; + storedCode?: Uint8Array; + readBack?: () => Promise>; + encryption?: boolean; +}) { + const workflowRun: WorkflowRun = { + runId: RUN_ID, + workflowName: options.workflowName, + status: 'running', + input: await dehydrateWorkflowArguments([], RUN_ID, undefined, []), + deploymentId: 'dpl_current', + specVersion: SPEC_VERSION_CURRENT, + executionContext: options.executionContext, + ...(options.storedCode ? { dynamicWorkflowCode: options.storedCode } : {}), + startedAt: new Date('2026-07-30T00:00:00.000Z'), + createdAt: new Date('2026-07-30T00:00:00.000Z'), + updatedAt: new Date('2026-07-30T00:00:00.000Z'), + }; + + const createdEvents: any[] = []; + const eventsCreate = vi.fn(async (_runId: string, data: any) => { + createdEvents.push(data); + if (data.eventType === 'run_started') { + return { run: workflowRun, events: [] as Event[] }; + } + return { + event: { + eventId: `evnt_${createdEvents.length}`, + runId: RUN_ID, + createdAt: new Date(), + ...data, + }, + }; + }); + const runsGet = vi.fn(async () => ({ + ...workflowRun, + ...(await options.readBack?.()), + })); + + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn(async () => 'dpl_current'), + createQueueHandler: vi.fn( + ( + _prefix: string, + handler: (message: unknown, metadata: unknown) => Promise + ) => + async () => { + await handler( + { runId: RUN_ID, requestedAt: new Date() }, + { + requestId: 'req_test', + attempt: 1, + queueName: '__wkf_workflow_dynamic', + messageId: 'msg_test', + } + ); + return new Response(null, { status: 204 }); + } + ), + events: { + create: eventsCreate, + list: vi.fn(async () => ({ + data: [] as Event[], + hasMore: false, + cursor: 'cursor_test', + })), + }, + runs: { get: runsGet }, + queue: vi.fn(async () => ({ messageId: null })), + getEncryptionKeyForRun: vi.fn(async () => + options.encryption === false ? undefined : KEY_MATERIAL + ), + } as any); + + const response = await workflowEntrypoint(STATIC_WORKFLOW_CODE)( + new Request('https://example.test') + ); + const eventTypes = createdEvents.map((event) => event.eventType); + const runFailed = createdEvents.find( + (event) => event.eventType === 'run_failed' + ); + return { response, eventTypes, runFailed, runsGet }; +} + +function contextFor(metadata: DynamicWorkflowMetadata, encryption = true) { + return { + features: { encryption }, + dynamicWorkflow: metadata, + }; +} + +async function encryptedCode(code: string) { + return dehydrateDynamicWorkflowCode(code, await importKey(KEY_MATERIAL)); +} + +describe('dynamic workflow delivery', () => { + let errorLog: ReturnType; + let runLogger: Record>; + + beforeEach(() => { + vi.stubEnv(DYNAMIC_WORKFLOWS_ENV, '1'); + errorLog = vi.spyOn(runtimeLogger, 'error').mockImplementation(() => {}); + vi.spyOn(runtimeLogger, 'warn').mockImplementation(() => {}); + vi.spyOn(runtimeLogger, 'info').mockImplementation(() => {}); + // The per-run logger the handler scopes to this delivery. + runLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + child: vi.fn(), + forRun: vi.fn(), + }; + runLogger.child.mockReturnValue(runLogger); + runLogger.forRun.mockReturnValue(runLogger); + vi.spyOn(runtimeLogger, 'forRun').mockReturnValue(runLogger as never); + }); + + afterEach(() => { + exporter.reset(); + setWorld(undefined); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + function failureMessages(): string[] { + return errorLog.mock.calls.map(([, metadata]) => + String((metadata as { error?: unknown } | undefined)?.error) + ); + } + + it('executes encrypted stored code and records it on the span and in one log line', async () => { + const compiled = await compileReturning(7); + + const { response, eventTypes } = await deliver({ + workflowName: compiled.workflowName, + executionContext: contextFor(compiled.metadata), + storedCode: await encryptedCode(compiled.workflowCode), + }); + + expect(response.status).toBe(204); + expect(eventTypes).toContain('run_completed'); + expect(eventTypes).not.toContain('run_failed'); + + const span = exporter + .getFinishedSpans() + .find((s) => s.name.startsWith('workflow.execute')); + expect(span?.attributes['workflow.dynamic']).toBe(true); + expect(span?.attributes['workflow.dynamic.source_hash']).toBe( + compiled.metadata.sourceHash + ); + + // Scoped to this run by `forRun`, once for the invocation. + expect(runtimeLogger.forRun).toHaveBeenCalledWith( + RUN_ID, + expect.anything() + ); + const executionLogs = runLogger.info.mock.calls.filter( + ([message]) => message === 'Executing stored dynamic workflow code' + ); + expect(executionLogs).toEqual([ + [ + 'Executing stored dynamic workflow code', + { sourceHash: compiled.metadata.sourceHash }, + ], + ]); + }); + + it('fails the run when this deployment has not opted in', async () => { + vi.stubEnv(DYNAMIC_WORKFLOWS_ENV, undefined); + const compiled = await compileReturning(7); + + const { response, eventTypes, runFailed } = await deliver({ + workflowName: compiled.workflowName, + executionContext: contextFor(compiled.metadata), + storedCode: await encryptedCode(compiled.workflowCode), + }); + + // Acked, not thrown: a redelivery would reach the same verdict. + expect(response.status).toBe(204); + expect(eventTypes).not.toContain('run_completed'); + expect(runFailed?.eventData.errorCode).toBe(RUN_ERROR_CODES.RUNTIME_ERROR); + expect(failureMessages().join('\n')).toMatch( + /has not enabled dynamic workflows.*WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1/ + ); + expect( + exporter + .getFinishedSpans() + .some((s) => s.attributes['workflow.dynamic'] === true) + ).toBe(false); + }); + + it('fails a static run carrying a forged marker instead of retrying', async () => { + const compiled = await compileReturning(7); + + const { response, eventTypes, runFailed, runsGet } = await deliver({ + workflowName: 'workflow//./src/static//staticWorkflow', + executionContext: contextFor(compiled.metadata), + storedCode: await encryptedCode(compiled.workflowCode), + }); + + expect(response.status).toBe(204); + expect(eventTypes).not.toContain('run_completed'); + expect(runFailed?.eventData.errorCode).toBe(RUN_ERROR_CODES.RUNTIME_ERROR); + expect(failureMessages().join('\n')).toMatch( + /does not match the dynamic id/ + ); + expect(runsGet).not.toHaveBeenCalled(); + }); + + it('fails the run when its stored code is missing', async () => { + const compiled = await compileReturning(7); + + const { response, runFailed, runsGet } = await deliver({ + workflowName: compiled.workflowName, + executionContext: contextFor(compiled.metadata), + readBack: async () => ({}), + }); + + expect(response.status).toBe(204); + expect(runsGet).toHaveBeenCalledWith(RUN_ID, { resolveData: 'all' }); + expect(runFailed?.eventData.errorCode).toBe(RUN_ERROR_CODES.RUNTIME_ERROR); + expect(failureMessages().join('\n')).toMatch( + /stored workflow code is missing/ + ); + }); + + it('fails the run when its stored code is plaintext although it has a key', async () => { + const compiled = await compileReturning(7); + + const { response, eventTypes, runFailed } = await deliver({ + workflowName: compiled.workflowName, + executionContext: contextFor(compiled.metadata), + storedCode: await dehydrateDynamicWorkflowCode( + compiled.workflowCode, + undefined + ), + }); + + expect(response.status).toBe(204); + expect(eventTypes).not.toContain('run_completed'); + expect(runFailed?.eventData.errorCode).toBe(RUN_ERROR_CODES.RUNTIME_ERROR); + expect(failureMessages().join('\n')).toMatch(/must be encrypted.*"devl"/); + }); + + it('fails the run when its stored code is sealed to the run public key', async () => { + const compiled = await compileReturning(7); + const { publicKey } = await deriveRunKeyPair(KEY_MATERIAL); + + const { response, eventTypes, runFailed } = await deliver({ + workflowName: compiled.workflowName, + executionContext: contextFor(compiled.metadata), + storedCode: await dehydrateDynamicWorkflowCode( + compiled.workflowCode, + sealTo(publicKey) + ), + }); + + expect(response.status).toBe(204); + expect(eventTypes).not.toContain('run_completed'); + expect(runFailed?.eventData.errorCode).toBe(RUN_ERROR_CODES.RUNTIME_ERROR); + expect(failureMessages().join('\n')).toMatch(/must be encrypted.*"encp"/); + }); + + it('executes plaintext stored code in a World without encryption', async () => { + const compiled = await compileReturning(3); + + const { eventTypes } = await deliver({ + workflowName: compiled.workflowName, + executionContext: contextFor(compiled.metadata, false), + storedCode: await dehydrateDynamicWorkflowCode( + compiled.workflowCode, + undefined + ), + encryption: false, + }); + + expect(eventTypes).toContain('run_completed'); + expect(eventTypes).not.toContain('run_failed'); + }); + + it('fails the run when reading the stored code back hits corrupt data', async () => { + const compiled = await compileReturning(7); + + const { response, runFailed } = await deliver({ + workflowName: compiled.workflowName, + executionContext: contextFor(compiled.metadata), + readBack: async () => { + throw new WorkflowWorldError('corrupt stored code', { + code: 'WORLD_CONTRACT_ERROR', + }); + }, + }); + + expect(response.status).toBe(204); + expect(runFailed?.eventData.errorCode).toBe( + RUN_ERROR_CODES.WORLD_CONTRACT_ERROR + ); + }); + + it('redelivers when reading the stored code back fails transiently', async () => { + const compiled = await compileReturning(7); + + await expect( + deliver({ + workflowName: compiled.workflowName, + executionContext: contextFor(compiled.metadata), + readBack: async () => { + throw new WorkflowWorldError('backend unavailable', { status: 503 }); + }, + }) + ).rejects.toThrow('backend unavailable'); + }); +}); diff --git a/packages/core/src/runtime/dynamic-workflow.test.ts b/packages/core/src/runtime/dynamic-workflow.test.ts new file mode 100644 index 0000000000..07d8099a88 --- /dev/null +++ b/packages/core/src/runtime/dynamic-workflow.test.ts @@ -0,0 +1,547 @@ +import { createContext, runInContext, Script } from 'node:vm'; +import { WorkflowRuntimeError } from '@workflow/errors'; +import { describe, expect, it } from 'vitest'; +import { + compileDynamicWorkflow, + DYNAMIC_WORKFLOW_SOURCE_MAX_BYTES, + readDynamicWorkflowMetadata, +} from './dynamic-workflow.js'; + +const SOURCE = ` +async function workflow(input) { + "use workflow"; + const user = await steps.fetchUser(input.userId); + await steps.sendEmail(user.email); + return { ok: true }; +} +`; + +const STEPS = { + fetchUser: { stepId: 'step//./src/steps//fetchUser' }, + sendEmail: { stepId: 'step//./src/steps//sendEmail' }, +}; + +describe('compileDynamicWorkflow', () => { + describe('generated code', () => { + it('registers the function under the generated workflow id', async () => { + const compiled = await compileDynamicWorkflow(SOURCE, { steps: STEPS }); + + expect(compiled.workflowName).toMatch( + /^workflow\/\/dynamic\/[0-9a-f]{32}\/\/workflow$/ + ); + expect(compiled.workflowCode).toContain( + `__dynamicGlobalThis.__private_workflows.set(${JSON.stringify(compiled.workflowName)}, __dynamicWorkflow)` + ); + // The id has to be on the function too — the runtime reads it back off + // the registered function, same as a build-time transform stamps it. + expect(compiled.workflowCode).toContain( + `Object.defineProperty(__dynamicWorkflow, "workflowId"` + ); + expect(compiled.workflowCode).toContain(SOURCE.trim()); + }); + + it('binds only the aliases it was given, through WORKFLOW_USE_STEP', async () => { + const compiled = await compileDynamicWorkflow(SOURCE, { steps: STEPS }); + + expect(compiled.workflowCode).toContain( + '["fetchUser"]: __dynamicUseStep("step//./src/steps//fetchUser")' + ); + expect(compiled.workflowCode).toContain( + '["sendEmail"]: __dynamicUseStep("step//./src/steps//sendEmail")' + ); + // Frozen so ordinary generated code that reaches for a step it was not + // given fails the run rather than silently adding one. + expect(compiled.workflowCode).toContain( + 'const __dynamicSteps = Object.freeze({' + ); + expect(compiled.workflowCode).not.toContain('notAllowed'); + }); + + it('exposes sleep and createHook from the VM globals', async () => { + const compiled = await compileDynamicWorkflow(SOURCE, { steps: STEPS }); + + expect(compiled.workflowCode).toContain( + 'const __dynamicSleep = __dynamicGlobalThis[Symbol.for("WORKFLOW_SLEEP")]' + ); + expect(compiled.workflowCode).toContain( + 'const __dynamicCreateHook = __dynamicGlobalThis[Symbol.for("WORKFLOW_CREATE_HOOK")]' + ); + }); + + it('honours a custom exportName', async () => { + const source = `async function orchestrate() { "use workflow"; await steps.fetchUser(); }`; + + const compiled = await compileDynamicWorkflow(source, { + steps: STEPS, + exportName: 'orchestrate', + }); + + expect(compiled.workflowName).toMatch(/\/\/orchestrate$/); + expect(compiled.metadata.exportName).toBe('orchestrate'); + expect(compiled.workflowCode).toContain( + '__dynamicGlobalThis.__private_workflows.set(' + ); + }); + }); + + describe('workflow id derivation', () => { + it('is stable across calls', async () => { + const a = await compileDynamicWorkflow(SOURCE, { steps: STEPS }); + const b = await compileDynamicWorkflow(SOURCE, { steps: STEPS }); + + expect(a.workflowName).toBe(b.workflowName); + expect(a.metadata.sourceHash).toBe(b.metadata.sourceHash); + }); + + it('does not depend on the order the steps were declared in', async () => { + // Otherwise the same workflow would land on two different queue topics + // depending on how the caller happened to build the object. + const a = await compileDynamicWorkflow(SOURCE, { steps: STEPS }); + const b = await compileDynamicWorkflow(SOURCE, { + steps: { + sendEmail: STEPS.sendEmail, + fetchUser: STEPS.fetchUser, + }, + }); + + expect(a.workflowName).toBe(b.workflowName); + }); + + it('changes when the source changes', async () => { + const a = await compileDynamicWorkflow(SOURCE, { steps: STEPS }); + const b = await compileDynamicWorkflow( + SOURCE.replace('{ ok: true }', '{ ok: false }'), + { steps: STEPS } + ); + + expect(a.workflowName).not.toBe(b.workflowName); + }); + + it('changes when a step binding changes', async () => { + // The bindings are as much part of what executes as the source is: the + // same text over different steps is a different workflow. + const a = await compileDynamicWorkflow(SOURCE, { steps: STEPS }); + const b = await compileDynamicWorkflow(SOURCE, { + steps: { + ...STEPS, + sendEmail: { stepId: 'step//./src/steps//sendSms' }, + }, + }); + + expect(a.workflowName).not.toBe(b.workflowName); + }); + }); + + describe('metadata', () => { + it('records the version, hash, export name and step bindings', async () => { + const compiled = await compileDynamicWorkflow(SOURCE, { steps: STEPS }); + + expect(compiled.metadata).toEqual({ + version: 1, + sourceHash: expect.stringMatching(/^[0-9a-f]{64}$/), + exportName: 'workflow', + steps: { + fetchUser: 'step//./src/steps//fetchUser', + sendEmail: 'step//./src/steps//sendEmail', + }, + }); + }); + + it('carries no source or code, so it stays inside the plaintext budget', async () => { + // The metadata is readable without decrypting anything, so the code + // must not be in it — that is the whole point of storing the code + // encrypted behind a ref. + const compiled = await compileDynamicWorkflow(SOURCE, { steps: STEPS }); + + const encoded = JSON.stringify(compiled.metadata); + expect(encoded).not.toContain('use workflow'); + expect(new TextEncoder().encode(encoded).byteLength).toBeLessThan(1024); + }); + }); + + describe('validation', () => { + it('rejects source with no matching async function', async () => { + await expect( + compileDynamicWorkflow('const x = 1;', { steps: STEPS }) + ).rejects.toThrow(/must declare `async function workflow/); + }); + + it('rejects a non-async function', async () => { + await expect( + compileDynamicWorkflow('function workflow() { "use workflow"; }', { + steps: STEPS, + }) + ).rejects.toThrow(/must declare `async function workflow/); + }); + + it.each([ + [ + 'TypeScript annotations', + 'async function workflow(input: { id: string }) {\n "use workflow";\n return 1;\n}', + undefined, + ], + [ + 'a reserved word as the function name', + 'async function class(input) {\n "use workflow";\n return 1;\n}', + 'class', + ], + [ + 'an unbalanced brace', + 'async function workflow(input) {\n "use workflow";\n if (input) {\n return 1;\n}', + undefined, + ], + ])('rejects source the engine cannot parse: %s', async (_label, source, exportName) => { + // The shallow checks above pass all of these; without parsing the + // generated code here they would only fail at replay, on every + // delivery of a run that already exists. + await expect( + compileDynamicWorkflow(source, { steps: STEPS, exportName }) + ).rejects.toThrow(/not valid JavaScript/); + }); + + it('rejects an inline "use step" function', async () => { + // Nothing transforms dynamic source, so the directive would be inert + // and the function would run inside the workflow VM. + const source = `async function workflow(input) { + "use workflow"; + async function fetchUser(id) { + "use step"; + return fetch("https://example.com/" + id); + } + return fetchUser(input.id); +}`; + await expect( + compileDynamicWorkflow(source, { steps: STEPS }) + ).rejects.toThrow(/cannot declare "use step"/); + }); + + it.each([ + [ + 'explicit resource management syntax', + `async function workflow() { + "use workflow"; + using resource = null; + return resource; +}`, + ], + [ + 'regular expression modifiers', + `async function workflow() { + "use workflow"; + return /(?i:a)/.test("A"); +}`, + ], + ])('rejects syntax newer than the pinned 2024 grammar: %s', async (_label, source) => { + await expect( + compileDynamicWorkflow(source, { steps: STEPS }) + ).rejects.toThrow(/not valid JavaScript/); + }); + + it('accepts representative ECMAScript 2024 syntax', async () => { + const source = ` +async function workflow(input = {}) { + "use workflow"; + const copy = structuredClone(input); + return { ...copy, value: copy?.value ?? 1 }; +} +`; + const compiled = await compileDynamicWorkflow(source, { steps: STEPS }); + expect(() => new Script(compiled.workflowCode)).not.toThrow(); + }); + + it.each([ + [ + 'nested declaration', + 'function outer() { async function workflow() { "use workflow"; } }', + ], + [ + 'comment spoof', + '// async function workflow() { "use workflow"; }\nconst value = 1;', + ], + [ + 'string spoof', + 'const value = `async function workflow() { "use workflow"; }`;', + ], + ])('rejects a %s instead of a top-level declaration', async (_label, source) => { + await expect( + compileDynamicWorkflow(source, { steps: STEPS }) + ).rejects.toThrow(/at top level/); + }); + + it.each([ + ['Object', 'const Object = null;'], + ['Map', 'let Map = null;'], + ['Symbol', 'function Symbol() {}'], + ['globalThis', 'var globalThis = null;'], + ['__dynamicUseStep', 'const __dynamicUseStep = null;'], + ['__dynamicWorkflow', 'let __dynamicWorkflow = null;'], + ['__dynamicGlobalThis', 'function __dynamicGlobalThis() {}'], + ['steps', 'var steps = null;'], + ['sleep', 'const sleep = null;'], + ['createHook', 'let createHook = null;'], + ['Error', 'class Error {}'], + ])('isolates caller %s declarations from wrapper bindings', async (_binding, declaration) => { + const source = ` +${declaration} +async function workflow() { + "use workflow"; + return 1; +} +`; + const compiled = await compileDynamicWorkflow(source, { steps: STEPS }); + const sandbox = { + [Symbol.for('WORKFLOW_USE_STEP')]: () => async () => undefined, + [Symbol.for('WORKFLOW_SLEEP')]: async () => undefined, + [Symbol.for('WORKFLOW_CREATE_HOOK')]: () => Promise.resolve(), + }; + const context = createContext(sandbox); + runInContext(compiled.workflowCode, context); + const workflow = runInContext( + `globalThis.__private_workflows.get(${JSON.stringify(compiled.workflowName)})`, + context + ) as { workflowId?: string }; + expect(workflow).toBeTypeOf('function'); + expect(workflow.workflowId).toBe(compiled.workflowName); + }); + + it('emits __proto__ as a callable own step without changing the catalog prototype', async () => { + const source = ` +async function workflow() { + "use workflow"; + return { + own: Object.hasOwn(steps, "__proto__"), + ordinaryPrototype: Object.getPrototypeOf(steps) === Object.prototype, + result: await steps.__proto__(), + }; +} +`; + const stepId = 'step//./src/steps//prototypeAlias'; + const compiled = await compileDynamicWorkflow(source, { + steps: { ['__proto__']: { stepId } }, + }); + const sandbox = { + [Symbol.for('WORKFLOW_USE_STEP')]: (id: string) => async () => id, + [Symbol.for('WORKFLOW_SLEEP')]: async () => undefined, + [Symbol.for('WORKFLOW_CREATE_HOOK')]: () => Promise.resolve(), + }; + const context = createContext(sandbox); + runInContext(compiled.workflowCode, context); + const workflow = runInContext( + `globalThis.__private_workflows.get(${JSON.stringify(compiled.workflowName)})`, + context + ) as () => Promise>; + + expect(await workflow()).toEqual({ + own: true, + ordinaryPrototype: true, + result: stepId, + }); + expect(compiled.metadata.steps).toEqual( + Object.fromEntries([['__proto__', stepId]]) + ); + expect(compiled.workflowCode).toContain('["__proto__"]:'); + }); + + it('registers a workflow that closes over all injected bindings', async () => { + const source = ` +async function workflow() { + "use workflow"; + await steps.fetchUser("u_1"); + await sleep("1s"); + return createHook({ token: "hook_1" }); +} +`; + const compiled = await compileDynamicWorkflow(source, { steps: STEPS }); + const calls: string[] = []; + const hook = Promise.resolve('hook-result'); + const sandbox = { + [Symbol.for('WORKFLOW_USE_STEP')]: (stepId: string) => async () => { + calls.push(stepId); + }, + [Symbol.for('WORKFLOW_SLEEP')]: async () => { + calls.push('sleep'); + }, + [Symbol.for('WORKFLOW_CREATE_HOOK')]: () => hook, + }; + const context = createContext(sandbox); + runInContext(compiled.workflowCode, context); + const workflow = runInContext( + `globalThis.__private_workflows.get(${JSON.stringify(compiled.workflowName)})`, + context + ) as () => Promise; + expect(await workflow()).toBe('hook-result'); + expect(calls).toEqual(['step//./src/steps//fetchUser', 'sleep']); + }); + + it('accepts a genuine top-level async declaration without evaluating source', async () => { + const marker = '__dynamicWorkflowValidationExecuted'; + delete (globalThis as Record)[marker]; + const source = ` +globalThis.${marker} = true; +async function workflow() { + "use workflow"; + return 1; +} +`; + await expect( + compileDynamicWorkflow(source, { steps: STEPS }) + ).resolves.toMatchObject({ metadata: { exportName: 'workflow' } }); + expect((globalThis as Record)[marker]).toBeUndefined(); + }); + + it('rejects a missing "use workflow" directive', async () => { + await expect( + compileDynamicWorkflow( + 'async function workflow() { await steps.fetchUser(); }', + { steps: STEPS } + ) + ).rejects.toThrow(/"use workflow" directive/); + }); + + it.each([ + ['named import', 'import { x } from "y";\n'], + ['namespace import', 'import * as y from "y";\n'], + ['bare import', 'import "y";\n'], + ['indented import', ' import { x } from "y";\n'], + ['export function', 'export function helper() {}\n'], + ['export const', 'export const helper = 1;\n'], + ['export default', 'export default 1;\n'], + ['export list', 'export { workflow };\n'], + ['export list without a space', 'export{ workflow };\n'], + ['export star', 'export * from "y";\n'], + ])('rejects module syntax: %s', async (_label, prefix) => { + // The code is evaluated as a script in the workflow VM, so module + // syntax is a replay-time syntax error — catching it here turns a run + // that could never execute into a failed call. + await expect( + compileDynamicWorkflow(prefix + SOURCE, { steps: STEPS }) + ).rejects.toThrow(/cannot use `import` or `export`/); + }); + + it.each([ + ['an identifier starting with import', 'let importData = null;\n'], + ['an identifier starting with export', 'let exportName = "x";\n'], + ['a member of exports', 'const exporter = { exports: 1 };\n'], + ])('accepts %s at the start of a line', async (_label, prefix) => { + // Only the keyword followed by whitespace or module-syntax punctuation + // is module syntax. An identifier that merely begins with the word is + // ordinary code, and rejecting it would cost the caller a working + // workflow for nothing. + await expect( + compileDynamicWorkflow(prefix + SOURCE, { steps: STEPS }) + ).resolves.toMatchObject({ metadata: { version: 1 } }); + }); + + it('accepts source that merely mentions import inside a string', async () => { + // The check is anchored to the start of a line for exactly this: a + // false positive costs the caller a working workflow, and prose inside + // string arguments is normal in generated orchestration. + const source = ` +async function workflow() { + "use workflow"; + await steps.fetchUser("nothing to import here"); +} +`; + await expect( + compileDynamicWorkflow(source, { steps: STEPS }) + ).resolves.toMatchObject({ metadata: { version: 1 } }); + }); + + it('rejects source over the size limit', async () => { + const filler = `\n// ${'x'.repeat(DYNAMIC_WORKFLOW_SOURCE_MAX_BYTES)}`; + + await expect( + compileDynamicWorkflow(SOURCE + filler, { steps: STEPS }) + ).rejects.toThrow(/over the \d+-byte limit/); + }); + + it('rejects an empty step map', async () => { + await expect( + compileDynamicWorkflow(SOURCE, { steps: {} }) + ).rejects.toThrow(/at least one registered step/); + }); + + it('rejects a step value with no stepId', async () => { + await expect( + compileDynamicWorkflow(SOURCE, { + steps: { fetchUser: {} as { stepId: string } }, + }) + ).rejects.toThrow(/imported step function or an object with/); + }); + + it.each([ + ['a step alias', { steps: { 'not-an-identifier': STEPS.fetchUser } }], + ['an export name', { steps: STEPS, exportName: 'not-an-identifier' }], + ])('rejects %s that is not a JavaScript identifier', async (_label, options) => { + // These are interpolated into generated code, so anything that is not + // an identifier would produce a syntax error at replay time at best. + await expect( + compileDynamicWorkflow(SOURCE, options as never) + ).rejects.toThrow(WorkflowRuntimeError); + }); + + it('rejects an export name the workflow queue cannot address', async () => { + await expect( + compileDynamicWorkflow( + 'async function $workflow() { "use workflow"; return 1; }', + { steps: STEPS, exportName: '$workflow' } + ) + ).rejects.toThrow(/"\$" cannot appear in workflow queue names/); + }); + + it('still accepts "$" in step aliases', async () => { + await expect( + compileDynamicWorkflow( + 'async function workflow() { "use workflow"; return await steps.$fetch(); }', + { steps: { $fetch: STEPS.fetchUser } } + ) + ).resolves.toMatchObject({ + workflowName: expect.stringMatching(/\/\/workflow$/), + }); + }); + }); +}); + +describe('readDynamicWorkflowMetadata', () => { + it('reads back what compile produced', async () => { + const compiled = await compileDynamicWorkflow(SOURCE, { steps: STEPS }); + + expect( + readDynamicWorkflowMetadata({ dynamicWorkflow: compiled.metadata }) + ).toEqual(compiled.metadata); + }); + + it.each([ + ['undefined context', undefined], + ['no marker', { workflowCoreVersion: '5.0.0' }], + ['a non-object marker', { dynamicWorkflow: 'yes' }], + ['an unknown version', { dynamicWorkflow: { version: 2 } }], + [ + 'a missing exportName', + { dynamicWorkflow: { version: 1, sourceHash: 'a' } }, + ], + [ + 'a missing sourceHash', + { dynamicWorkflow: { version: 1, exportName: 'workflow' } }, + ], + ])('returns undefined for %s', (_label, executionContext) => { + // executionContext is client-supplied and passes through the backend + // unchecked, so the runtime validates the shape rather than trusting it: + // the alternative to "not dynamic" is executing arbitrary stored bytes as + // code. + expect(readDynamicWorkflowMetadata(executionContext)).toBeUndefined(); + }); + + it('defaults absent step bindings to an empty map', () => { + expect( + readDynamicWorkflowMetadata({ + dynamicWorkflow: { version: 1, sourceHash: 'abc', exportName: 'wf' }, + }) + ).toEqual({ + version: 1, + sourceHash: 'abc', + exportName: 'wf', + steps: {}, + }); + }); +}); diff --git a/packages/core/src/runtime/dynamic-workflow.ts b/packages/core/src/runtime/dynamic-workflow.ts new file mode 100644 index 0000000000..d0a511ff6c --- /dev/null +++ b/packages/core/src/runtime/dynamic-workflow.ts @@ -0,0 +1,445 @@ +/** + * Dynamic workflow source: starting a run from a workflow function that is + * not in the deployment's build-time manifest. + * + * The normal path compiles workflow functions at build time — the SWC plugin + * rewrites `"use workflow"` bodies, the builder bundles them, and `start()` + * names one by the `workflowId` the transform stamped on it. Dynamic source + * covers orchestration the application assembles after the deployment exists, + * over a fixed catalog of deployed steps. + * + * This module owns the compile half of that: validating the source, deriving + * a stable workflow id from it, and generating the VM code that registers it. + * The generated code — not the source — is what gets stored with the run and + * replayed, so the module is also the format boundary: a run replays byte-for-byte + * the code it started on, and changing the generator here does not retroactively + * change what an in-flight run executes. + * + * Deliberately not a security sandbox. The workflow VM enforces determinism, + * not isolation from malicious JavaScript, so dynamic source is trusted + * application code with the full privileges of the deployment's functions. + * The `steps` allowlist below is a convenience that keeps ordinary code from + * reaching a step it was not given; it is not a capability boundary against + * code that is actively trying to escape one. Deployments opt in with + * `WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS` before they start or execute it. + */ + +import { WorkflowRuntimeError } from '@workflow/errors'; +import { + type ExpressionStatement, + type FunctionDeclaration, + type Program, + parse, +} from 'acorn'; +import type { StartOptions } from './start.js'; + +/** + * A step exposed to dynamic source. + * + * Either an imported step function, or an explicit `{ stepId }` for a step + * whose function is not importable from the calling context. + * + * The function arm is deliberately typed as "any function" rather than as + * something carrying `stepId`: the build-time transform stamps `.stepId` on + * step functions at *runtime*, and nothing adds it to their declared type. A + * type that demanded it would reject the documented call — `steps: { fetchUser, + * sendEmail }` with real imports — and only accept the escape hatch. The + * runtime check in `resolveStepId` is the real gate, and it fails with a + * message naming the alias when a value turns out to carry no step id. + */ +export type DynamicWorkflowStepReference = + | { readonly stepId: string } + | ((...args: any[]) => unknown); + +export interface DynamicWorkflowOptions { + /** + * Already-registered step functions to expose to the source, keyed by the + * alias it calls them under (`steps.(...)`). + * + * Each value is either an imported step function — the SDK transform stamps + * a `.stepId` on it — or an explicit `{ stepId }` reference for a step whose + * function is not importable from the calling context. + * + * There is no way to register a *new* step from dynamic source: only the + * orchestration is dynamic, and every step it can reach was deployed with + * the app. + */ + steps: Record; + + /** + * Name of the async workflow function in the source. Defaults to + * `"workflow"`. + */ + exportName?: string; +} + +/** `start()` options for the dynamic-source overload. */ +export type DynamicStartOptions = StartOptions & { + experimental_dynamic: DynamicWorkflowOptions; +}; + +/** + * Plaintext metadata recorded on `executionContext.dynamicWorkflow`. + * + * Deliberately small and non-sensitive: it has to fit the execution-context + * budget (2 KB on `world-vercel`) and it is readable without decrypting + * anything, which is what lets observability show that a run is dynamic — + * and which steps it was allowed to call — while the code itself stays + * encrypted behind the run's ref. + */ +export interface DynamicWorkflowMetadata { + version: 1; + /** SHA-256 over the source and its step bindings. */ + sourceHash: string; + /** Name of the workflow function inside the source. */ + exportName: string; + /** Alias → step id map the source was compiled against. */ + steps: Record; +} + +/** + * Maximum size of the *source* `start()` accepts. + * + * This is a product limit rather than a storage one — generated + * orchestration functions are small, and a caller handing us a megabyte of + * "workflow" has almost certainly made a mistake worth surfacing at the call + * site. The storage layer's own cap is far higher; see the Dynamic Workflows + * docs. + */ +export const DYNAMIC_WORKFLOW_SOURCE_MAX_BYTES = 128 * 1024; + +/** + * Largest serialized code payload sent inline on `run_created`. + * + * Above this the code is uploaded separately and referenced, because it stops + * fitting comfortably in the creating write's metadata budget (the + * `world-vercel` backend caps the inline field at 32 KB). Set below that cap + * so a compression ratio worse than expected does not push a run over it. + * + * Almost every real definition is under this: 24 KB of *compressed, + * encrypted* bytes is a lot of JavaScript. + */ +export const DYNAMIC_WORKFLOW_CODE_INLINE_MAX_BYTES = 24 * 1024; + +const SAFE_DYNAMIC_IDENTIFIER = /^[a-zA-Z_$][a-zA-Z0-9_$]*$/; + +/** + * The export name becomes the final segment of the generated workflow id, + * which is also the queue topic name. Queue names do not accept `$`, so this + * is the intersection of JavaScript identifiers and the queue-name alphabet. + */ +const SAFE_DYNAMIC_EXPORT_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + +/** + * Rejects module syntax, which the generated wrapper cannot host: the code is + * evaluated as a script in the workflow VM, so an `import` or `export` in it + * is a syntax error at replay time rather than at `start()` time. Catching it + * here turns a run that can never execute into a failed call. + * + * Anchored to the start of a line, which is where module syntax lives in any + * formatted source. Matching it anywhere would reject a `steps.notify("… + * import …")` whose *string* happens to contain the word — a false positive + * that costs a caller a working workflow, which is much worse than the + * remaining false negative (an `import` indented behind other code on one + * line, which still fails loudly at replay). + * + * The keyword must be followed by whitespace or a module-syntax token (`*`, + * `{`, `(`, a quote), never by another identifier character, so ordinary + * identifiers that merely start with the word — `importData = …`, + * `exports.x = …` — are not mistaken for module syntax. + * + * This is only an early, targeted diagnostic. Acorn subsequently parses the + * complete source in script mode, so module forms this expression does not + * recognize still fail before the run is created. + */ +const UNSUPPORTED_DYNAMIC_MODULE_SYNTAX = + /^[ \t]*(?:import(?:\s+[\w$]|\s*(?:[*{(]|['"]))|export(?:\s+(?:async\s+)?(?:function|const|let|var|class|default)\b|\s*[{*]))/m; + +function assertDynamicWorkflowIdentifier(kind: string, value: string): void { + if (!SAFE_DYNAMIC_IDENTIFIER.test(value)) { + throw new WorkflowRuntimeError( + `Invalid dynamic workflow ${kind} ${JSON.stringify(value)}. Use a valid JavaScript identifier: letters, digits, "_" and "$", not starting with a digit.` + ); + } +} + +/** + * Stable stringify for the hash input. `JSON.stringify` on an object is + * insertion-ordered, so two callers passing the same steps in a different + * order would otherwise hash differently and produce two workflow ids for + * one workflow. + */ +function stableJsonStringify(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map(stableJsonStringify).join(',')}]`; + } + if (value && typeof value === 'object') { + return `{${Object.entries(value as Record) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, val]) => `${JSON.stringify(key)}:${stableJsonStringify(val)}`) + .join(',')}}`; + } + return JSON.stringify(value); +} + +async function sha256Hex(input: string): Promise { + // Web Crypto rather than `node:crypto`: this module is reachable from + // `start()` in every runtime the SDK supports, including edge. + const digest = await crypto.subtle.digest( + 'SHA-256', + new TextEncoder().encode(input) + ); + return Array.from(new Uint8Array(digest)) + .map((byte) => byte.toString(16).padStart(2, '0')) + .join(''); +} + +// Pinned to the oldest supported replay engine (Node 22 / V8 12.4). +// Raise only after every supported Node and QuickJS runtime accepts the grammar. +const DYNAMIC_WORKFLOW_ECMA_VERSION = 2024; + +function parseDynamicWorkflowSource(source: string): Program { + try { + return parse(source, { + ecmaVersion: DYNAMIC_WORKFLOW_ECMA_VERSION, + sourceType: 'script', + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new WorkflowRuntimeError( + `Dynamic workflow source is not valid JavaScript: ${message}. The source is evaluated as-is, so it cannot contain TypeScript syntax or module declarations.` + ); + } +} + +function assertGeneratedWorkflowCodeParses(workflowCode: string): void { + try { + parse(workflowCode, { + ecmaVersion: DYNAMIC_WORKFLOW_ECMA_VERSION, + sourceType: 'script', + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new WorkflowRuntimeError( + `Generated dynamic workflow code is not valid JavaScript: ${message}.` + ); + } +} + +function validateDynamicWorkflowSource( + source: string, + exportName: string +): void { + const byteLength = new TextEncoder().encode(source).byteLength; + if (byteLength > DYNAMIC_WORKFLOW_SOURCE_MAX_BYTES) { + throw new WorkflowRuntimeError( + `Dynamic workflow source is ${byteLength} bytes, over the ${DYNAMIC_WORKFLOW_SOURCE_MAX_BYTES}-byte limit.` + ); + } + + if (UNSUPPORTED_DYNAMIC_MODULE_SYNTAX.test(source)) { + throw new WorkflowRuntimeError( + 'Dynamic workflow source cannot use `import` or `export`. Reach registered steps through the injected `steps` object instead.' + ); + } + + const program = parseDynamicWorkflowSource(source); + const declarations = program.body.filter( + (node): node is FunctionDeclaration => + node.type === 'FunctionDeclaration' && node.id.name === exportName + ); + const declaration = declarations.length === 1 ? declarations[0] : undefined; + if (!declaration || !declaration.async || declaration.generator) { + throw new WorkflowRuntimeError( + `Dynamic workflow source must declare \`async function ${exportName}(...)\` at top level.` + ); + } + + const firstStatement = declaration.body.body[0] as + | ExpressionStatement + | undefined; + if ( + firstStatement?.type !== 'ExpressionStatement' || + firstStatement.directive !== 'use workflow' + ) { + throw new WorkflowRuntimeError( + `Dynamic workflow function ${JSON.stringify(exportName)} must open with a "use workflow" directive.` + ); + } + + // No transform runs over dynamic source, so a `"use step"` directive in it + // would not split a step out: the function would simply run inside the + // workflow VM, as ordinary (and non-deterministic) workflow code. Steps + // come from `experimental_dynamic.steps` only. + if (/(?:"use step"|'use step')/.test(source)) { + throw new WorkflowRuntimeError( + 'Dynamic workflow source cannot declare "use step" functions. Register the step with the deployment and expose it through `experimental_dynamic.steps` instead.' + ); + } +} + +function resolveStepId(alias: string, value: unknown): string { + const stepId = + (value && typeof value === 'object') || typeof value === 'function' + ? (value as { stepId?: unknown }).stepId + : undefined; + + if (typeof stepId !== 'string' || stepId.length === 0) { + throw new WorkflowRuntimeError( + `Dynamic workflow step ${JSON.stringify(alias)} must be an imported step function or an object with a non-empty \`stepId\`.` + ); + } + + return stepId; +} + +export interface CompiledDynamicWorkflow { + /** Generated workflow id, also the run's `workflowName`. */ + workflowName: string; + /** Workflow VM code to store with the run and replay from. */ + workflowCode: string; + /** Plaintext metadata for `executionContext.dynamicWorkflow`. */ + metadata: DynamicWorkflowMetadata; +} + +/** + * The workflow id a dynamic definition registers under. + * + * Half the digest. Long enough that a collision is not a practical concern and + * short enough to keep queue topic names and observability rows readable. + * Delivery recomputes it from a run's `dynamicWorkflow` marker and refuses a + * run whose `workflowName` does not match, so a marker cannot turn a static + * workflow's run into one that executes stored code. + */ +export function dynamicWorkflowName( + metadata: Pick +): string { + return `workflow//dynamic/${metadata.sourceHash.slice(0, 32)}//${metadata.exportName}`; +} + +/** + * Validate dynamic source and generate the workflow VM code for it. + * + * The workflow id is derived from the source and its step bindings rather + * than accepted from the caller. Two consequences, both intentional: the same + * definition always lands on the same queue topic and groups together in + * observability, and a caller cannot claim a static workflow's id — or + * another definition's — for arbitrary code. + */ +export async function compileDynamicWorkflow( + source: string, + options: DynamicWorkflowOptions +): Promise { + const exportName = options.exportName ?? 'workflow'; + assertDynamicWorkflowIdentifier('exportName', exportName); + if (!SAFE_DYNAMIC_EXPORT_NAME.test(exportName)) { + throw new WorkflowRuntimeError( + `Invalid dynamic workflow exportName ${JSON.stringify(exportName)}. Use letters, digits, and "_", not starting with a digit; "$" cannot appear in workflow queue names.` + ); + } + validateDynamicWorkflowSource(source, exportName); + + if (!options.steps || Object.keys(options.steps).length === 0) { + throw new WorkflowRuntimeError( + 'Dynamic workflow options must expose at least one registered step through `experimental_dynamic.steps`.' + ); + } + + const stepEntries = Object.entries(options.steps) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([alias, value]) => { + assertDynamicWorkflowIdentifier('step alias', alias); + return [alias, resolveStepId(alias, value)] as const; + }); + const steps = Object.fromEntries(stepEntries); + + const sourceHash = await sha256Hex( + `${source}\n${stableJsonStringify(steps)}` + ); + + const workflowName = dynamicWorkflowName({ sourceHash, exportName }); + + const stepBindings = stepEntries + .map( + ([alias, stepId]) => + ` [${JSON.stringify(alias)}]: __dynamicUseStep(${JSON.stringify(stepId)})` + ) + .join(',\n'); + + // The caller's script runs in its own nested lexical scope. Its declarations + // therefore keep script-like relationships with one another but cannot + // shadow bindings used to initialize or register the wrapper. The outer + // factory parameters intentionally expose only the supported runtime + // bindings, which the selected workflow closes over. + const workflowCode = `const __dynamicGlobalThis = globalThis; +__dynamicGlobalThis.__private_workflows ??= new Map(); +const __dynamicUseStep = __dynamicGlobalThis[Symbol.for("WORKFLOW_USE_STEP")]; +if (typeof __dynamicUseStep !== "function") { + throw new Error("Dynamic workflows require a workflow VM that provides WORKFLOW_USE_STEP."); +} +const __dynamicSteps = Object.freeze({ +${stepBindings} +}); +const __dynamicSleep = __dynamicGlobalThis[Symbol.for("WORKFLOW_SLEEP")]; +const __dynamicCreateHook = __dynamicGlobalThis[Symbol.for("WORKFLOW_CREATE_HOOK")]; +const __dynamicWorkflow = ((steps, sleep, createHook) => (() => { +${source} +return ${exportName}; +})())(__dynamicSteps, __dynamicSleep, __dynamicCreateHook); +Object.defineProperty(__dynamicWorkflow, "workflowId", { + value: ${JSON.stringify(workflowName)}, + writable: false, + enumerable: false, + configurable: false +}); +__dynamicGlobalThis.__private_workflows.set(${JSON.stringify(workflowName)}, __dynamicWorkflow); +`; + + // The caller's source can be valid in isolation but collide with bindings + // supplied by the generated wrapper. Parse the complete script without + // evaluating it so such a run fails before upload, creation, or queueing. + assertGeneratedWorkflowCodeParses(workflowCode); + + return { + workflowName, + workflowCode, + metadata: { version: 1, sourceHash, exportName, steps }, + }; +} + +/** + * Read the dynamic-workflow marker off a run's `executionContext`. + * + * The runtime uses this to decide whether a run replays from the deployment's + * bundle or from its own stored code, so it validates the shape rather than + * trusting it: `executionContext` is client-supplied and passes through the + * backend unchecked. + * + * Returns undefined for every static run, which is the overwhelming majority + * — this is on the hot path of each delivery. + */ +export function readDynamicWorkflowMetadata( + executionContext: unknown +): DynamicWorkflowMetadata | undefined { + if (!executionContext || typeof executionContext !== 'object') { + return undefined; + } + const marker = (executionContext as { dynamicWorkflow?: unknown }) + .dynamicWorkflow; + if (!marker || typeof marker !== 'object') return undefined; + + const candidate = marker as Partial; + if (candidate.version !== 1) return undefined; + if (typeof candidate.exportName !== 'string') return undefined; + if (typeof candidate.sourceHash !== 'string') return undefined; + + return { + version: 1, + sourceHash: candidate.sourceHash, + exportName: candidate.exportName, + steps: + candidate.steps && typeof candidate.steps === 'object' + ? candidate.steps + : {}, + }; +} diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index c3b2ba3ae4..a47a33ac8a 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -1,7 +1,7 @@ import { PreconditionFailedError, WorkflowWorldError } from '@workflow/errors'; import type { Event, World } from '@workflow/world'; import { slotToEventId } from '@workflow/world'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { bytesToBase64, deriveRunKeyPair, seal } from '../sealed-box.js'; import { decrypt, @@ -10,8 +10,10 @@ import { peekFormatPrefix, SerializationFormat, } from '../serialization.js'; +import { DYNAMIC_WORKFLOWS_ENV } from './constants.js'; import { appendUniqueEvents, + DYNAMIC_WORKFLOW_VERSION, findEventSlotGap, getWorkflowQueueName, handleHealthCheckMessage, @@ -263,6 +265,26 @@ describe('healthCheck response parsing', () => { expect(result.workflowCoreVersion).toBe('5.0.0-beta.7'); }); + it('surfaces dynamicWorkflowVersion when present in the response', async () => { + const world = makeWorldWithResponse( + JSON.stringify({ healthy: true, dynamicWorkflowVersion: 1 }) + ); + + const result = await healthCheck(world, { timeout: 1000 }); + + expect(result.dynamicWorkflowVersion).toBe(1); + }); + + it('omits malformed dynamicWorkflowVersion values', async () => { + const world = makeWorldWithResponse( + JSON.stringify({ healthy: true, dynamicWorkflowVersion: '1' }) + ); + + const result = await healthCheck(world, { timeout: 1000 }); + + expect(result.dynamicWorkflowVersion).toBeUndefined(); + }); + it('omits workflowCoreVersion when the response does not include the field', async () => { // Independent of specVersion — the field is omitted by any responder // running an older `@workflow/core` that predates the addition of @@ -1099,6 +1121,44 @@ describe('health check run public key', () => { }); }); +describe('health check dynamic workflow version', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + async function respond() { + const { getWorldLazy } = await import('./get-world-lazy.js'); + const write = vi.fn().mockResolvedValue(undefined); + vi.mocked(getWorldLazy).mockReturnValue({ + streams: { write, close: vi.fn().mockResolvedValue(undefined) }, + } as any); + await handleHealthCheckMessage( + { __healthCheck: true, correlationId: 'corr_dynamic' }, + 'workflow' + ); + return JSON.parse(write.mock.calls[0][2] as string); + } + + it('omits dynamicWorkflowVersion when the deployment has not opted in', async () => { + vi.stubEnv(DYNAMIC_WORKFLOWS_ENV, undefined); + const response = await respond(); + expect(response.healthy).toBe(true); + expect(response).not.toHaveProperty('dynamicWorkflowVersion'); + }); + + it('omits dynamicWorkflowVersion for a value other than 1 or true', async () => { + vi.stubEnv(DYNAMIC_WORKFLOWS_ENV, 'false'); + expect(await respond()).not.toHaveProperty('dynamicWorkflowVersion'); + }); + + it('advertises dynamicWorkflowVersion when the deployment has opted in', async () => { + vi.stubEnv(DYNAMIC_WORKFLOWS_ENV, 'true'); + expect((await respond()).dynamicWorkflowVersion).toBe( + DYNAMIC_WORKFLOW_VERSION + ); + }); +}); + describe('queueMessages', () => { const entries = (n: number) => Array.from({ length: n }, (_, i) => ({ diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 8c2de7e3e7..3a81cba10b 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -35,6 +35,7 @@ import { import * as Attribute from '../telemetry/semantic-conventions.js'; import { getSpanKind, trace } from '../telemetry.js'; import { version as workflowCoreVersion } from '../version.js'; +import { isDynamicWorkflowsEnabled } from './constants.js'; import { getWorldLazy } from './get-world-lazy.js'; /** Default timeout for health checks in milliseconds */ @@ -79,6 +80,9 @@ function getHealthCheckStreamName(correlationId: string): string { return `__health_check__${correlationId}`; } +/** Version of the dynamic-workflow runtime contract advertised by deployments. */ +export const DYNAMIC_WORKFLOW_VERSION = 1; + /** * Result of a health check operation. */ @@ -127,6 +131,11 @@ export interface HealthCheckResult { * field is missing or malformed. */ format?: 'json' | 'text'; + /** + * Version of dynamic-workflow execution supported by the target runtime. + * Present only when the target has opted in to dynamic workflows. + */ + dynamicWorkflowVersion?: number; } /** @@ -205,6 +214,11 @@ export async function handleHealthCheckMessage( // the *consumer's* hook-resume protocol version, exactly what a // cross-deployment caller needs to gate its parallel resume path on. hookResumeInputVersion: HOOK_RESUME_INPUT_VERSION, + // Advertised only when this deployment has opted in to executing + // dynamic workflow code; an absent field reads as "unsupported". + ...(isDynamicWorkflowsEnabled() + ? { dynamicWorkflowVersion: DYNAMIC_WORKFLOW_VERSION } + : {}), ...(encryptionPublicKey ? { encryptionPublicKey } : {}), timestamp: Date.now(), }); @@ -365,6 +379,9 @@ function parseHealthCheckResponse( if (typeof r.hookResumeInputVersion === 'number') { parsed.hookResumeInputVersion = r.hookResumeInputVersion; } + if (typeof r.dynamicWorkflowVersion === 'number') { + parsed.dynamicWorkflowVersion = r.dynamicWorkflowVersion; + } return parsed; } diff --git a/packages/core/src/runtime/runs.test.ts b/packages/core/src/runtime/runs.test.ts index 257b28cf0b..d90d06c119 100644 --- a/packages/core/src/runtime/runs.test.ts +++ b/packages/core/src/runtime/runs.test.ts @@ -324,6 +324,30 @@ describe('recreateRunFromExisting', () => { expect(vi.mocked(start).mock.calls[0][2]?.specVersion).toBe(7); }); + + it('refuses a dynamic run rather than creating one with no code behind it', async () => { + // Starting by name alone would create a run carrying the dynamic + // workflow id and none of the stored code, which no delivery could run. + const world = createMockWorld({ + run: { + runId: 'wrun_dynamic', + workflowName: 'workflow//dynamic/abc123//workflow', + executionContext: { + dynamicWorkflow: { + version: 1, + sourceHash: 'abc123', + exportName: 'workflow', + steps: {}, + }, + }, + }, + }); + + await expect( + recreateRunFromExisting(world, 'wrun_dynamic') + ).rejects.toThrow(/dynamic workflow run; re-running it is not supported/); + expect(start).not.toHaveBeenCalled(); + }); }); describe('Run.exists', () => { diff --git a/packages/core/src/runtime/runs.ts b/packages/core/src/runtime/runs.ts index dbbaaa96f8..5cb06fcd8f 100644 --- a/packages/core/src/runtime/runs.ts +++ b/packages/core/src/runtime/runs.ts @@ -1,4 +1,8 @@ -import { EntityConflictError, StreamError } from '@workflow/errors'; +import { + EntityConflictError, + StreamError, + WorkflowRuntimeError, +} from '@workflow/errors'; import { BULK_CANCEL_MAX_RUN_IDS, type BulkCancelWorkflowRunResult, @@ -10,6 +14,7 @@ import { } from '@workflow/world'; import { deriveRunPayloadKeys } from '../serialization/encryption.js'; import { hydrateWorkflowArguments } from '../serialization.js'; +import { readDynamicWorkflowMetadata } from './dynamic-workflow.js'; import { getWorkflowQueueName } from './helpers.js'; import { specVersionForRunWrite } from './run-spec-version.js'; import { start } from './start.js'; @@ -88,6 +93,18 @@ export async function recreateRunFromExisting( ): Promise { try { const run = await world.runs.get(runId, { resolveData: 'all' }); + // A dynamic run's workflow function is not in the deployment's bundle; + // it lives on the run, encrypted under that run's key. Starting a new run + // by workflow name alone would create one with the dynamic id and no code + // behind it, which no delivery could ever execute. Carrying the code + // over means decrypting it and re-encrypting under the new run's key, + // which this path does not do yet, so refuse rather than create a run + // that fails on its first delivery. + if (readDynamicWorkflowMetadata(run.executionContext)) { + throw new WorkflowRuntimeError( + `Run ${runId} is a dynamic workflow run; re-running it is not supported. Start it again from its source with start(source, args, { experimental_dynamic }).` + ); + } const rawKey = await world.getEncryptionKeyForRun?.(run); const encryptionKey = rawKey ? await deriveRunPayloadKeys(rawKey) diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 1fd3f0c239..4b7a42a728 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -32,6 +32,8 @@ import { runPayloadKeys, SerializationFormat, } from '../serialization.js'; +import { serializeTraceCarrier } from '../telemetry.js'; +import { DYNAMIC_WORKFLOWS_ENV } from './constants.js'; import type { Run } from './run.js'; import type { WorkflowFunction } from './start.js'; import { _resetLatestNoOpWarnForTests, start } from './start.js'; @@ -50,7 +52,527 @@ vi.mock('../telemetry.js', () => ({ })); describe('start', () => { + describe('dynamic workflow preflight', () => { + const source = 'async function workflow() { "use workflow"; return 1; }'; + let eventsCreate: ReturnType; + let queue: ReturnType; + let upload: ReturnType; + + beforeEach(() => { + eventsCreate = vi.fn(); + queue = vi.fn(); + upload = vi.fn(); + vi.stubEnv(DYNAMIC_WORKFLOWS_ENV, '1'); + }); + + afterEach(() => { + setWorld(undefined); + vi.clearAllMocks(); + vi.unstubAllEnvs(); + }); + + describe('deployment opt-in', () => { + function optInWorld() { + return { + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + getBackendCapabilities: vi + .fn() + .mockResolvedValue({ dynamicWorkflowStorageVersion: 1 }), + getEncryptionKeyForRun: vi.fn(), + uploadDynamicWorkflowCode: upload, + events: { create: eventsCreate }, + queue, + } as any; + } + + it.each([ + ['unset', undefined], + ['empty', ''], + ['0', '0'], + ['false', 'false'], + ['yes', 'yes'], + ])('refuses a dynamic start with the opt-in %s, before any world call', async (_label, value) => { + if (value === undefined) { + vi.stubEnv(DYNAMIC_WORKFLOWS_ENV, undefined); + } else { + vi.stubEnv(DYNAMIC_WORKFLOWS_ENV, value); + } + const world = optInWorld(); + setWorld(world); + + const error = await start(source, { + experimental_dynamic: { + steps: { noop: { stepId: 'step//./test//noop' } }, + }, + }).catch((err: unknown) => err); + + expect(WorkflowRuntimeError.is(error)).toBe(true); + expect((error as Error).message).toMatch( + /Dynamic workflows are disabled on this deployment.*WORKFLOW_EXPERIMENTAL_DYNAMIC_WORKFLOWS=1/ + ); + expect(world.getDeploymentId).not.toHaveBeenCalled(); + expect(world.getBackendCapabilities).not.toHaveBeenCalled(); + expect(world.getEncryptionKeyForRun).not.toHaveBeenCalled(); + expect(upload).not.toHaveBeenCalled(); + expect(eventsCreate).not.toHaveBeenCalled(); + expect(queue).not.toHaveBeenCalled(); + }); + + it.each([ + '1', + 'true', + 'TRUE', + ])('accepts the opt-in value %s', async (value) => { + vi.stubEnv(DYNAMIC_WORKFLOWS_ENV, value); + eventsCreate.mockImplementation(async (runId, event) => ({ + run: { + runId, + status: 'pending', + dynamicWorkflowCode: event.eventData.dynamicWorkflowCode, + }, + })); + queue.mockResolvedValue(undefined); + setWorld(optInWorld()); + + await expect( + start(source, { + experimental_dynamic: { + steps: { noop: { stepId: 'step//./test//noop' } }, + }, + }) + ).resolves.toBeDefined(); + expect(eventsCreate).toHaveBeenCalledOnce(); + expect(queue).toHaveBeenCalledOnce(); + }); + }); + + it.each([ + [ + 'explicit resource management syntax', + `async function workflow() { + "use workflow"; + using resource = null; + return resource; +}`, + ], + [ + 'regular expression modifiers', + `async function workflow() { + "use workflow"; + return /(?i:a)/.test("A"); +}`, + ], + ])('rejects unsupported %s before start side effects', async (_label, unsupportedSource) => { + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + getBackendCapabilities: vi + .fn() + .mockResolvedValue({ dynamicWorkflowStorageVersion: 1 }), + uploadDynamicWorkflowCode: upload, + events: { create: eventsCreate }, + queue, + } as any); + + await expect( + start(unsupportedSource, { + experimental_dynamic: { + steps: { noop: { stepId: 'step//./test//noop' } }, + }, + }) + ).rejects.toThrow(/not valid JavaScript/); + expect(upload).not.toHaveBeenCalled(); + expect(eventsCreate).not.toHaveBeenCalled(); + expect(queue).not.toHaveBeenCalled(); + }); + + it('rejects a $-prefixed export name before start side effects', async () => { + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + getBackendCapabilities: vi + .fn() + .mockResolvedValue({ dynamicWorkflowStorageVersion: 1 }), + uploadDynamicWorkflowCode: upload, + events: { create: eventsCreate }, + queue, + } as any); + + await expect( + start('async function $workflow() { "use workflow"; return 1; }', { + experimental_dynamic: { + exportName: '$workflow', + steps: { noop: { stepId: 'step//./test//noop' } }, + }, + }) + ).rejects.toThrow(/cannot appear in workflow queue names/); + expect(upload).not.toHaveBeenCalled(); + expect(eventsCreate).not.toHaveBeenCalled(); + expect(queue).not.toHaveBeenCalled(); + }); + + it('validates the queue name before start side effects', async () => { + const getDeploymentId = vi.fn().mockResolvedValue('deploy_123'); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId, + events: { create: eventsCreate }, + queue, + } as any); + const invalidQueueWorkflow = Object.assign(async () => undefined, { + workflowId: 'workflow//./test//$invalid', + }); + + await expect(start(invalidQueueWorkflow as never, [])).rejects.toThrow( + /Invalid workflow name/ + ); + expect(getDeploymentId).not.toHaveBeenCalled(); + expect(eventsCreate).not.toHaveBeenCalled(); + expect(queue).not.toHaveBeenCalled(); + }); + + describe('same-deployment only', () => { + const dynamicSteps = { + steps: { noop: { stepId: 'step//./test//noop' } }, + }; + + function dynamicWorld(overrides: Record) { + return { + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + getBackendCapabilities: vi + .fn() + .mockResolvedValue({ dynamicWorkflowStorageVersion: 1 }), + getEncryptionKeyForRun: vi.fn(), + uploadDynamicWorkflowCode: upload, + events: { create: eventsCreate }, + queue, + streams: { get: vi.fn() }, + ...overrides, + } as any; + } + + function expectNoSideEffects(world: any) { + // No capability probe (a queue call), no key lookup, no upload, and + // no run creation. + expect(queue).not.toHaveBeenCalled(); + expect(world.streams.get).not.toHaveBeenCalled(); + expect(world.getBackendCapabilities).not.toHaveBeenCalled(); + expect(world.getEncryptionKeyForRun).not.toHaveBeenCalled(); + expect(upload).not.toHaveBeenCalled(); + expect(eventsCreate).not.toHaveBeenCalled(); + } + + it('rejects an explicit other deployment before probing it', async () => { + const world = dynamicWorld({}); + setWorld(world); + + const error = await start(source, { + deploymentId: 'dpl_other', + experimental_dynamic: dynamicSteps, + }).catch((err: unknown) => err); + + expect(WorkflowRuntimeError.is(error)).toBe(true); + expect((error as Error).message).toMatch( + /only start on the current deployment.*"dpl_other" from "deploy_123"/ + ); + expectNoSideEffects(world); + }); + + it("rejects 'latest' when it resolves to another deployment", async () => { + const world = dynamicWorld({ + resolveLatestDeploymentId: vi.fn().mockResolvedValue('dpl_newer'), + }); + setWorld(world); + + await expect( + start(source, { + deploymentId: 'latest', + experimental_dynamic: dynamicSteps, + }) + ).rejects.toThrow(/only start on the current deployment.*"dpl_newer"/); + expectNoSideEffects(world); + }); + + it('rejects a concrete target when the current deployment is unknown', async () => { + const world = dynamicWorld({ + getDeploymentId: vi + .fn() + .mockRejectedValue(new Error('no current deployment')), + }); + setWorld(world); + + await expect( + start(source, { + deploymentId: 'deploy_123', + experimental_dynamic: dynamicSteps, + }) + ).rejects.toThrow( + /only start on the current deployment.*an unknown current deployment/ + ); + expectNoSideEffects(world); + }); + + it('accepts an explicit deploymentId naming the current deployment', async () => { + eventsCreate.mockImplementation(async (runId, event) => ({ + run: { + runId, + status: 'pending', + dynamicWorkflowCode: event.eventData.dynamicWorkflowCode, + }, + })); + queue.mockResolvedValue(undefined); + const world = dynamicWorld({}); + setWorld(world); + + await expect( + start(source, { + deploymentId: 'deploy_123', + experimental_dynamic: dynamicSteps, + }) + ).resolves.toBeDefined(); + // Same deployment: no capability probe, only the run's own queue + // message. + expect(world.streams.get).not.toHaveBeenCalled(); + expect(queue).toHaveBeenCalledOnce(); + expect(queue.mock.calls[0][0]).toContain('workflow//dynamic/'); + }); + }); + + it.each([ + [ + 'nested declaration', + 'function outer() { async function workflow() { "use workflow"; } }', + ], + [ + 'comment spoof', + '// async function workflow() { "use workflow"; }\nconst value = 1;', + ], + [ + 'string spoof', + 'const value = `async function workflow() { "use workflow"; }`;', + ], + ])('rejects a %s before start side effects', async (_label, invalidSource) => { + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + getBackendCapabilities: vi + .fn() + .mockResolvedValue({ dynamicWorkflowStorageVersion: 1 }), + uploadDynamicWorkflowCode: upload, + events: { create: eventsCreate }, + queue, + } as any); + + await expect( + start(invalidSource, { + experimental_dynamic: { + steps: { noop: { stepId: 'step//./test//noop' } }, + }, + }) + ).rejects.toThrow(/at top level/); + expect(upload).not.toHaveBeenCalled(); + expect(eventsCreate).not.toHaveBeenCalled(); + expect(queue).not.toHaveBeenCalled(); + }); + + it.each([ + ['Object', 'const Object = null;'], + ['Map', 'let Map = null;'], + ['Symbol', 'function Symbol() {}'], + ['globalThis', 'var globalThis = null;'], + ['__dynamicUseStep', 'const __dynamicUseStep = null;'], + ['__dynamicWorkflow', 'let __dynamicWorkflow = null;'], + ['__dynamicGlobalThis', 'function __dynamicGlobalThis() {}'], + ['steps', 'var steps = null;'], + ['sleep', 'const sleep = null;'], + ['createHook', 'let createHook = null;'], + ['Error', 'class Error {}'], + ])('starts safely with caller %s declarations', async (_binding, declaration) => { + eventsCreate.mockImplementation(async (runId, event) => ({ + run: { + runId, + status: 'pending', + dynamicWorkflowCode: event.eventData.dynamicWorkflowCode, + }, + })); + queue.mockResolvedValue(undefined); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + getBackendCapabilities: vi + .fn() + .mockResolvedValue({ dynamicWorkflowStorageVersion: 1 }), + uploadDynamicWorkflowCode: upload, + events: { create: eventsCreate }, + queue, + } as any); + const isolatedSource = ` +${declaration} +async function workflow() { + "use workflow"; + return 1; +} +`; + + await expect( + start(isolatedSource, { + experimental_dynamic: { + steps: { noop: { stepId: 'step//./test//noop' } }, + }, + }) + ).resolves.toBeDefined(); + expect(upload).not.toHaveBeenCalled(); + expect(eventsCreate).toHaveBeenCalledOnce(); + expect(queue).toHaveBeenCalledOnce(); + }); + + it('allows a genuine top-level declaration to reach start side effects', async () => { + eventsCreate.mockImplementation(async (runId, event) => ({ + run: { + runId, + status: 'pending', + dynamicWorkflowCode: event.eventData.dynamicWorkflowCode, + }, + })); + queue.mockResolvedValue(undefined); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + getBackendCapabilities: vi + .fn() + .mockResolvedValue({ dynamicWorkflowStorageVersion: 1 }), + events: { create: eventsCreate }, + queue, + } as any); + + await expect( + start(source, { + experimental_dynamic: { + steps: { noop: { stepId: 'step//./test//noop' } }, + }, + }) + ).resolves.toBeDefined(); + expect(eventsCreate).toHaveBeenCalledOnce(); + expect(queue).toHaveBeenCalledOnce(); + }); + + it('rejects absent backend attestation before start side effects', async () => { + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + getEncryptionKeyForRun: vi.fn(), + uploadDynamicWorkflowCode: upload, + events: { create: eventsCreate }, + queue, + } as any); + + await expect( + start(source, { + experimental_dynamic: { + steps: { noop: { stepId: 'step//./test//noop' } }, + }, + }) + ).rejects.toThrow(/backend storage capability version 1/); + expect(upload).not.toHaveBeenCalled(); + expect(eventsCreate).not.toHaveBeenCalled(); + expect(queue).not.toHaveBeenCalled(); + }); + + it('rejects an empty backend capability set before start side effects', async () => { + // world-vercel maps a backend without the capabilities route (404) to + // an empty set, which must fail closed like an absent attestation. + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + getBackendCapabilities: vi.fn().mockResolvedValue({}), + getEncryptionKeyForRun: vi.fn(), + uploadDynamicWorkflowCode: upload, + events: { create: eventsCreate }, + queue, + } as any); + + await expect( + start(source, { + experimental_dynamic: { + steps: { noop: { stepId: 'step//./test//noop' } }, + }, + }) + ).rejects.toThrow(/backend storage capability version 1/); + expect(upload).not.toHaveBeenCalled(); + expect(eventsCreate).not.toHaveBeenCalled(); + expect(queue).not.toHaveBeenCalled(); + }); + + it('rejects execution-context validation before upload, create, or queue', async () => { + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + getBackendCapabilities: vi + .fn() + .mockResolvedValue({ dynamicWorkflowStorageVersion: 1 }), + validateRunExecutionContext: vi.fn(() => { + throw new Error('execution context too large'); + }), + uploadDynamicWorkflowCode: upload, + events: { create: eventsCreate }, + queue, + } as any); + + await expect( + start(source, { + experimental_dynamic: { + steps: { noop: { stepId: 'step//./test//noop' } }, + }, + }) + ).rejects.toThrow('execution context too large'); + expect(upload).not.toHaveBeenCalled(); + expect(eventsCreate).not.toHaveBeenCalled(); + expect(queue).not.toHaveBeenCalled(); + }); + + it('does not apply execution-context validation to static starts', async () => { + const validateRunExecutionContext = vi.fn(() => { + throw new Error('execution context too large'); + }); + eventsCreate.mockImplementation(async (runId) => ({ + run: { runId, status: 'pending' }, + })); + queue.mockResolvedValue(undefined); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + validateRunExecutionContext, + events: { create: eventsCreate }, + queue, + } as any); + const staticWorkflow = Object.assign(async () => undefined, { + workflowId: 'workflow//./test//staticWorkflow', + }); + + // A large trace carrier (baggage / tracestate) is the static-start + // shape that could approach a World's execution-context limit. + const traceCarrier = { tracestate: 'x'.repeat(4096) }; + vi.mocked(serializeTraceCarrier).mockResolvedValueOnce(traceCarrier); + + await expect(start(staticWorkflow as never, [])).resolves.toBeDefined(); + expect(validateRunExecutionContext).not.toHaveBeenCalled(); + expect(eventsCreate).toHaveBeenCalledOnce(); + expect( + eventsCreate.mock.calls[0][1].eventData.executionContext.traceCarrier + ).toEqual(traceCarrier); + expect(queue).toHaveBeenCalledOnce(); + }); + }); + describe('error handling', () => { + it('requires experimental_dynamic for source at runtime', async () => { + await expect( + start('async function workflow() { "use workflow"; }' as never, []) + ).rejects.toThrow(/no `experimental_dynamic` options/); + }); + it('should throw WorkflowRuntimeError when workflow is undefined', async () => { await expect( // @ts-expect-error - intentionally passing undefined diff --git a/packages/core/src/runtime/start.ts b/packages/core/src/runtime/start.ts index e712fffbd8..84d23e4233 100644 --- a/packages/core/src/runtime/start.ts +++ b/packages/core/src/runtime/start.ts @@ -32,6 +32,7 @@ import { deriveRunKeyPair, } from '../sealed-box.js'; import { + dehydrateDynamicWorkflowCode, dehydrateWorkflowArguments, type PayloadKey, SerializationFormat, @@ -41,8 +42,19 @@ import { contextStorage } from '../step/context-storage.js'; import * as Attribute from '../telemetry/semantic-conventions.js'; import { serializeTraceCarrier, trace } from '../telemetry.js'; import { version as workflowCoreVersion } from '../version.js'; +import { + DYNAMIC_WORKFLOWS_ENV, + isDynamicWorkflowsEnabled, +} from './constants.js'; +import { + compileDynamicWorkflow, + DYNAMIC_WORKFLOW_CODE_INLINE_MAX_BYTES, + type DynamicStartOptions, + type DynamicWorkflowMetadata, +} from './dynamic-workflow.js'; import { getWorldLazy } from './get-world-lazy.js'; import { + DYNAMIC_WORKFLOW_VERSION, getWorkflowQueueName, type HealthCheckResult, healthCheck, @@ -461,6 +473,24 @@ export type StartOptions = | StartOptionsWithDeploymentId | StartOptionsWithoutDeploymentId; +export type { + DynamicStartOptions, + DynamicWorkflowOptions, + DynamicWorkflowStepReference, +} from './dynamic-workflow.js'; + +/** + * Dynamic starts are same-deployment only, so the process calling `start()` is + * also the deployment that executes the run, and it must have opted in. + */ +function assertDynamicWorkflowsEnabled(): void { + if (!isDynamicWorkflowsEnabled()) { + throw new WorkflowRuntimeError( + `Dynamic workflows are disabled on this deployment, so no run was created. Set ${DYNAMIC_WORKFLOWS_ENV}=1 on the deployment to enable them.` + ); + } +} + /** * Represents an imported workflow function. */ @@ -507,15 +537,63 @@ export function start( options?: StartOptionsWithoutDeploymentId ): Promise>; +// Dynamic source overloads. The return type is `unknown`: the workflow's +// shape is only known to whatever produced the source, so there is nothing +// for TypeScript to infer from. +export function start( + source: string, + args: unknown[], + options: DynamicStartOptions +): Promise>; + +export function start( + source: string, + options: DynamicStartOptions +): Promise>; + export async function start( - workflow: WorkflowFunction | WorkflowMetadata, - argsOrOptions?: TArgs | StartOptions, - options?: StartOptions + workflow: WorkflowFunction | WorkflowMetadata | string, + argsOrOptions?: TArgs | StartOptions | DynamicStartOptions, + options?: StartOptions | DynamicStartOptions ) { 'use step'; - return await waitedUntil(() => { - // @ts-expect-error this field is added by our client transform - const workflowName = workflow?.workflowId; + return await waitedUntil(async () => { + let args: Serializable[] = []; + let opts: StartOptions | DynamicStartOptions = options ?? {}; + if (Array.isArray(argsOrOptions)) { + args = argsOrOptions as Serializable[]; + } else if (typeof argsOrOptions === 'object' && argsOrOptions !== null) { + opts = argsOrOptions; + } + + // Dynamic source: compile it up front so the derived workflow id is + // available for the span name, the queue topic, and the ref key — all of + // which are decided before anything is written. + let dynamicWorkflow: + | { code: string; metadata: DynamicWorkflowMetadata } + | undefined; + let workflowName: string | undefined; + if (typeof workflow === 'string') { + const dynamicOptions = (opts as Partial) + .experimental_dynamic; + if (!dynamicOptions) { + throw new WorkflowRuntimeError( + "'start' was given workflow source but no `experimental_dynamic` options. Pass `{ experimental_dynamic: { steps } }` to declare which registered steps the source may call." + ); + } + const compiled = await compileDynamicWorkflow(workflow, dynamicOptions); + // Checked after validation (which only parses the source) and before + // any world call, trace span, upload, or write. + assertDynamicWorkflowsEnabled(); + workflowName = compiled.workflowName; + dynamicWorkflow = { + code: compiled.workflowCode, + metadata: compiled.metadata, + }; + } else { + // @ts-expect-error this field is added by our client transform + workflowName = workflow?.workflowId; + } if (!workflowName) { throw new WorkflowRuntimeError( @@ -523,6 +601,10 @@ export async function start( { slug: 'start-invalid-workflow-function' } ); } + // Validate the queue destination before any serialization, upload, or + // run creation. The queue write runs beside run creation, so validating it + // there could leave a created but unscheduled run behind. + const queueName = getWorkflowQueueName(workflowName, opts.namespace); const spanName = `workflow.start ${workflowDisplayName(workflowName)}`; return trace(spanName, async (span) => { @@ -531,14 +613,6 @@ export async function start( ...Attribute.WorkflowOperation('start'), }); - let args: Serializable[] = []; - let opts: StartOptions = options ?? {}; - if (Array.isArray(argsOrOptions)) { - args = argsOrOptions as Serializable[]; - } else if (typeof argsOrOptions === 'object') { - opts = argsOrOptions; - } - span?.setAttributes({ ...Attribute.WorkflowArgumentsCount(args.length), }); @@ -601,6 +675,20 @@ export async function start( } } + const crossDeployment = deploymentId !== currentDeploymentId; + // A dynamic run executes stored code, so it may only target the + // deployment that validated and opted in to it: this one. Rejected + // before the capability probe, key lookup, upload, or run creation. + if (dynamicWorkflow && crossDeployment) { + const current = + currentDeploymentId === undefined + ? 'an unknown current deployment' + : JSON.stringify(currentDeploymentId); + throw new WorkflowRuntimeError( + `Dynamic workflows can only start on the current deployment. This start targets ${JSON.stringify(deploymentId)} from ${current}, so no run was created.` + ); + } + // Decide whether to write byte streams in the framed wire format. // For same-deployment starts (the common case) we know the target is // running this same SDK version, so framing is safe. For cross- @@ -625,6 +713,18 @@ export async function start( : ulid() }`; + if (dynamicWorkflow) { + const backendCapabilities = await world.getBackendCapabilities?.(); + if ( + backendCapabilities?.dynamicWorkflowStorageVersion !== + DYNAMIC_WORKFLOW_VERSION + ) { + throw new WorkflowRuntimeError( + `Dynamic workflows require backend storage capability version ${DYNAMIC_WORKFLOW_VERSION}. No compatible capability was attested, so no run was created.` + ); + } + } + let framedByteStreams: boolean; let targetSupportsCompression: boolean; // The consumer's hook-resume protocol version, stamped onto the new @@ -642,7 +742,6 @@ export async function start( // probe) otherwise. See `resolveCrossDeploymentSpecVersion`. let targetSpecVersion: number; let specVersionSource: SpecVersionSource; - const crossDeployment = deploymentId !== currentDeploymentId; if (!crossDeployment) { framedByteStreams = true; targetSupportsCompression = true; @@ -881,6 +980,31 @@ export async function start( : undefined; } + // Build the complete execution context before serializing or uploading + // dynamic source and before either run-creation side effect. + const workflowVm = getWorkflowVmFromEnv(); + const executionContext = { + traceCarrier, + workflowCoreVersion, + features: { encryption: !!encryptionKey }, + ...(targetHookResumeInputVersion !== undefined + ? { hookResumeInputVersion: targetHookResumeInputVersion } + : {}), + ...(workflowVm ? { workflowVm } : {}), + ...(opts.replayedFromRunId + ? { replayedFromRunId: opts.replayedFromRunId } + : {}), + ...(dynamicWorkflow + ? { dynamicWorkflow: dynamicWorkflow.metadata } + : {}), + }; + // A dynamic run's marker is what can push the context past a World's + // limit, so only dynamic starts are validated here; static starts keep + // relying on the World's own write-time checks. + if (dynamicWorkflow) { + world.validateRunExecutionContext?.(executionContext); + } + // Create run via run_created event (event-sourced architecture) // Pass client-generated runId - server will accept and use it // Compress workflow arguments only when the run itself is marked as @@ -901,6 +1025,59 @@ export async function start( compression ); + // Dynamic workflow code goes through the same serialization pipeline as + // the arguments — compressed, then encrypted with the run's key — and is + // stored with the run, because every replay of a dynamic run has to + // evaluate the exact code it started on and that code is nowhere else. + // + // Two shapes on the wire: the bytes inline on `run_created` (the common + // case, no extra round-trip), or a ref to a separate upload when the + // payload is too large for the creating write's metadata budget. Worlds + // without an upload path always take the inline branch — they store run + // records whole, so there is no budget to exceed. + let dynamicWorkflowCode: Uint8Array | undefined; + let dynamicWorkflowCodeRef: string | undefined; + if (dynamicWorkflow) { + const serializedCode = await dehydrateDynamicWorkflowCode( + dynamicWorkflow.code, + encryptionKey, + compression + ); + if ( + serializedCode.byteLength > DYNAMIC_WORKFLOW_CODE_INLINE_MAX_BYTES && + world.uploadDynamicWorkflowCode + ) { + dynamicWorkflowCodeRef = await world.uploadDynamicWorkflowCode( + runId, + { workflowName, code: serializedCode } + ); + } else { + dynamicWorkflowCode = serializedCode; + } + span?.setAttributes({ + ...Attribute.WorkflowDynamic(true), + ...Attribute.WorkflowDynamicSourceHash( + dynamicWorkflow.metadata.sourceHash + ), + ...Attribute.WorkflowDynamicCodeBytes(serializedCode.byteLength), + ...Attribute.WorkflowDynamicCodeStorage( + dynamicWorkflowCodeRef ? 'ref' : 'inline' + ), + }); + } + + /** + * Shared by `run_created` and the queue message's `runInput`: the + * resilient-start path re-creates the run from the queue message, and a + * dynamic run created without its code could never replay. + */ + const dynamicWorkflowSeed = dynamicWorkflow + ? { + ...(dynamicWorkflowCode ? { dynamicWorkflowCode } : {}), + ...(dynamicWorkflowCodeRef ? { dynamicWorkflowCodeRef } : {}), + } + : {}; + // The environment this caller's own `run_created` write is attributed // to. Stamped into the queue message's `runInput` (NOT into // `run_created`, whose tenant the backend already knows) so the @@ -919,35 +1096,6 @@ export async function start( // is absent. const creatorEnvironment = world.getEnvironment?.(); - // If WORKFLOW_VM is set on the client starting the run, stamp the - // engine choice into the run's executionContext so the run keeps - // executing on the engine it started on (the same deployment can - // serve both VM engines). Unknown values throw; see - // getWorkflowVmFromEnv(). - const workflowVm = getWorkflowVmFromEnv(); - - const executionContext = { - traceCarrier, - workflowCoreVersion, - features: { encryption: !!encryptionKey }, - // Attest that the *consumer* deployment's runtime re-ensures a - // `hook_received` event from a queue message's `hookInput` on replay. - // An OLDER producer resuming this run reads the marker (mirrored onto - // the hook's resumeContext by the server) to decide whether its lazy - // fast path is safe. For a cross-deployment start the consumer is the - // target deployment, so we stamp the *target's* value carried back on - // the health-check probe, never the caller's. Omitted when we could - // not attest the target (older target, timeout, or no probe channel), - // which fails the resume gate closed to the sequential path. - ...(targetHookResumeInputVersion !== undefined - ? { hookResumeInputVersion: targetHookResumeInputVersion } - : {}), - ...(workflowVm ? { workflowVm } : {}), - ...(opts.replayedFromRunId - ? { replayedFromRunId: opts.replayedFromRunId } - : {}), - }; - // Call events.create (run_created) and queue in parallel. // If events.create fails with 429/5xx, the run was still accepted // via the queue and creation will be re-tried async by the runtime. @@ -964,12 +1112,13 @@ export async function start( executionContext, ...(encryptionPublicKey ? { encryptionPublicKey } : {}), ...attributeSeed, + ...dynamicWorkflowSeed, }, }, { v1Compat } ), world.queue( - getWorkflowQueueName(workflowName, opts.namespace), + queueName, { runId, traceCarrier, @@ -986,6 +1135,7 @@ export async function start( ? { environment: creatorEnvironment } : {}), ...attributeSeed, + ...dynamicWorkflowSeed, }, } : {}), @@ -1038,6 +1188,26 @@ export async function start( `Server returned different runId than requested: expected ${runId}, got ${result.run.runId}` ); } + // Verify the backend actually stored the dynamic workflow code. + // + // A backend that predates dynamic-source support ignores the field + // rather than rejecting it — dropping unrecognized metadata is by + // design — so the write succeeds and the run looks fine. It is not: + // nothing can ever replay it, and the failure would surface much + // later as an unregistered-workflow error on a queue delivery with no + // hint that the backend's age was the cause. The created run echoes + // what it persisted, so this check costs nothing and moves the + // failure to the call site. + if ( + dynamicWorkflow && + (result.run as { dynamicWorkflowCode?: unknown }) + .dynamicWorkflowCode === undefined + ) { + throw new WorkflowRuntimeError( + `Workflow run ${runId} was created, but this deployment's Workflow backend did not store its dynamic workflow code, so the run can never be replayed. ` + + 'Dynamic workflows require a backend with encrypted dynamic-source storage; upgrade it, or start a workflow function from the build-time manifest instead.' + ); + } } // These argument-stream ops are flushed in the background; the promise diff --git a/packages/core/src/serialization-format.test.ts b/packages/core/src/serialization-format.test.ts index 5a19584a8f..a0591d5daf 100644 --- a/packages/core/src/serialization-format.test.ts +++ b/packages/core/src/serialization-format.test.ts @@ -200,6 +200,27 @@ describe('hydrateResourceIO', () => { expect(hydrated.output).toEqual({ status: 'completed' }); }); + it("should hydrate a dynamic run's workflow code alongside its input", () => { + const code = 'async function workflow() { "use workflow"; }'; + const run = { + runId: 'wrun_dyn', + input: makeDevlPayload(['arg']), + dynamicWorkflowCode: makeDevlPayload(code), + }; + + const hydrated = hydrateResourceIO(run, testRevivers); + expect(hydrated.input).toEqual(['arg']); + expect(hydrated.dynamicWorkflowCode).toBe(code); + }); + + it('should not add dynamicWorkflowCode to a static run', () => { + const hydrated = hydrateResourceIO( + { runId: 'wrun_static', input: makeDevlPayload(['arg']) }, + testRevivers + ); + expect('dynamicWorkflowCode' in hydrated).toBe(false); + }); + it('should hydrate event eventData.result', () => { const resultPayload = makeDevlPayload({ key: 'value' }); diff --git a/packages/core/src/serialization-format.ts b/packages/core/src/serialization-format.ts index b8060f6915..158a1b72d2 100644 --- a/packages/core/src/serialization-format.ts +++ b/packages/core/src/serialization-format.ts @@ -729,11 +729,17 @@ function hydrateStepIO< * Hydrate the data fields of a workflow run resource. */ function hydrateWorkflowIO< - T extends { input?: any; output?: any; error?: any }, + T extends { + input?: any; + output?: any; + error?: any; + dynamicWorkflowCode?: any; + }, >(resource: T, revivers: Revivers): T { let hydratedInput = resource.input; let hydratedOutput = resource.output; let hydratedError = resource.error; + let hydratedDynamicWorkflowCode = resource.dynamicWorkflowCode; if (resource.input != null) { try { @@ -762,11 +768,27 @@ function hydrateWorkflowIO< } } + // A dynamic run's own workflow code, stored through the same pipeline as + // `input`. Absent on every static run, so this is a no-op for them. + if (resource.dynamicWorkflowCode != null) { + try { + hydratedDynamicWorkflowCode = hydrateData( + resource.dynamicWorkflowCode, + revivers + ); + } catch { + // Leave un-hydrated + } + } + return { ...resource, input: hydratedInput, output: hydratedOutput, error: hydratedError, + ...(resource.dynamicWorkflowCode != null + ? { dynamicWorkflowCode: hydratedDynamicWorkflowCode } + : {}), }; } diff --git a/packages/core/src/serialization.test.ts b/packages/core/src/serialization.test.ts index 62a967a680..96d576179c 100644 --- a/packages/core/src/serialization.test.ts +++ b/packages/core/src/serialization.test.ts @@ -6,10 +6,12 @@ import { RetryableError, RUN_ERROR_CODES, RuntimeDecryptionError, + SerializationError, StreamError, WorkflowWorldError, } from '@workflow/errors'; import { WORKFLOW_DESERIALIZE, WORKFLOW_SERIALIZE } from '@workflow/serde'; +import { stringify } from 'devalue'; import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import { registerSerializationClass } from './class-serialization.js'; import { decrypt, encrypt, importKey } from './encryption.js'; @@ -27,17 +29,20 @@ import { import { cancelAbortReaders, decodeFormatPrefix, + dehydrateDynamicWorkflowCode, dehydrateRunError, dehydrateStepArguments, dehydrateStepError, dehydrateStepReturnValue, dehydrateWorkflowArguments, dehydrateWorkflowReturnValue, + encodeWithFormatPrefix, getCommonRevivers, getDeserializeStream, getSerializeStream, getStreamType, getWorkflowReducers, + hydrateDynamicWorkflowCode, hydrateRunError, hydrateStepArguments, hydrateStepError, @@ -4789,6 +4794,132 @@ describe('dehydrate/hydrateStepError', () => { }); }); +describe('dehydrate/hydrateDynamicWorkflowCode', () => { + const code = + 'globalThis.__private_workflows ??= new Map();\nasync function workflow() { "use workflow"; return 1; }\n'; + + it('round-trips through compression and encryption', async () => { + const previousCodec = process.env.WORKFLOW_COMPRESSION_CODEC; + process.env.WORKFLOW_COMPRESSION_CODEC = 'gzip'; + try { + const compressibleCode = `${code}${'// repetitive generated code\n'.repeat(100)}`; + const material = new Uint8Array(32).fill(0x7d); + const keys = runPayloadKeys( + await importKey(material), + await deriveRunKeyPair(material) + ); + const stored = await dehydrateDynamicWorkflowCode( + compressibleCode, + keys, + true + ); + expect(isEncrypted(stored)).toBe(true); + const decrypted = await decryptEnvelope(stored, keys); + expect(decodeFormatPrefix(decrypted as Uint8Array).format).toBe( + SerializationFormat.GZIP + ); + expect(await hydrateDynamicWorkflowCode(stored, keys)).toBe( + compressibleCode + ); + } finally { + if (previousCodec === undefined) { + delete process.env.WORKFLOW_COMPRESSION_CODEC; + } else { + process.env.WORKFLOW_COMPRESSION_CODEC = previousCodec; + } + } + }); + + it('round-trips unencrypted and uncompressed', async () => { + const stored = await dehydrateDynamicWorkflowCode(code, undefined); + expect(await hydrateDynamicWorkflowCode(stored, undefined)).toBe(code); + }); + + it('is readable by the generic hydrator the CLI and UI use', async () => { + const stored = await dehydrateDynamicWorkflowCode(code, undefined); + expect(hydrateData(stored, {})).toBe(code); + }); + + describe('when encryption is required', () => { + const material = new Uint8Array(32).fill(0x3c); + async function runKeys() { + return runPayloadKeys( + await importKey(material), + await deriveRunKeyPair(material) + ); + } + + it('rejects plaintext code when the run has a key', async () => { + const stored = await dehydrateDynamicWorkflowCode(code, undefined); + await expect( + hydrateDynamicWorkflowCode(stored, await runKeys()) + ).rejects.toThrow(/must be encrypted.*"devl"/); + }); + + it('rejects plaintext code when the run was started with encryption', async () => { + const stored = await dehydrateDynamicWorkflowCode(code, undefined); + await expect( + hydrateDynamicWorkflowCode(stored, undefined, { + encryptionRequired: true, + }) + ).rejects.toBeInstanceOf(SerializationError); + }); + + it('rejects code sealed to the run public key', async () => { + const { publicKey } = await deriveRunKeyPair(material); + const stored = await dehydrateDynamicWorkflowCode( + code, + sealTo(publicKey) + ); + expect(decodeFormatPrefix(stored).format).toBe( + SerializationFormat.SEALED + ); + await expect( + hydrateDynamicWorkflowCode(stored, await runKeys()) + ).rejects.toThrow(/must be encrypted.*"encp"/); + }); + + it('rejects compressed plaintext code when the run has a key', async () => { + const stored = await dehydrateDynamicWorkflowCode( + `${code}${'// padding\n'.repeat(200)}`, + undefined, + true + ); + await expect( + hydrateDynamicWorkflowCode(stored, await runKeys()) + ).rejects.toBeInstanceOf(SerializationError); + }); + + it('accepts code encrypted with the run key', async () => { + const keys = await runKeys(); + const stored = await dehydrateDynamicWorkflowCode(code, keys); + await expect( + hydrateDynamicWorkflowCode(stored, keys, { encryptionRequired: true }) + ).resolves.toBe(code); + }); + }); + + it('rejects a payload that is not a string with SerializationError', async () => { + const prefixed = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + new TextEncoder().encode(stringify({ a: 1 })) + ); + await expect( + hydrateDynamicWorkflowCode(prefixed, undefined) + ).rejects.toBeInstanceOf(SerializationError); + }); + + it('rejects a corrupt payload with SerializationError, not a raw SyntaxError', async () => { + const prefixed = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + new TextEncoder().encode('{not json') + ); + await expect( + hydrateDynamicWorkflowCode(prefixed, undefined) + ).rejects.toBeInstanceOf(SerializationError); + }); +}); + describe('dehydrate/hydrateRunError', () => { // The run-error helpers use the workflow reducers (vs. the step reducers // used above), but the surface contract is the same. These tests cover the diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index dba0297419..6784ccd75b 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -4181,6 +4181,134 @@ export async function hydrateStepError( ); } +/** + * Serialize a dynamic run's generated workflow VM code for storage. + * + * Dynamic workflow code is application source, so it gets the same treatment + * as any other run payload: compressed (it is plain text, which compresses + * very well), then encrypted with the run's key. At rest it is opaque + * ciphertext, which is the point — generated orchestration can name internal + * step ids, prompts, and business rules, and observability surfaces must not + * read it without going through the decrypt flow. + * + * The code is a plain string, so it needs none of the reducers the other + * payloads go through, but it is still written as real devalue under the + * `DEVALUE_V1` prefix: the generic hydrators (`hydrateData` behind the CLI + * and the observability UI) trust that prefix and hand the bytes to + * devalue's `parse`, which rejects a bare JSON string as invalid input. + * + * @param code - Generated workflow VM code. + * @param key - Encryption key (undefined to skip encryption). + * @param compression - Whether the target run may carry compressed payloads. + */ +export async function dehydrateDynamicWorkflowCode( + code: string, + key: PayloadKey | undefined, + compression = false +): Promise { + try { + const payload = new TextEncoder().encode(stringify(code)); + const serialized = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + payload + ) as Uint8Array; + // Compress before encrypting — encrypted bytes don't compress. + const compressionStats: CompressionStats = {}; + const compressed = await compress( + serialized, + compression, + compressionStats + ); + const encrypted = (await maybeEncrypt( + compressed as Uint8Array, + key + )) as Uint8Array; + await recordCompression(compressionStats, 'serialize'); + return encrypted; + } catch (error) { + const cause = unwrapSerializationCause(error); + const { message, hint } = formatSerializationError( + 'dynamic workflow code', + cause + ); + throw new SerializationError(message, { hint, cause }); + } +} + +/** + * Hydrate a dynamic run's workflow VM code back into source, on the replay + * path. + * + * When the run has key material, or its execution context records that it was + * started with encryption, only the symmetric `encr` envelope is accepted. + * Plaintext would let anyone who can write the run record supply code, and a + * sealed `encp` envelope can be produced by anyone holding the run's public + * key, so neither is evidence that the run's own key encrypted it. This gives + * confidentiality and narrows who can supply code; it is not an integrity + * guarantee against a holder of the run key. + * + * Without key material (Worlds with no encryption), plaintext is accepted. + * + * @param value - Stored bytes from the run's `dynamicWorkflowCode`. + * @param key - Encryption key (undefined when encryption is disabled). + * @param options.encryptionRequired - The run was started with encryption + * (`executionContext.features.encryption`), so plaintext is refused even + * when no key was resolved. + * @throws SerializationError when the payload is not `encr` although + * encryption is required, or does not decode to a string: a corrupted or + * foreign payload must not reach the workflow VM as code. + */ +export async function hydrateDynamicWorkflowCode( + value: Uint8Array | unknown, + key: PayloadKey | undefined, + options: { encryptionRequired?: boolean } = {} +): Promise { + if (key !== undefined || options.encryptionRequired === true) { + const envelope = peekFormatPrefix(value); + if (envelope !== SerializationFormat.ENCRYPTED) { + throw new SerializationError( + `Dynamic workflow code must be encrypted with the run's key ("${SerializationFormat.ENCRYPTED}"), but the stored payload is ${envelope ? `"${envelope}"` : 'not a recognized format'}.` + ); + } + } + + const compressionStats: CompressionStats = {}; + const decrypted = await decompress( + await decrypt(value, key), + compressionStats + ); + await recordCompression(compressionStats, 'deserialize'); + + if (!(decrypted instanceof Uint8Array)) { + throw new SerializationError( + 'Dynamic workflow code did not decode to binary data.' + ); + } + + const { format, payload } = decodeFormatPrefix(decrypted); + if (format !== SerializationFormat.DEVALUE_V1) { + throw new SerializationError( + `Unsupported serialization format for dynamic workflow code: ${format}` + ); + } + + let code: unknown; + try { + code = parse(new TextDecoder().decode(payload)); + } catch (cause) { + throw new SerializationError( + 'Dynamic workflow code payload is not valid devalue.', + { cause } + ); + } + if (typeof code !== 'string') { + throw new SerializationError( + `Dynamic workflow code decoded to ${typeof code}, expected a string.` + ); + } + return code; +} + /** * Called from the workflow handler when the workflow itself throws. * Dehydrates the thrown value from within the workflow execution environment diff --git a/packages/core/src/telemetry/semantic-conventions.ts b/packages/core/src/telemetry/semantic-conventions.ts index 007db3f6db..607aff24f1 100644 --- a/packages/core/src/telemetry/semantic-conventions.ts +++ b/packages/core/src/telemetry/semantic-conventions.ts @@ -82,6 +82,27 @@ export const WorkflowExecutionMode = SemanticConvention<'replay' | 'retained'>( 'workflow.execution.mode' ); +/** + * Whether the run executes dynamic workflow code stored with the run rather + * than the deployment's workflow bundle. + */ +export const WorkflowDynamic = SemanticConvention('workflow.dynamic'); + +/** SHA-256 of a dynamic run's source and step bindings. */ +export const WorkflowDynamicSourceHash = SemanticConvention( + 'workflow.dynamic.source_hash' +); + +/** Size in bytes of a dynamic run's serialized workflow code. */ +export const WorkflowDynamicCodeBytes = SemanticConvention( + 'workflow.dynamic.code_bytes' +); + +/** Whether a dynamic run's code was stored inline or behind a ref. */ +export const WorkflowDynamicCodeStorage = SemanticConvention<'inline' | 'ref'>( + 'workflow.dynamic.code_storage' +); + /** Whether the compiled application workflow bundle was cached. */ export const WorkflowBundleCompileCacheHit = SemanticConvention( 'workflow.bundle.compile.cache_hit' diff --git a/packages/core/src/vm/script-cache.test.ts b/packages/core/src/vm/script-cache.test.ts index 1dc2b34db5..e826fb108e 100644 --- a/packages/core/src/vm/script-cache.test.ts +++ b/packages/core/src/vm/script-cache.test.ts @@ -1,5 +1,9 @@ import { type Context, runInContext } from 'node:vm'; import { afterEach, describe, expect, it } from 'vitest'; +import { + compileDynamicWorkflowBundle, + compileWorkflowBundle, +} from '../workflow.js'; import { createContext } from './index.js'; import { clearWorkflowScriptCache, @@ -49,6 +53,31 @@ describe('script-cache', () => { clearWorkflowScriptCache(); }); + it('keeps dynamic workflow compilation out of the static cache', async () => { + const staticScripts = await compileWorkflowBundle( + SAMPLE_BUNDLE, + 'my/workflow' + ); + const staticCacheSize = workflowScriptCacheSize(); + + for (let i = 0; i < 12; i++) { + const name = `dynamic/workflow-${i}`; + const code = `globalThis.__private_workflows = new Map(); globalThis.__private_workflows.set(${JSON.stringify(name)}, async function workflow() { return ${i}; });`; + const dynamic = await compileDynamicWorkflowBundle(code, name); + const { context } = createContext({ seed, fixedTimestamp }); + dynamic.bundleScript.runInContext(context); + const workflow = dynamic.workflowLookupScript.runInContext( + context + ) as () => Promise | number; + expect(await workflow()).toBe(i); + } + + expect(workflowScriptCacheSize()).toBe(staticCacheSize); + expect( + (await compileWorkflowBundle(SAMPLE_BUNDLE, 'my/workflow')).bundleScript + ).toBe(staticScripts.bundleScript); + }); + it('returns the same compiled Script for identical (code, filename)', () => { const a = getScript(SAMPLE_BUNDLE, 'workflows/a.ts'); const b = getScript(SAMPLE_BUNDLE, 'workflows/a.ts'); diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index 4c2dbdb1cf..d8c52bbada 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -7,6 +7,7 @@ import { afterEach, assert, describe, expect, it, vi } from 'vitest'; import { DEFERRED_CHECK_DELAY_MS } from './events-consumer.js'; import type { WorkflowSuspension } from './global.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; +import { compileDynamicWorkflow } from './runtime/dynamic-workflow.js'; import { setWorld } from './runtime/world.js'; import { dehydrateStepReturnValue, @@ -32,6 +33,51 @@ describe('runWorkflow', () => { `; describe('successful workflow execution', () => { + it('replays dynamic source with caller globals isolated from registration', async () => { + const compiled = await compileDynamicWorkflow( + ` +const Object = null; +var globalThis = null; +function __dynamicWorkflow() {} +async function workflow() { + "use workflow"; + return 42; +} +`, + { steps: { unused: { stepId: 'step//./test//unused' } } } + ); + const workflowRun: WorkflowRun = { + runId: 'wrun_dynamic', + workflowName: compiled.workflowName, + status: 'running', + input: await dehydrateWorkflowArguments( + [], + 'wrun_dynamic', + noEncryptionKey, + [] + ), + createdAt: new Date(), + updatedAt: new Date(), + startedAt: new Date(), + deploymentId: 'test-deployment', + }; + + const result = await runWorkflow( + compiled.workflowCode, + workflowRun, + [], + noEncryptionKey + ); + expect( + await hydrateWorkflowReturnValue( + result, + workflowRun.runId, + noEncryptionKey, + [] + ) + ).toBe(42); + }); + it('should execute a simple workflow successfully', async () => { const ops: Promise[] = []; const workflowCode = `function workflow() { return "success"; }${getWorkflowTransformCode('workflow')}`; diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 61d3180f2d..24b99a2adc 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -1,4 +1,4 @@ -import type { Script } from 'node:vm'; +import { Script } from 'node:vm'; import type { Span } from '@opentelemetry/api'; import { ERROR_SLUGS, @@ -151,17 +151,30 @@ export interface CompiledWorkflowScripts { * this promise while `run_started` loads the replay snapshot, then evaluates * the scripts only after it has created the fresh context. */ -export function compileWorkflowBundle( +function compileWorkflowScripts( workflowCode: string, - workflowName: string + workflowName: string, + cache: 'shared' | 'none' ): Promise { const parsedName = parseWorkflowName(workflowName); const filename = parsedName?.moduleSpecifier || workflowName; const workflowLookupCode = `globalThis.__private_workflows?.get(${JSON.stringify(workflowName)})`; return trace('workflow.bundle.compile', async (span) => { - const bundle = getCachedWorkflowScript(workflowCode, filename); - const lookup = getCachedWorkflowScript(workflowLookupCode, filename); + const bundle = + cache === 'shared' + ? getCachedWorkflowScript(workflowCode, filename) + : { + script: new Script(workflowCode, { filename }), + cacheHit: false, + }; + const lookup = + cache === 'shared' + ? getCachedWorkflowScript(workflowLookupCode, filename) + : { + script: new Script(workflowLookupCode, { filename }), + cacheHit: false, + }; span?.setAttributes({ // This attribute intentionally describes the workflow bundle. The tiny // lookup script may miss when another workflow from the same source file @@ -175,6 +188,21 @@ export function compileWorkflowBundle( }); } +export function compileWorkflowBundle( + workflowCode: string, + workflowName: string +): Promise { + return compileWorkflowScripts(workflowCode, workflowName, 'shared'); +} + +/** Compile invocation-scoped dynamic source without touching the static cache. */ +export function compileDynamicWorkflowBundle( + workflowCode: string, + workflowName: string +): Promise { + return compileWorkflowScripts(workflowCode, workflowName, 'none'); +} + /** * A live workflow VM, parked at a suspension boundary. `resume` advances it * by appending events instead of replaying from scratch. diff --git a/packages/web-shared/src/components/sidebar/attribute-panel.tsx b/packages/web-shared/src/components/sidebar/attribute-panel.tsx index 08af853965..de0ebbf7f4 100644 --- a/packages/web-shared/src/components/sidebar/attribute-panel.tsx +++ b/packages/web-shared/src/components/sidebar/attribute-panel.tsx @@ -294,6 +294,7 @@ const attributeOrder: AttributeKey[] = [ 'eventData', 'input', 'output', + 'dynamicWorkflowCode', 'attributes', 'resumeAt', ]; @@ -345,6 +346,7 @@ const attributeDisplayNames: Partial> = { lastReceivedAt: 'Last Received', disposedAt: 'Disposed', receivedCount: 'Times Resolved', + dynamicWorkflowCode: 'Workflow Code', }; /** @@ -687,6 +689,38 @@ const attributeToDisplayFn: Record< // cross-run writers to seal payloads to this run. Not actionable for users // and not secret, so hidden rather than rendered as 44 opaque base64 chars. encryptionPublicKey: (_value: unknown) => null, + // A dynamic run's own workflow code — the code the run actually executed, + // which for these runs exists nowhere in the deployment's source. Shown as + // its own collapsed section: it is the most useful thing on a dynamic run's + // detail view, and irrelevant (absent) on every other run. + // + // Follows the same locked-by-default treatment as input/output: generated + // orchestration names internal step ids, prompts and business rules, so the + // decrypt flow is the gate on reading it. + dynamicWorkflowCode: (value: unknown, context?: DisplayContext) => { + if (isEncryptedMarker(value)) { + return ( + + + + ); + } + if (isExpiredMarker(value)) return ; + if (typeof value !== 'string' || value.trim().length === 0) return null; + return ( + + + + ); + }, }; const resolvableAttributes = [ @@ -696,6 +730,7 @@ const resolvableAttributes = [ 'metadata', 'attributes', 'eventData', + 'dynamicWorkflowCode', ]; // Attributes whose displayFn renders its own section header via Collapsible, @@ -707,6 +742,7 @@ const selfHeaderedAttributes = new Set([ 'metadata', 'attributes', 'eventData', + 'dynamicWorkflowCode', ]); const ExpiredDataMessage = () => ( @@ -737,6 +773,7 @@ const loadingSectionLabels: Partial> = { input: 'Input', output: 'Output', eventData: 'Event Data', + dynamicWorkflowCode: 'Workflow Code', }; export const AttributeBlock = ({ diff --git a/packages/web-shared/src/lib/hydration.test.ts b/packages/web-shared/src/lib/hydration.test.ts new file mode 100644 index 0000000000..859e7c64db --- /dev/null +++ b/packages/web-shared/src/lib/hydration.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest'; +import { + hasEncryptedFields, + hydrateResourceIO, + isEncryptedMarker, +} from './hydration'; + +/** + * A `devl`-prefixed payload, the unencrypted form stored by the SDK. The + * body is devalue's flattened form: an array whose first element is the + * root value (enough for the primitives and flat arrays used here). + */ +function devl(value: unknown): Uint8Array { + const flattened = Array.isArray(value) + ? [value.map((_, i) => i + 1), ...value] + : [value]; + const body = new TextEncoder().encode(JSON.stringify(flattened)); + const out = new Uint8Array(4 + body.byteLength); + out.set(new TextEncoder().encode('devl'), 0); + out.set(body, 4); + return out; +} + +/** An `encr`-prefixed payload: ciphertext the browser cannot read as-is. */ +function encr(): Uint8Array { + const out = new Uint8Array(4 + 16); + out.set(new TextEncoder().encode('encr'), 0); + return out; +} + +describe('hydrateResourceIO: dynamicWorkflowCode', () => { + it("hydrates a dynamic run's code to the source string", () => { + const code = 'async function workflow() { "use workflow"; }'; + const run = hydrateResourceIO({ + runId: 'wrun_1', + input: devl(['arg']), + dynamicWorkflowCode: devl(code), + }); + expect(run.input).toEqual(['arg']); + expect(run.dynamicWorkflowCode).toBe(code); + }); + + it('marks encrypted code the same way it marks encrypted input', () => { + const run = hydrateResourceIO({ + runId: 'wrun_1', + input: encr(), + dynamicWorkflowCode: encr(), + }); + expect(isEncryptedMarker(run.input)).toBe(true); + expect(isEncryptedMarker(run.dynamicWorkflowCode)).toBe(true); + }); + + it('counts encrypted code toward hasEncryptedFields', () => { + // With the input already decrypted, the code alone has to keep the + // Decrypt affordance available. + const run = hydrateResourceIO({ + runId: 'wrun_1', + input: devl(['arg']), + dynamicWorkflowCode: encr(), + }); + expect(hasEncryptedFields(run)).toBe(true); + }); +}); diff --git a/packages/web-shared/src/lib/hydration.ts b/packages/web-shared/src/lib/hydration.ts index be039aba24..4427cc6064 100644 --- a/packages/web-shared/src/lib/hydration.ts +++ b/packages/web-shared/src/lib/hydration.ts @@ -459,6 +459,20 @@ function toDisplayMarker(value: unknown): unknown { return value; } +/** + * Top-level fields of a run, step, or hook that hold serialized (and possibly + * encrypted) payloads. `dynamicWorkflowCode` is a run's own workflow code, + * present only on runs started from source; it is stored through the same + * pipeline as `input` and is gated behind the same decrypt flow. + */ +const TOP_LEVEL_SERIALIZED_FIELDS = [ + 'input', + 'output', + 'metadata', + 'error', + 'dynamicWorkflowCode', +] as const; + /** * Post-process hydrated resource data: replace encrypted Uint8Array values * and expired stubs with display-friendly marker objects in known data fields. @@ -468,7 +482,7 @@ function replaceEncryptedAndExpiredWithMarkers(resource: T): T { const r = resource as Record; const result = { ...r }; - for (const key of ['input', 'output', 'metadata', 'error']) { + for (const key of TOP_LEVEL_SERIALIZED_FIELDS) { result[key] = toDisplayMarker(result[key]); } @@ -552,7 +566,7 @@ export async function hydrateResourceIOAsync( const result = { ...r }; // Decrypt + hydrate top-level serialized fields (runs, steps, hooks) - for (const field of ['input', 'output', 'metadata', 'error']) { + for (const field of TOP_LEVEL_SERIALIZED_FIELDS) { if (field in result) { result[field] = await hydrateField(result[field]); } @@ -585,7 +599,7 @@ export function hasEncryptedFields(resource: unknown): boolean { if (!resource || typeof resource !== 'object') return false; const r = resource as Record; - for (const key of ['input', 'output', 'metadata', 'error']) { + for (const key of TOP_LEVEL_SERIALIZED_FIELDS) { if (isEncryptedMarker(r[key])) return true; } diff --git a/packages/web/app/components/run-actions.tsx b/packages/web/app/components/run-actions.tsx index a135392dc1..ed8f81783f 100644 --- a/packages/web/app/components/run-actions.tsx +++ b/packages/web/app/components/run-actions.tsx @@ -44,6 +44,7 @@ export interface RunActionsBaseProps { runStatus: WorkflowRunStatus | undefined; events?: Event[]; eventsLoading?: boolean; + replayDisabledReason?: string; callbacks?: RunActionCallbacks; } @@ -56,6 +57,7 @@ interface UseRunActionsOptions { runId: string; runStatus: WorkflowRunStatus | undefined; events?: Event[]; + replayDisabledReason?: string; callbacks?: RunActionCallbacks; } @@ -64,6 +66,7 @@ function useRunActions({ runId, runStatus, events, + replayDisabledReason, callbacks, }: UseRunActionsOptions) { const [rerunning, setRerunning] = useState(false); @@ -75,7 +78,7 @@ function useRunActions({ const hasPendingSleeps = eventAnalysis.hasPendingSleeps; const handleReplay = useCallback(async () => { - if (rerunning) return null; + if (rerunning || replayDisabledReason) return null; try { setRerunning(true); @@ -94,7 +97,7 @@ function useRunActions({ } finally { setRerunning(false); } - }, [env, runId, rerunning, callbacks]); + }, [env, runId, rerunning, replayDisabledReason, callbacks]); const handleReenqueue = useCallback(async () => { if (reenqueuing) return; @@ -224,6 +227,7 @@ export function RunActionsDropdownItems({ events, eventsLoading, callbacks, + replayDisabledReason, stopPropagation = false, }: RunActionsDropdownItemsProps) { const { @@ -236,7 +240,14 @@ export function RunActionsDropdownItems({ handleReenqueue, handleWakeUp, handleCancel, - } = useRunActions({ env, runId, runStatus, events, callbacks }); + } = useRunActions({ + env, + runId, + runStatus, + events, + replayDisabledReason, + callbacks, + }); const onReplay = (e: React.MouseEvent) => { if (stopPropagation) e.stopPropagation(); @@ -262,10 +273,22 @@ export function RunActionsDropdownItems({ return ( <> - - - {rerunning ? 'Replaying...' : 'Replay Run'} - + + + + + {rerunning ? 'Replaying...' : 'Replay Run'} + + + {replayDisabledReason ? ( + + {replayDisabledReason} + + ) : null} + {/* Re-enqueue - always shown */} @@ -330,6 +353,7 @@ export function RunActionsButtons({ eventsLoading, loading, callbacks, + replayDisabledReason, onCancelClick, onRerunClick, }: RunActionsButtonsProps) { @@ -345,12 +369,14 @@ export function RunActionsButtons({ const canCancel = isRunActive; // Rerun button logic - const canRerun = !loading && !isRunActive; - const rerunDisabledReason = loading - ? 'Loading run data...' - : isRunActive - ? 'Cannot re-run while workflow is still running' - : ''; + const canRerun = !loading && !isRunActive && !replayDisabledReason; + const rerunDisabledReason = replayDisabledReason + ? replayDisabledReason + : loading + ? 'Loading run data...' + : isRunActive + ? 'Cannot re-run while workflow is still running' + : ''; // Cancel button logic const cancelDisabledReason = diff --git a/packages/web/app/components/run-detail-view.tsx b/packages/web/app/components/run-detail-view.tsx index a17509fea2..f05b4f2b5d 100644 --- a/packages/web/app/components/run-detail-view.tsx +++ b/packages/web/app/components/run-detail-view.tsx @@ -457,12 +457,15 @@ export function RunDetailView({ } }; + const isDynamicRun = Boolean(run.executionContext?.dynamicWorkflow); + const handleRerunClick = () => { + if (isDynamicRun) return; setShowRerunDialog(true); }; const handleConfirmRerun = async () => { - if (rerunning) return; + if (rerunning || isDynamicRun) return; try { setRerunning(true); @@ -590,6 +593,11 @@ export function RunDetailView({ events={allEvents} eventsLoading={loading} loading={loading} + replayDisabledReason={ + isDynamicRun + ? 'Dynamic runs cannot be replayed as a new run.' + : undefined + } onRerunClick={handleRerunClick} onCancelClick={handleCancelClick} callbacks={{ onSuccess: update }} diff --git a/packages/web/app/components/runs-table.tsx b/packages/web/app/components/runs-table.tsx index 3b4ee75233..c4feeed7aa 100644 --- a/packages/web/app/components/runs-table.tsx +++ b/packages/web/app/components/runs-table.tsx @@ -52,6 +52,7 @@ import { } from '~/lib/client/listing-window'; import { useTableSelection } from '~/lib/hooks/use-table-selection'; import { fetchEvents, fetchRun } from '~/lib/rpc-client'; +import { getRunReplayDisabledReason } from '~/lib/run-replay'; import type { EnvMap } from '~/lib/types'; import { bulkCancelRuns, @@ -82,30 +83,40 @@ function RunActionsDropdownContentInner({ onSuccess: () => void; }) { const [events, setEvents] = useState(undefined); - const [isLoading, setIsLoading] = useState(true); + const [eventsLoading, setEventsLoading] = useState(true); const [run, setRun] = useState(undefined); + const [runIdentityLoading, setRunIdentityLoading] = useState(true); const status = run?.status || runStatus; useEffect(() => { - setIsLoading(true); - - Promise.all([ - fetchRun(env, runId, 'none'), - fetchEvents(env, runId, { limit: 1000, sortOrder: 'desc' }), - ]) - .then(([runResult, eventsResult]) => { + setRun(undefined); + setRunIdentityLoading(true); + fetchRun(env, runId, 'none') + .then((runResult) => { if (runResult.success) { setRun(runResult.data); } + }) + .catch((err: unknown) => { + console.error('Failed to fetch run:', err); + }) + .finally(() => { + setRunIdentityLoading(false); + }); + + setEvents(undefined); + setEventsLoading(true); + fetchEvents(env, runId, { limit: 1000, sortOrder: 'desc' }) + .then((eventsResult) => { if (eventsResult.success) { setEvents(eventsResult.data.data); } }) .catch((err: unknown) => { - console.error('Failed to fetch run or events:', err); + console.error('Failed to fetch events:', err); }) .finally(() => { - setIsLoading(false); + setEventsLoading(false); }); }, [env, runId]); @@ -115,7 +126,8 @@ function RunActionsDropdownContentInner({ runId={runId} runStatus={status} events={events} - eventsLoading={isLoading} + eventsLoading={eventsLoading} + replayDisabledReason={getRunReplayDisabledReason(run, runIdentityLoading)} stopPropagation callbacks={{ onSuccess }} /> diff --git a/packages/web/app/lib/run-replay.test.ts b/packages/web/app/lib/run-replay.test.ts new file mode 100644 index 0000000000..3091ebd78d --- /dev/null +++ b/packages/web/app/lib/run-replay.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest'; +import { getRunReplayDisabledReason } from './run-replay'; + +describe('getRunReplayDisabledReason', () => { + it('disables Replay until the full run identity loads', () => { + expect(getRunReplayDisabledReason(undefined, true)).toBe( + 'Loading run identity...' + ); + }); + + it('fails closed when the full run could not be loaded', () => { + expect(getRunReplayDisabledReason(undefined, false)).toBe( + 'Unable to verify whether this run can be replayed.' + ); + }); + + it('disables dynamic runs and preserves terminal static Replay', () => { + expect( + getRunReplayDisabledReason( + { executionContext: { dynamicWorkflow: { sourceHash: 'hash' } } }, + false + ) + ).toBe('Dynamic runs cannot be replayed as a new run.'); + expect( + getRunReplayDisabledReason({ executionContext: {} }, false) + ).toBeUndefined(); + }); +}); diff --git a/packages/web/app/lib/run-replay.ts b/packages/web/app/lib/run-replay.ts new file mode 100644 index 0000000000..5efcd78a47 --- /dev/null +++ b/packages/web/app/lib/run-replay.ts @@ -0,0 +1,17 @@ +import type { WorkflowRun } from '@workflow/world'; + +export function getRunReplayDisabledReason( + run: Pick | undefined, + runIdentityLoading: boolean +): string | undefined { + if (runIdentityLoading) { + return 'Loading run identity...'; + } + if (!run) { + return 'Unable to verify whether this run can be replayed.'; + } + if (run.executionContext?.dynamicWorkflow) { + return 'Dynamic runs cannot be replayed as a new run.'; + } + return undefined; +} diff --git a/packages/workflow/src/api.ts b/packages/workflow/src/api.ts index 8e002aaac3..97a4232014 100644 --- a/packages/workflow/src/api.ts +++ b/packages/workflow/src/api.ts @@ -35,6 +35,9 @@ export { type WorkflowRunWritableStreamOptions, } from '@workflow/core/runtime/run'; export { + type DynamicStartOptions, + type DynamicWorkflowOptions, + type DynamicWorkflowStepReference, type StartOptions, start, } from '@workflow/core/runtime/start'; diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index 079bc7deed..e1ec8fdef4 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -73,6 +73,9 @@ export function createWorld(args?: Partial): LocalWorld { const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { specVersion: mintedSpecVersion(), + getBackendCapabilities: async () => ({ + dynamicWorkflowStorageVersion: 1, + }), capabilities: { hookRetention: { active: true }, // world-local deduplicates concurrent `hook_received` writes sharing a diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 2aa683dc95..0fa9d5bb6d 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -1127,6 +1127,7 @@ export function createEventsStorage( attributes?: Record; allowReservedAttributes?: true; encryptionPublicKey?: string; + dynamicWorkflowCode?: SerializedData; }; if ( runInputData.deploymentId && @@ -1163,6 +1164,9 @@ export function createEventsStorage( // run from the queued message, which is exactly when the key // would otherwise be lost for the rest of the run's life. encryptionPublicKey: runInputData.encryptionPublicKey, + // Same reasoning: a dynamic run recreated without its code + // would be created unable to replay. + dynamicWorkflowCode: runInputData.dynamicWorkflowCode, createdAt: now, updatedAt: now, }; @@ -1643,6 +1647,16 @@ export function createEventsStorage( if (data.eventType === 'run_started' && 'eventData' in event) { delete (event as any).eventData; } + // Dynamic source is materialized onto the run and never retained on + // the creation event as a second durable copy. + if (event.eventType === 'run_created' && event.eventData) { + const { + dynamicWorkflowCode: _dynamicWorkflowCode, + dynamicWorkflowCodeRef: _dynamicWorkflowCodeRef, + ...eventData + } = event.eventData; + event = { ...event, eventData }; + } // Strip only the step `input` from the lazy step_started event row: // it belongs on the synthetic step_created written above. stepName is // preserved for the client replay consumer's step-name divergence @@ -1741,6 +1755,7 @@ export function createEventsStorage( attributes?: Record; allowReservedAttributes?: true; encryptionPublicKey?: string; + dynamicWorkflowCode?: SerializedData; }; validateAttributeChanges( Object.entries(runData.attributes ?? {}).map(([key, value]) => ({ @@ -1766,6 +1781,13 @@ export function createEventsStorage( completedAt: undefined, attributes: runData.attributes ?? {}, encryptionPublicKey: runData.encryptionPublicKey, + // A dynamic run's own workflow code. Stored on the run record + // rather than the event because the run is what replay reads; + // there is no ref layer here, so the bytes sit inline in the + // record's JSON. `start()` never sends a ref against this world — + // it has no upload path to send one to — so the inline field is + // the only shape to handle. + dynamicWorkflowCode: runData.dynamicWorkflowCode, createdAt: now, updatedAt: now, }; @@ -1829,6 +1851,12 @@ export function createEventsStorage( updatedAt: now, attributes: currentRun.attributes, encryptionPublicKey: currentRun.encryptionPublicKey, + // Carried through every transition for the same reason the + // public key is: these rebuild the record field by field, so + // anything not named here is dropped the first time the run + // changes status — and a dynamic run that loses its code can + // never be replayed. + dynamicWorkflowCode: currentRun.dynamicWorkflowCode, } ); run = written.run; @@ -1859,6 +1887,12 @@ export function createEventsStorage( updatedAt: now, attributes: currentRun.attributes, encryptionPublicKey: currentRun.encryptionPublicKey, + // Carried through every transition for the same reason the + // public key is: these rebuild the record field by field, so + // anything not named here is dropped the first time the run + // changes status — and a dynamic run that loses its code can + // never be replayed. + dynamicWorkflowCode: currentRun.dynamicWorkflowCode, } ); run = written.run; @@ -1900,6 +1934,12 @@ export function createEventsStorage( updatedAt: now, attributes: currentRun.attributes, encryptionPublicKey: currentRun.encryptionPublicKey, + // Carried through every transition for the same reason the + // public key is: these rebuild the record field by field, so + // anything not named here is dropped the first time the run + // changes status — and a dynamic run that loses its code can + // never be replayed. + dynamicWorkflowCode: currentRun.dynamicWorkflowCode, } ); run = written.run; @@ -1933,6 +1973,12 @@ export function createEventsStorage( updatedAt: now, attributes: currentRun.attributes, encryptionPublicKey: currentRun.encryptionPublicKey, + // Carried through every transition for the same reason the + // public key is: these rebuild the record field by field, so + // anything not named here is dropped the first time the run + // changes status — and a dynamic run that loses its code can + // never be replayed. + dynamicWorkflowCode: currentRun.dynamicWorkflowCode, } ); run = written.run; diff --git a/packages/world-local/src/storage/filters.ts b/packages/world-local/src/storage/filters.ts index 49ea5fe020..a866ee661c 100644 --- a/packages/world-local/src/storage/filters.ts +++ b/packages/world-local/src/storage/filters.ts @@ -10,7 +10,8 @@ export { stripEventDataRefs } from '@workflow/world'; /** * Filter run data based on resolveData setting. - * When resolveData is 'none', strips input/output to reduce payload size. + * When resolveData is 'none', strips input/output and a dynamic run's stored + * workflow code. */ export function filterRunData( run: WorkflowRun, @@ -29,8 +30,9 @@ export function filterRunData( resolveData: 'none' | 'all' ): WorkflowRun | WorkflowRunWithoutData { if (resolveData === 'none') { + const { dynamicWorkflowCode: _dynamicWorkflowCode, ...rest } = run; return { - ...run, + ...rest, input: undefined, output: undefined, } as WorkflowRunWithoutData; diff --git a/packages/world-local/src/storage/run-retention.test.ts b/packages/world-local/src/storage/run-retention.test.ts index f845a8c0e3..3182a8fad9 100644 --- a/packages/world-local/src/storage/run-retention.test.ts +++ b/packages/world-local/src/storage/run-retention.test.ts @@ -43,10 +43,12 @@ describe('run retention (world-local)', () => { const STEP_INPUT = new Uint8Array([7, 7, 7]); const STEP_OUTPUT = new Uint8Array([8, 8, 8]); const HOOK_METADATA = new Uint8Array([9, 9, 9]); + const DYNAMIC_CODE = new Uint8Array([10, 10, 10]); /** A run carrying `attributes`, one completed step, and one stream. */ async function startRun( - attributes?: Record + attributes?: Record, + extraRunData: Record = {} ): Promise { const created = await storage.events.create(null, { eventType: 'run_created', @@ -56,6 +58,7 @@ describe('run retention (world-local)', () => { workflowName: 'retention-workflow', input: RUN_INPUT, ...(attributes ? { attributes, allowReservedAttributes: true } : {}), + ...extraRunData, }, }); const run = created.run; @@ -129,6 +132,31 @@ describe('run retention (world-local)', () => { expect(expiredAt?.getTime()).toBeLessThanOrEqual(Date.now()); }); + it("drops a dynamic run's stored workflow code", async () => { + // The code is application source stored on the run record, as much + // user data as the input it ran on. + const run = await startRun( + { [RETENTION_ATTRIBUTE]: '0' }, + { dynamicWorkflowCode: DYNAMIC_CODE } + ); + expect((await storage.runs.get(run.runId)).dynamicWorkflowCode).toEqual( + DYNAMIC_CODE + ); + const createdEvent = ( + await storage.events.list({ runId: run.runId, pagination: {} }) + ).data.find((event) => event.eventType === 'run_created'); + expect( + (createdEvent as { eventData?: Record }).eventData + ?.dynamicWorkflowCode + ).toBeUndefined(); + + await complete(run.runId); + + const persisted = await storage.runs.get(run.runId); + expect(persisted.dynamicWorkflowCode).toBeUndefined(); + expect(persisted.expiredAt).toBeInstanceOf(Date); + }); + it('keeps the run listable, with its metadata intact', async () => { const run = await startRun({ [RETENTION_ATTRIBUTE]: '0' }); await complete(run.runId); diff --git a/packages/world-local/src/storage/run-retention.ts b/packages/world-local/src/storage/run-retention.ts index ac77bbd0c8..3127f5c060 100644 --- a/packages/world-local/src/storage/run-retention.ts +++ b/packages/world-local/src/storage/run-retention.ts @@ -56,6 +56,10 @@ export function withRunPayloadsPurged( input: undefined, output: undefined, error: undefined, + // A dynamic run's stored workflow code is application source, and as + // much user data as the input it ran on. The run is terminal, so nothing + // replays it again. + dynamicWorkflowCode: undefined, expiredAt: purgedAt, }; } @@ -96,6 +100,13 @@ export async function purgeRunEntityData( for (const field of getEventDataRefFields(String(event.eventType))) { delete (eventData as Record)[field]; } + if ( + event.eventType === 'run_created' || + event.eventType === 'run_started' + ) { + delete (eventData as Record).dynamicWorkflowCode; + delete (eventData as Record).dynamicWorkflowCodeRef; + } }), scrubHookMetadata(basedir, runId), purgeRunStreamData(basedir, runId, tag), diff --git a/packages/world-local/src/storage/runs-storage.test.ts b/packages/world-local/src/storage/runs-storage.test.ts index fe06a0dc32..bbb70839b5 100644 --- a/packages/world-local/src/storage/runs-storage.test.ts +++ b/packages/world-local/src/storage/runs-storage.test.ts @@ -6,6 +6,7 @@ import { ATTRIBUTE_KEY_MAX_LENGTH, AttributeValidationError, RESERVED_ATTRIBUTE_KEY_PREFIX, + SPEC_VERSION_CURRENT, type Storage, } from '@workflow/world'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; @@ -232,3 +233,59 @@ describe('runs.experimentalSetAttributes (world-local)', () => { expect(Object.keys(refreshed.attributes ?? {})).toHaveLength(20); }); }); + +describe("runs resolveData: 'none' (world-local)", () => { + let testDir: string; + let storage: Storage; + const DYNAMIC_CODE = new Uint8Array([10, 10, 10]); + + beforeEach(async () => { + testDir = await fs.mkdtemp(path.join(os.tmpdir(), 'resolve-none-test-')); + storage = createStorage(testDir); + }); + + afterEach(async () => { + await fs.rm(testDir, { recursive: true, force: true }); + }); + + async function dynamicRun() { + const created = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: + 'workflow//dynamic/0123456789abcdef0123456789abcdef//workflow', + input: new Uint8Array([1]), + dynamicWorkflowCode: DYNAMIC_CODE, + }, + }); + if (!created.run) throw new Error('Expected run to be created'); + return created.run; + } + + it("omits a dynamic run's stored code from get, getMany, and list", async () => { + const run = await dynamicRun(); + + const got = await storage.runs.get(run.runId, { resolveData: 'none' }); + expect(got).not.toHaveProperty('dynamicWorkflowCode'); + expect(got.input).toBeUndefined(); + + const [many] = await storage.runs.getMany!([run.runId], { + resolveData: 'none', + }); + expect(many).not.toHaveProperty('dynamicWorkflowCode'); + + const listed = await storage.runs.list({ resolveData: 'none' }); + const found = listed.data.find((r) => r.runId === run.runId); + expect(found).toBeDefined(); + expect(found).not.toHaveProperty('dynamicWorkflowCode'); + }); + + it("returns a dynamic run's stored code with resolveData: 'all'", async () => { + const run = await dynamicRun(); + + const got = await storage.runs.get(run.runId, { resolveData: 'all' }); + expect(got.dynamicWorkflowCode).toEqual(DYNAMIC_CODE); + }); +}); diff --git a/packages/world-local/src/storage/runs-storage.ts b/packages/world-local/src/storage/runs-storage.ts index 91713b5939..09e3a9720f 100644 --- a/packages/world-local/src/storage/runs-storage.ts +++ b/packages/world-local/src/storage/runs-storage.ts @@ -218,15 +218,10 @@ export function createRunsStorage( getId: (run) => run.runId, }); - // If resolveData is "none", replace input/output with undefined if (resolveData === 'none') { return { ...result, - data: result.data.map((run) => ({ - ...run, - input: undefined, - output: undefined, - })) as WorkflowRunWithoutData[], + data: result.data.map((run) => filterRunData(run, 'none')), }; } diff --git a/packages/world-postgres/src/drizzle/migrations/0024_add_dynamic_workflow_code.sql b/packages/world-postgres/src/drizzle/migrations/0024_add_dynamic_workflow_code.sql new file mode 100644 index 0000000000..81b4034ed2 --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0024_add_dynamic_workflow_code.sql @@ -0,0 +1 @@ +ALTER TABLE "workflow"."workflow_runs" ADD COLUMN "dynamic_workflow_code_cbor" bytea; diff --git a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json index 43f706432c..09143309f7 100644 --- a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json +++ b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json @@ -169,6 +169,13 @@ "when": 1789689600000, "tag": "0023_add_hook_claimed_from", "breakpoints": true + }, + { + "idx": 24, + "version": "7", + "when": 1789776000000, + "tag": "0024_add_dynamic_workflow_code", + "breakpoints": true } ] } diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 23e0a76541..d7c5e4647e 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -114,6 +114,17 @@ export const runs = schema.table( * older SDKs, which fall back to the symmetric path. */ encryptionPublicKey: varchar('encryption_public_key'), + /** + * A dynamic run's own workflow VM code, serialized through the same + * pipeline as the run's input (compressed, then encrypted with the run's + * key). Set only on runs started from source rather than from a workflow + * function in the deployment's build-time manifest — the code is nowhere + * else, so every replay reads it back from here. + * + * Write-once at `run_created`: replay must execute the same code the run + * started on. Null on every static run. + */ + dynamicWorkflowCode: Cbor()('dynamic_workflow_code_cbor'), createdAt: timestamp('created_at').defaultNow().notNull(), updatedAt: timestamp('updated_at') .defaultNow() @@ -126,7 +137,7 @@ export const runs = schema.table( Cborized< Omit & { input?: unknown }, 'input' | 'output' | 'executionContext' | 'error' - > + > & { dynamicWorkflowCode?: SerializedData } >, (tb) => [index().on(tb.workflowName), index().on(tb.status)] ); diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 7ba9f7245a..f250e62d95 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -75,6 +75,9 @@ export function createWorld( return { specVersion: mintedSpecVersion(), + getBackendCapabilities: async () => ({ + dynamicWorkflowStorageVersion: 1, + }), capabilities: { hookRetention: { active: true }, hookResumeDedup: true, diff --git a/packages/world-postgres/src/retention.ts b/packages/world-postgres/src/retention.ts index 1128107203..2fb6ce9957 100644 --- a/packages/world-postgres/src/retention.ts +++ b/packages/world-postgres/src/retention.ts @@ -72,6 +72,9 @@ export async function purgeRunUserData( outputJson: NULL, error: NULL, errorJson: NULL, + // A dynamic run's stored workflow code is application source, and + // as much user data as the input it ran on. + dynamicWorkflowCode: NULL, }) .where(eq(runs.runId, runId)); diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 18fb776315..c6a3082d2f 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -1152,7 +1152,14 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { attributes?: Record; allowReservedAttributes?: true; encryptionPublicKey?: string; + dynamicWorkflowCode?: SerializedData; + dynamicWorkflowCodeRef?: string; }; + if (runInputData.dynamicWorkflowCodeRef !== undefined) { + throw new Error( + 'Postgres World does not support deferred dynamic workflow code refs' + ); + } if ( runInputData.deploymentId && runInputData.workflowName && @@ -1188,6 +1195,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // run from the queued message, which is exactly when the key // would otherwise be lost for the rest of the run's life. encryptionPublicKey: runInputData.encryptionPublicKey, + dynamicWorkflowCode: runInputData.dynamicWorkflowCode, status: 'pending', }) .onConflictDoNothing() @@ -1489,7 +1497,14 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { attributes?: Record; allowReservedAttributes?: true; encryptionPublicKey?: string; + dynamicWorkflowCode?: SerializedData; + dynamicWorkflowCodeRef?: string; }; + if (eventData.dynamicWorkflowCodeRef !== undefined) { + throw new Error( + 'Postgres World does not support deferred dynamic workflow code refs' + ); + } validateAttributeChanges( Object.entries(eventData.attributes ?? {}).map(([key, value]) => ({ key, @@ -1516,6 +1531,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { | undefined, attributes: eventData.attributes, encryptionPublicKey: eventData.encryptionPublicKey, + dynamicWorkflowCode: eventData.dynamicWorkflowCode, status: 'pending', }) .onConflictDoNothing() @@ -1534,12 +1550,17 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // first allocation, is what makes "no row" mean "created before slots // existed" for the rest of the run's life. const firstEventId = await openEventSlots(tx, effectiveRunId); + const { + dynamicWorkflowCode: _dynamicWorkflowCode, + dynamicWorkflowCodeRef: _dynamicWorkflowCodeRef, + ...storedEventData + } = eventData; const eventValue = await insertEventRow(tx, { runId: effectiveRunId, eventId: firstEventId, correlationId: data.correlationId, eventType: 'run_created', - eventData, + eventData: storedEventData, specVersion: effectiveSpecVersion, }); if (!eventValue) { @@ -3301,7 +3322,12 @@ function filterRunData( resolveData: ResolveData ): WorkflowRun | WorkflowRunWithoutData { if (resolveData === 'none') { - const { input: _, output: __, ...rest } = run; + const { + input: _, + output: __, + dynamicWorkflowCode: _dynamicWorkflowCode, + ...rest + } = run; return { input: undefined, output: undefined, ...rest }; } diff --git a/packages/world-postgres/test/retention.test.ts b/packages/world-postgres/test/retention.test.ts index 31a8545eb4..d7cc6071ea 100644 --- a/packages/world-postgres/test/retention.test.ts +++ b/packages/world-postgres/test/retention.test.ts @@ -20,7 +20,15 @@ type EventsStorage = ReturnType; /** Every payload-bearing column, CBOR half and legacy JSON twin alike. */ const PAYLOAD_COLUMNS = { - runs: ['input_cbor', 'input', 'output_cbor', 'output', 'error_cbor', 'error'], + runs: [ + 'input_cbor', + 'input', + 'output_cbor', + 'output', + 'error_cbor', + 'error', + 'dynamic_workflow_code_cbor', + ], steps: [ 'input_cbor', 'input', @@ -97,6 +105,9 @@ describe('Retention ($retention: 0)', () => { workflowName: 'retention-workflow', input: new Uint8Array([1, 2, 3]), executionContext: { userId: 'user-1' }, + // A dynamic run's stored code is a run payload too, and the only one + // that lives in a column of its own. + dynamicWorkflowCode: new Uint8Array([14, 15, 16]), ...(attributes ? { attributes, allowReservedAttributes: true } : {}), }, }); diff --git a/packages/world-postgres/test/storage.test.ts b/packages/world-postgres/test/storage.test.ts index 794c89b69e..7f38cd90d9 100644 --- a/packages/world-postgres/test/storage.test.ts +++ b/packages/world-postgres/test/storage.test.ts @@ -47,6 +47,7 @@ async function createRun( input: Uint8Array; executionContext?: Record; attributes?: Record; + dynamicWorkflowCode?: Uint8Array; } ): Promise { const result = await events.create(null, { @@ -320,6 +321,31 @@ describe('Storage (Postgres integration)', () => { name: 'WorkflowRunNotFoundError', }); }); + + it("omits a dynamic run's stored code with resolveData: 'none'", async () => { + const code = new Uint8Array([10, 10, 10]); + const created = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: + 'workflow//dynamic/0123456789abcdef0123456789abcdef//workflow', + input: new Uint8Array([1]), + dynamicWorkflowCode: code, + }); + + const none = await runs.get(created.runId, { resolveData: 'none' }); + expect(none).not.toHaveProperty('dynamicWorkflowCode'); + const [many] = await runs.getMany!([created.runId], { + resolveData: 'none', + }); + expect(many).not.toHaveProperty('dynamicWorkflowCode'); + const listed = await runs.list({ resolveData: 'none' }); + expect( + listed.data.find((run) => run.runId === created.runId) + ).not.toHaveProperty('dynamicWorkflowCode'); + + const all = await runs.get(created.runId, { resolveData: 'all' }); + expect(all.dynamicWorkflowCode).toEqual(code); + }); }); describe('getMany', () => { diff --git a/packages/world-vercel/src/backend-capabilities.test.ts b/packages/world-vercel/src/backend-capabilities.test.ts new file mode 100644 index 0000000000..6499497c61 --- /dev/null +++ b/packages/world-vercel/src/backend-capabilities.test.ts @@ -0,0 +1,55 @@ +import { MockAgent } from 'undici'; +import { describe, expect, it } from 'vitest'; +import { createGetBackendCapabilities } from './backend-capabilities.js'; +import { WORKFLOW_SERVER_URL_OVERRIDE } from './utils.js'; + +const ORIGIN = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; +const PATH = '/api/v2/capabilities'; + +function agentReplying(status: number, body?: unknown) { + const agent = new MockAgent(); + agent.disableNetConnect(); + agent + .get(ORIGIN) + .intercept({ path: PATH, method: 'GET' }) + .reply(status, body ?? '') + .persist(); + return agent; +} + +describe('createGetBackendCapabilities', () => { + it('returns the advertised capabilities', async () => { + const agent = agentReplying(200, { dynamicWorkflowStorageVersion: 1 }); + const getCapabilities = createGetBackendCapabilities({ + token: 'test-token', + dispatcher: agent, + }); + + await expect(getCapabilities()).resolves.toEqual({ + dynamicWorkflowStorageVersion: 1, + }); + }); + + it('maps a 404 from a backend without the route to an empty set', async () => { + const agent = agentReplying(404, { error: 'Not Found' }); + const getCapabilities = createGetBackendCapabilities({ + token: 'test-token', + dispatcher: agent, + }); + + await expect(getCapabilities()).resolves.toEqual({}); + }); + + it('propagates a server error rather than reading it as no capabilities', async () => { + const agent = agentReplying(503, { error: 'Service Unavailable' }); + const getCapabilities = createGetBackendCapabilities({ + token: 'test-token', + dispatcher: agent, + }); + + await expect(getCapabilities()).rejects.toMatchObject({ + name: 'WorkflowWorldError', + status: 503, + }); + }); +}); diff --git a/packages/world-vercel/src/backend-capabilities.ts b/packages/world-vercel/src/backend-capabilities.ts new file mode 100644 index 0000000000..efedeaf340 --- /dev/null +++ b/packages/world-vercel/src/backend-capabilities.ts @@ -0,0 +1,38 @@ +import { WorkflowWorldError } from '@workflow/errors'; +import type { BackendCapabilities } from '@workflow/world'; +import z from 'zod'; +import type { APIConfig } from './utils.js'; +import { makeRequest } from './utils.js'; + +const BackendCapabilitiesSchema = z.compile( + z.object({ + dynamicWorkflowStorageVersion: z.number().optional(), + }) +); + +/** + * Reads the backend's advertised capabilities. + * + * A backend that predates `/v2/capabilities` answers 404, which means it + * advertises nothing: that maps to an empty capability set, so callers fail + * closed with their own "backend does not support" error. Any other failure + * (5xx, transport) propagates, since it says nothing about what the backend + * supports. + */ +export function createGetBackendCapabilities(config?: APIConfig) { + return async (): Promise => { + try { + return await makeRequest({ + endpoint: '/v2/capabilities', + options: { method: 'GET' }, + config, + schema: BackendCapabilitiesSchema, + }); + } catch (error) { + if (WorkflowWorldError.is(error) && error.status === 404) { + return {}; + } + throw error; + } + }; +} diff --git a/packages/world-vercel/src/dynamic-code.ts b/packages/world-vercel/src/dynamic-code.ts new file mode 100644 index 0000000000..bded837221 --- /dev/null +++ b/packages/world-vercel/src/dynamic-code.ts @@ -0,0 +1,71 @@ +/** + * Deferred upload of a dynamic run's workflow VM code. + * + * A dynamic run carries its own workflow code, because that code is not in the + * deployment's build-time manifest. Small definitions ride the `run_created` + * frame's meta block and cost no extra round-trip — that is the common case + * and does not come through here. This module is the escape hatch for the + * tail: a definition too large to send inline is streamed to the backend + * first, and the returned ref key is what `run_created` carries instead of the + * bytes. + * + * The upload necessarily precedes the run it belongs to (the ref has to exist + * before the write that references it), so the backend accepts a runId that + * does not exist yet and authorizes on tenant. See + * `World.uploadDynamicWorkflowCode` for the contract. + */ + +import { z } from 'zod'; +import { getEventsDispatcher } from './http-client.js'; +import { instrumentedFetch } from './http-core.js'; +import { type APIConfig, getHttpConfig } from './utils.js'; + +const UploadResponseSchema = z.object({ + /** Ref key to send as `run_created`'s `eventData.dynamicWorkflowCodeRef`. */ + ref: z.string().min(1), + /** Stored byte count, echoed back for observability. */ + byteSize: z.number().int().nonnegative().optional(), +}); + +/** + * Stream serialized workflow code to the backend and return its ref key. + * + * @param runId - Client-minted ID of the run being started. The backend + * embeds it in the storage key so the object is reclaimed with the run. + * @param params.workflowName - The generated dynamic workflow name. The + * backend accepts it for compatibility but stores uploads in a reserved + * run-scoped staging namespace. + * @param params.code - Serialized (compressed + encrypted) workflow code. + */ +export async function uploadDynamicWorkflowCode( + runId: string, + params: { workflowName: string; code: Uint8Array }, + config?: APIConfig +): Promise { + const { baseUrl, headers } = await getHttpConfig(config); + const url = + `${baseUrl}/v4/runs/${encodeURIComponent(runId)}/dynamic-code` + + `?workflowName=${encodeURIComponent(params.workflowName)}`; + + const requestHeaders = new Headers(headers); + // The body is opaque bytes the backend streams straight to blob storage + // without decoding, so it is neither CBOR nor JSON on the way up. + requestHeaders.set('Content-Type', 'application/octet-stream'); + requestHeaders.set('Accept', 'application/json'); + requestHeaders.set('Content-Length', String(params.code.byteLength)); + + const opName = 'uploadDynamicWorkflowCode'; + const response = await instrumentedFetch({ + method: 'POST', + url, + headers: requestHeaders, + body: params.code, + dispatcher: getEventsDispatcher(config), + logLabel: opName, + // No buildError override: the upload's error bodies are the standard + // JSON `{ error, message }` envelope, which instrumentedFetch's default + // path already turns into the right typed error. + }); + + return UploadResponseSchema.parse(await response.json()).ref; +} diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index fe22f21a2f..ec085bbb40 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -300,6 +300,14 @@ interface CreateEventV4InputBase { * the run entity so cross-run writers can seal to it without holding the * run's symmetric key. */ encryptionPublicKey?: string; + /** A dynamic run's serialized workflow VM code, inline on run_created (and + * run_started for resilient start). Rides the frame meta as a CBOR byte + * string — the body slot on those events already carries the run's input. + * The backend stores it behind a ref on the run and never decodes it. */ + dynamicWorkflowCode?: Uint8Array; + /** Ref key of dynamic workflow code uploaded ahead of this write, for + * definitions too large to ride the meta inline. */ + dynamicWorkflowCodeRef?: string; /** Client-measured time-to-first-step ms, riding on the run's first * step_completed / step_failed. Consumed server-side for latency * metrics; not read back. */ @@ -648,6 +656,12 @@ function buildPostFrameMeta( if (input.encryptionPublicKey !== undefined) { meta.encryptionPublicKey = input.encryptionPublicKey; } + if (input.dynamicWorkflowCode !== undefined) { + meta.dynamicWorkflowCode = input.dynamicWorkflowCode; + } + if (input.dynamicWorkflowCodeRef !== undefined) { + meta.dynamicWorkflowCodeRef = input.dynamicWorkflowCodeRef; + } if (input.ttfs !== undefined) meta.ttfs = input.ttfs; if (input.stso !== undefined) meta.stso = input.stso; if (input.stepCount !== undefined) meta.stepCount = input.stepCount; diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 988ca2b51d..746c4d2bc4 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -159,6 +159,16 @@ interface SplitEventData { * it without holding the run's symmetric key. */ encryptionPublicKey?: string; + /** + * A dynamic run's serialized workflow VM code, inline on run_created (and + * on run_started for resilient start). Rides the meta rather than the + * frame body because the body slot already carries the run's `input`; the + * backend stores it behind a ref on the run and never decodes it. + */ + dynamicWorkflowCode?: Uint8Array; + /** Ref key of dynamic workflow code uploaded ahead of the write, for + * definitions too large to send inline. */ + dynamicWorkflowCodeRef?: string; /** Client-measured time-to-first-step ms (step_completed / step_failed). */ ttfs?: number; /** Client-measured step-to-step overhead ms (step_completed / step_failed). */ @@ -213,6 +223,8 @@ type MetaSourceField = | 'writer' | 'allowReservedAttributes' | 'encryptionPublicKey' + | 'dynamicWorkflowCode' + | 'dynamicWorkflowCodeRef' | 'ttfs' | 'stso' | 'stepCount' @@ -383,6 +395,16 @@ export function splitEventDataForV4(data: AnyEventRequest): SplitEventData { if (typeof eventData.encryptionPublicKey === 'string') { meta.encryptionPublicKey = eventData.encryptionPublicKey; } + // Dynamic workflow code arrives one of two ways and never both: the bytes + // inline for a small definition, or a ref to an earlier upload for a large + // one. Both are metadata as far as the frame is concerned — the single body + // slot on run_created/run_started is the run's input. + if (eventData.dynamicWorkflowCode instanceof Uint8Array) { + meta.dynamicWorkflowCode = eventData.dynamicWorkflowCode; + } + if (typeof eventData.dynamicWorkflowCodeRef === 'string') { + meta.dynamicWorkflowCodeRef = eventData.dynamicWorkflowCodeRef; + } // Client-measured latency telemetry on step terminal events (TTFS / STSO). // The server consumes these for metrics; they are not read back. if (typeof eventData.ttfs === 'number') { diff --git a/packages/world-vercel/src/execution-context.test.ts b/packages/world-vercel/src/execution-context.test.ts new file mode 100644 index 0000000000..6dd7b7de64 --- /dev/null +++ b/packages/world-vercel/src/execution-context.test.ts @@ -0,0 +1,42 @@ +import { WorkflowRuntimeError } from '@workflow/errors'; +import { describe, expect, it } from 'vitest'; +import { + MAX_EXECUTION_CONTEXT_BYTES, + validateRunExecutionContext, +} from './execution-context.js'; + +describe('validateRunExecutionContext', () => { + function contextAtSize(bytes: number): Record { + const emptyBytes = new TextEncoder().encode( + JSON.stringify({ value: '' }) + ).byteLength; + return { value: 'x'.repeat(bytes - emptyBytes) }; + } + + it('accepts exactly 2048 JSON UTF-8 bytes', () => { + expect(() => + validateRunExecutionContext(contextAtSize(MAX_EXECUTION_CONTEXT_BYTES)) + ).not.toThrow(); + }); + + it('rejects 2049 JSON UTF-8 bytes with a typed error naming the measured size', () => { + let error: unknown; + try { + validateRunExecutionContext( + contextAtSize(MAX_EXECUTION_CONTEXT_BYTES + 1) + ); + } catch (err) { + error = err; + } + expect(WorkflowRuntimeError.is(error)).toBe(true); + expect((error as Error).message).toMatch( + /Dynamic workflow execution context is 2049 bytes.*2048-byte limit.*experimental_dynamic\.steps/ + ); + }); + + it('rejects a context that is not JSON serializable with a typed error', () => { + expect(() => validateRunExecutionContext({ value: 1n })).toThrow( + WorkflowRuntimeError + ); + }); +}); diff --git a/packages/world-vercel/src/execution-context.ts b/packages/world-vercel/src/execution-context.ts new file mode 100644 index 0000000000..b5c8c7c90f --- /dev/null +++ b/packages/world-vercel/src/execution-context.ts @@ -0,0 +1,30 @@ +import { WorkflowRuntimeError } from '@workflow/errors'; + +export const MAX_EXECUTION_CONTEXT_BYTES = 2048; + +/** + * Checks a dynamic run's execution context against the backend's + * `MAX_EXECUTION_CONTEXT_BYTES` limit on its JSON encoding, before `start()` + * writes anything. The `dynamicWorkflow` marker's alias → step id map is what + * grows with the run, so the limit caps how many steps a dynamic run can bind. + */ +export function validateRunExecutionContext( + value: Record +): void { + let json: string; + try { + json = JSON.stringify(value); + } catch (error) { + throw new WorkflowRuntimeError( + 'Dynamic workflow execution context must be JSON serializable.', + { cause: error } + ); + } + + const bytes = new TextEncoder().encode(json).byteLength; + if (bytes > MAX_EXECUTION_CONTEXT_BYTES) { + throw new WorkflowRuntimeError( + `Dynamic workflow execution context is ${bytes} bytes, exceeding the ${MAX_EXECUTION_CONTEXT_BYTES}-byte limit, so no run was created. Bind fewer steps through \`experimental_dynamic.steps\` or use shorter aliases.` + ); + } +} diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index 3d002b22db..bee3561674 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -1,8 +1,11 @@ import type { World } from '@workflow/world'; import { mintedSpecVersion } from '@workflow/world'; import { createAnalytics } from './analytics.js'; +import { createGetBackendCapabilities } from './backend-capabilities.js'; import { createRunId, describeRun } from './create-run-id.js'; +import { uploadDynamicWorkflowCode } from './dynamic-code.js'; import { createGetEncryptionKeyForRun } from './encryption.js'; +import { validateRunExecutionContext } from './execution-context.js'; import { getDeadline } from './get-deadline.js'; import { instrumentObject } from './instrumentObject.js'; import { createQueue } from './queue.js'; @@ -12,12 +15,18 @@ import { createStreamer } from './streamer.js'; import { type APIConfig, resolveClientEnvironment } from './utils.js'; export { createAnalytics } from './analytics.js'; +export { createGetBackendCapabilities } from './backend-capabilities.js'; export { createRunId, describeRun, regionForRunId } from './create-run-id.js'; +export { uploadDynamicWorkflowCode } from './dynamic-code.js'; export { createGetEncryptionKeyForRun, deriveRunKey, fetchRunKey, } from './encryption.js'; +export { + MAX_EXECUTION_CONTEXT_BYTES, + validateRunExecutionContext, +} from './execution-context.js'; export { createQueue } from './queue.js'; export { createStorage } from './storage.js'; export { createStreamer } from './streamer.js'; @@ -78,6 +87,8 @@ export function createWorld(config?: APIConfig): World { // rollback or kill switch drop new resumes to the sequential path // immediately, without a redeploy of this adapter. }, + getBackendCapabilities: createGetBackendCapabilities(config), + validateRunExecutionContext, getRuntimeDeadline: getDeadline, ...createQueue(config), ...createStorage(config), @@ -96,6 +107,10 @@ export function createWorld(config?: APIConfig): World { // stamp it into the queue message and the consuming deployment can detect // that it was handed a run created against a different environment. getEnvironment: () => resolveClientEnvironment(config), + // Deferred storage for a dynamic run's workflow code, used only when the + // definition is too large to ride the `run_created` frame inline. + uploadDynamicWorkflowCode: (runId, params) => + uploadDynamicWorkflowCode(runId, params, config), getEncryptionKeyForRun: createGetEncryptionKeyForRun( projectId, config?.projectConfig?.teamId, diff --git a/packages/world-vercel/src/runs.test.ts b/packages/world-vercel/src/runs.test.ts index 087b6adf64..3988fc002f 100644 --- a/packages/world-vercel/src/runs.test.ts +++ b/packages/world-vercel/src/runs.test.ts @@ -3,6 +3,7 @@ import { MockAgent } from 'undici'; import { describe, expect, it } from 'vitest'; import { cancelWorkflowRuns, + getWorkflowRun, getWorkflowRuns, listWorkflowRuns, } from './runs.js'; @@ -71,6 +72,117 @@ describe('getWorkflowRuns', () => { }); }); +describe('run reads of dynamic workflow code', () => { + const code = new Uint8Array([10, 10, 10]); + /** A gzip-prefixed payload that throws if anything tries to decompress it. */ + const corruptCompressedCode = new Uint8Array([ + ...new TextEncoder().encode('gzip'), + 1, + 2, + 3, + ]); + + function dynamicRun(dynamicWorkflowCode: Uint8Array) { + return { + runId: 'wrun_dynamic', + status: 'running', + deploymentId: 'dpl_1', + workflowName: + 'workflow//dynamic/0123456789abcdef0123456789abcdef//workflow', + dynamicWorkflowCode, + createdAt: new Date('2026-01-01T00:00:00.000Z'), + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + }; + } + + function agentReturningRun( + remoteRefBehavior: 'lazy' | 'resolve', + dynamicWorkflowCode = code + ) { + const agent = new MockAgent(); + agent.disableNetConnect(); + agent + .get(ORIGIN) + .intercept({ + path: `/api/v2/runs/wrun_dynamic?remoteRefBehavior=${remoteRefBehavior}`, + method: 'GET', + }) + .reply(200, () => encode(dynamicRun(dynamicWorkflowCode)), { + headers: { 'content-type': 'application/cbor' }, + }); + return agent; + } + + it("omits the code with resolveData: 'none' without decompressing it", async () => { + const agent = agentReturningRun('lazy', corruptCompressedCode); + + const run = await getWorkflowRun( + 'wrun_dynamic', + { resolveData: 'none' }, + { token: 'test-token', dispatcher: agent } + ); + + expect(run).not.toHaveProperty('dynamicWorkflowCode'); + agent.assertNoPendingInterceptors(); + }); + + it("omits the code from a resolveData: 'none' list without decompressing it", async () => { + const agent = new MockAgent(); + agent.disableNetConnect(); + agent + .get(ORIGIN) + .intercept({ + path: '/api/v2/runs?remoteRefBehavior=lazy', + method: 'GET', + }) + .reply( + 200, + () => + encode({ + data: [dynamicRun(corruptCompressedCode)], + cursor: null, + hasMore: false, + }), + { headers: { 'content-type': 'application/cbor' } } + ); + + const runs = await listWorkflowRuns( + { resolveData: 'none' }, + { token: 'test-token', dispatcher: agent } + ); + + expect(runs.data).toHaveLength(1); + expect(runs.data[0]).not.toHaveProperty('dynamicWorkflowCode'); + agent.assertNoPendingInterceptors(); + }); + + it("classifies corrupt code on resolveData: 'all' as a contract error", async () => { + const agent = agentReturningRun('resolve', corruptCompressedCode); + + await expect( + getWorkflowRun( + 'wrun_dynamic', + { resolveData: 'all' }, + { token: 'test-token', dispatcher: agent } + ) + ).rejects.toMatchObject({ code: 'WORLD_CONTRACT_ERROR' }); + agent.assertNoPendingInterceptors(); + }); + + it("returns the code with resolveData: 'all'", async () => { + const agent = agentReturningRun('resolve'); + + const run = await getWorkflowRun( + 'wrun_dynamic', + { resolveData: 'all' }, + { token: 'test-token', dispatcher: agent } + ); + + expect(run.dynamicWorkflowCode).toEqual(code); + agent.assertNoPendingInterceptors(); + }); +}); + describe('cancelWorkflowRuns', () => { it('issues a single POST /v4/runs/cancel with the run IDs and cancelReason', async () => { const agent = new MockAgent(); diff --git a/packages/world-vercel/src/runs.ts b/packages/world-vercel/src/runs.ts index fbf9d70647..b85228a3a8 100644 --- a/packages/world-vercel/src/runs.ts +++ b/packages/world-vercel/src/runs.ts @@ -70,6 +70,9 @@ const WorkflowRunWireWithRefsSchema = z.compile( // Accept both Uint8Array (v2 format) and any (legacy v1 JSON format) input: z.union([z.instanceof(Uint8Array), z.any()]).optional(), output: z.union([z.instanceof(Uint8Array), z.any()]).optional(), + // Discarded by `filterRunData`: `resolveData: 'none'` never returns a + // dynamic run's code, whatever shape the backend sends it in. + dynamicWorkflowCode: z.any().optional(), }) ); @@ -91,7 +94,13 @@ function filterRunData( resolveData: 'none' | 'all' ): WorkflowRun | WorkflowRunWithoutData { if (resolveData === 'none') { - const { inputRef: _inputRef, outputRef: _outputRef, ...rest } = run; + // The code is dropped before normalizing, so it is never decompressed. + const { + inputRef: _inputRef, + outputRef: _outputRef, + dynamicWorkflowCode: _dynamicWorkflowCode, + ...rest + } = run; const deserialized = normalizeWorkflowRunData( deserializeError(rest) as unknown as Record ); diff --git a/packages/world-vercel/src/serialized-data.ts b/packages/world-vercel/src/serialized-data.ts index 8f17150fd4..a47e22b4dd 100644 --- a/packages/world-vercel/src/serialized-data.ts +++ b/packages/world-vercel/src/serialized-data.ts @@ -69,6 +69,22 @@ export function normalizeSerializedData(value: unknown): unknown { return decompress(format, bytes.subarray(FORMAT_PREFIX_LENGTH)); } +/** + * Corrupt stored code is a permanent contract failure rather than a transport + * failure, so delivery fails the run instead of retrying it. + */ +function normalizeDynamicWorkflowCode(value: unknown): unknown { + try { + return normalizeSerializedData(value); + } catch (cause) { + if (WorkflowWorldError.is(cause)) throw cause; + throw new WorkflowWorldError( + 'Stored dynamic workflow code could not be decompressed.', + { code: 'WORLD_CONTRACT_ERROR', cause } + ); + } +} + export function normalizeWorkflowRunData>( run: T ): T { @@ -77,6 +93,18 @@ export function normalizeWorkflowRunData>( input: normalizeSerializedData(run.input), output: normalizeSerializedData(run.output), error: normalizeSerializedData(run.error), + // A dynamic run's workflow code is a run payload like the others, so it + // can carry a compression wrapper and gets unwrapped the same way. Only + // reachable when encryption is off — with it on, the outermost prefix is + // `encr` and unwrapping happens after decryption instead. Absent stays + // absent, so a run without code does not gain the key. + ...(run.dynamicWorkflowCode !== undefined + ? { + dynamicWorkflowCode: normalizeDynamicWorkflowCode( + run.dynamicWorkflowCode + ), + } + : {}), }; } diff --git a/packages/world/src/events.test.ts b/packages/world/src/events.test.ts index 491c0d9e54..d97a498d21 100644 --- a/packages/world/src/events.test.ts +++ b/packages/world/src/events.test.ts @@ -1,6 +1,59 @@ import { describe, expect, it } from 'vitest'; import { CreateEventSchema, EventSchema } from './events'; +describe('dynamic workflow code storage shape', () => { + const inline = new Uint8Array([1, 2, 3]); + const creationData = { + deploymentId: 'dpl_1', + workflowName: 'workflow//dynamic/test//workflow', + input: new Uint8Array([4]), + }; + + for (const eventType of ['run_created', 'run_started'] as const) { + const base = { + eventType, + specVersion: 5, + eventData: eventType === 'run_created' ? creationData : {}, + }; + + it(`${eventType} accepts neither, inline only, or ref only`, () => { + expect(CreateEventSchema.safeParse(base).success).toBe(true); + expect( + CreateEventSchema.safeParse({ + ...base, + eventData: { ...base.eventData, dynamicWorkflowCode: inline }, + }).success + ).toBe(true); + expect( + CreateEventSchema.safeParse({ + ...base, + eventData: { ...base.eventData, dynamicWorkflowCodeRef: 'ref_1' }, + }).success + ).toBe(true); + }); + + it(`${eventType} rejects inline code and a ref together in request and stored schemas`, () => { + const eventData = { + ...base.eventData, + dynamicWorkflowCode: inline, + dynamicWorkflowCodeRef: 'ref_1', + }; + expect(CreateEventSchema.safeParse({ ...base, eventData }).success).toBe( + false + ); + expect( + EventSchema.safeParse({ + ...base, + eventData, + runId: 'wrun_00000000000000000000000000', + eventId: 'evnt_00000000000000000000000000', + createdAt: new Date().toISOString(), + }).success + ).toBe(false); + }); + } +}); + describe('hook_created token retention', () => { it('coerces tokenRetentionUntil to a Date', () => { const parsed = CreateEventSchema.parse({ diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index cbd150260e..02948baf94 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -564,21 +564,43 @@ const AttrSetEventSchema = z.compile( const RunCreatedEventSchema = z.compile( BaseEventSchema.extend({ eventType: z.literal('run_created'), - eventData: z.object({ - deploymentId: z.string(), - workflowName: z.string(), - input: SerializedDataSchema, - executionContext: z.record(z.string(), z.any()).optional(), - attributes: z.record(z.string(), z.string()).optional(), - allowReservedAttributes: z.literal(true).optional(), - /** - * The run's X25519 public key (base64), stamped by SDKs that support - * sealed (`encp`) envelopes. Persisted onto the run entity so that - * cross-run writers can seal payloads to this run without holding its - * symmetric key. Not secret. See `WorkflowRunBaseSchema`. - */ - encryptionPublicKey: z.string().optional(), - }), + eventData: z + .object({ + deploymentId: z.string(), + workflowName: z.string(), + input: SerializedDataSchema, + executionContext: z.record(z.string(), z.any()).optional(), + attributes: z.record(z.string(), z.string()).optional(), + allowReservedAttributes: z.literal(true).optional(), + /** + * A dynamic run's serialized workflow VM code. The World materializes it + * onto the run record and does not keep a second copy on the event. + * Mutually exclusive with `dynamicWorkflowCodeRef`. + */ + dynamicWorkflowCode: SerializedDataSchema.optional(), + /** + * Ref for dynamic workflow code uploaded before this write. Worlds must + * validate it against the caller and run before attaching it. + */ + dynamicWorkflowCodeRef: z.string().optional(), + /** + * The run's X25519 public key (base64), stamped by SDKs that support + * sealed (`encp`) envelopes. Persisted onto the run entity so that + * cross-run writers can seal payloads to this run without holding its + * symmetric key. Not secret. See `WorkflowRunBaseSchema`. + */ + encryptionPublicKey: z.string().optional(), + }) + .refine( + (value) => + value.dynamicWorkflowCode === undefined || + value.dynamicWorkflowCodeRef === undefined, + { + path: ['dynamicWorkflowCodeRef'], + message: + 'dynamicWorkflowCode and dynamicWorkflowCodeRef are mutually exclusive', + } + ), }) ); @@ -609,7 +631,20 @@ const RunStartedEventSchema = z.compile( * the run would silently lose its ability to receive sealed writes. */ encryptionPublicKey: z.string().optional(), + /** Dynamic code carried for resilient run creation. */ + dynamicWorkflowCode: SerializedDataSchema.optional(), + dynamicWorkflowCodeRef: z.string().optional(), }) + .refine( + (value) => + value.dynamicWorkflowCode === undefined || + value.dynamicWorkflowCodeRef === undefined, + { + path: ['dynamicWorkflowCodeRef'], + message: + 'dynamicWorkflowCode and dynamicWorkflowCodeRef are mutually exclusive', + } + ) .optional(), }) ); diff --git a/packages/world/src/interfaces.ts b/packages/world/src/interfaces.ts index 47234ff7c6..1789a9fd76 100644 --- a/packages/world/src/interfaces.ts +++ b/packages/world/src/interfaces.ts @@ -488,6 +488,12 @@ export interface Storage { }; } +/** Durable storage capabilities advertised by a World backend. */ +export interface BackendCapabilities { + /** Version of durable dynamic-workflow storage supported by the backend. */ + dynamicWorkflowStorageVersion?: number; +} + /** * Optional feature capabilities a World implementation declares so the core * runtime can enable optimizations that depend on backend behavior, instead @@ -643,6 +649,20 @@ export interface World extends Queue, Streamer, Storage { */ capabilities?: WorldCapabilities; + /** + * Fetches live backend capabilities. Dynamic starts require exact version + * support and fail closed when this method or its attestation is absent. + */ + getBackendCapabilities?(): Promise; + + /** + * Validates a dynamic run's complete execution context against + * World-specific limits. `start()` calls it only for dynamic starts, before + * any durable start side effect, so implementations throw to refuse the + * start. + */ + validateRunExecutionContext?(value: Record): void; + /** * Absolute wall-clock time when the current function invocation will be * terminated by the hosting platform, if known. Used to optimize runtime behavior. @@ -729,6 +749,41 @@ export interface World extends Queue, Streamer, Storage { */ createRunId?(options?: Readonly>): string; + /** + * Upload a dynamic run's serialized workflow VM code to the World's blob + * storage ahead of `run_created`, returning the ref key to attach to the + * run. + * + * The inline path — sending the bytes on `run_created` itself — is the + * common case and needs nothing from this method: a generated orchestration + * function is usually a couple of KB, and keeping it on the creating write + * costs no extra round-trip. This exists for the tail: a definition too + * large to ride the event wire, which has to be streamed separately and + * referenced. + * + * Worlds that store run records whole (local, Postgres) have no size + * pressure and leave this unset; `start()` then always sends inline, and a + * definition over its own source limit is rejected client-side rather than + * silently truncated. + * + * The upload necessarily precedes the run it belongs to, so implementations + * must accept a `runId` that does not exist yet, and must scope the stored + * object to the caller's tenant and that run so it is reclaimed with the + * run's other storage. + * + * @param runId - The client-minted ID of the run being started. + * @param params.workflowName - The run's generated dynamic workflow name. + * Worlds that embed it in the storage key need it passed in, because the + * run record does not exist yet to read it from. + * @param params.code - Serialized (compressed + encrypted) workflow code. + * @returns The ref key to send as `run_created`'s + * `eventData.dynamicWorkflowCodeRef`. + */ + uploadDynamicWorkflowCode?( + runId: string, + params: { workflowName: string; code: Uint8Array } + ): Promise; + /** * The environment this World's writes are attributed to by the backend * (`@workflow/world-vercel`: `'production' | 'preview' | 'development'`). diff --git a/packages/world/src/queue.test.ts b/packages/world/src/queue.test.ts index 795e3f67ad..e9d1bc3098 100644 --- a/packages/world/src/queue.test.ts +++ b/packages/world/src/queue.test.ts @@ -284,6 +284,32 @@ describe('RunInputSchema environment', () => { // and processes the message normally. If this ever becomes `.strict()`, every // in-flight message from a newer client starts failing validation on older // deployments. + it('accepts neither, inline code only, or a code ref only', () => { + expect(RunInputSchema.safeParse(baseRunInput).success).toBe(true); + expect( + RunInputSchema.safeParse({ + ...baseRunInput, + dynamicWorkflowCode: new Uint8Array([1]), + }).success + ).toBe(true); + expect( + RunInputSchema.safeParse({ + ...baseRunInput, + dynamicWorkflowCodeRef: 'ref_1', + }).success + ).toBe(true); + }); + + it('rejects inline code and a code ref together', () => { + expect( + RunInputSchema.safeParse({ + ...baseRunInput, + dynamicWorkflowCode: new Uint8Array([1]), + dynamicWorkflowCodeRef: 'ref_1', + }).success + ).toBe(false); + }); + it('tolerates unknown keys by stripping them, so old consumers keep working', () => { const parsed = RunInputSchema.parse({ ...baseRunInput, diff --git a/packages/world/src/queue.ts b/packages/world/src/queue.ts index 4f489054f3..d611ed6082 100644 --- a/packages/world/src/queue.ts +++ b/packages/world/src/queue.ts @@ -1,4 +1,5 @@ import { z } from 'zod/v4'; +import { SerializedDataSchema } from './serialization.js'; export type QueueKind = 'workflow'; @@ -103,41 +104,56 @@ export type TraceCarrier = z.infer; * run_started event so the server can create the run if it doesn't exist yet. */ export const RunInputSchema = z.compile( - z.object({ - input: z.unknown(), - deploymentId: z.string(), - workflowName: z.string(), - specVersion: z.number(), - executionContext: z.record(z.string(), z.any()).optional(), - /** Initial plaintext run attributes, for resilient run creation. */ - attributes: z.record(z.string(), z.string()).optional(), - /** - * Permits reserved `$`-prefixed keys in `attributes`, mirrored from the - * `start()` option so resilient run creation validates the same way as - * the original `run_created` attempt. - */ - allowReservedAttributes: z.literal(true).optional(), - /** - * The environment the creating client's writes are attributed to, as - * reported by {@link World.getEnvironment} at `start()` time (on Vercel: - * `'production' | 'preview' | 'development'`). - * - * This exists so the resilient-start path can be checked for a tenant - * mismatch. `start()` writes `run_created` under the caller's own - * credentials while pinning the queue message to a deployment, and those - * two can disagree: if the message is consumed by a deployment in a - * DIFFERENT environment, that consumer's `run_started` re-creates the run - * under ITS tenant, so the same client-minted `wrun_` id ends up existing - * in two environments: one stuck pending forever, the other executing. - * Carrying the creator's environment lets the consumer compare it against - * its own and refuse the delivery instead of forking the run. - * - * Absent for worlds with no environment dimension (local, Postgres), and - * for older SDKs. Consumers must treat it as advisory and skip the check - * when it is missing. - */ - environment: z.string().optional(), - }) + z + .object({ + input: z.unknown(), + deploymentId: z.string(), + workflowName: z.string(), + specVersion: z.number(), + executionContext: z.record(z.string(), z.any()).optional(), + /** Dynamic workflow code carried for resilient run creation. */ + dynamicWorkflowCode: SerializedDataSchema.optional(), + /** Ref for deferred dynamic workflow code; mutually exclusive with bytes. */ + dynamicWorkflowCodeRef: z.string().optional(), + /** Initial plaintext run attributes, for resilient run creation. */ + attributes: z.record(z.string(), z.string()).optional(), + /** + * Permits reserved `$`-prefixed keys in `attributes`, mirrored from the + * `start()` option so resilient run creation validates the same way as + * the original `run_created` attempt. + */ + allowReservedAttributes: z.literal(true).optional(), + /** + * The environment the creating client's writes are attributed to, as + * reported by {@link World.getEnvironment} at `start()` time (on Vercel: + * `'production' | 'preview' | 'development'`). + * + * This exists so the resilient-start path can be checked for a tenant + * mismatch. `start()` writes `run_created` under the caller's own + * credentials while pinning the queue message to a deployment, and those + * two can disagree: if the message is consumed by a deployment in a + * DIFFERENT environment, that consumer's `run_started` re-creates the run + * under ITS tenant, so the same client-minted `wrun_` id ends up existing + * in two environments: one stuck pending forever, the other executing. + * Carrying the creator's environment lets the consumer compare it against + * its own and refuse the delivery instead of forking the run. + * + * Absent for worlds with no environment dimension (local, Postgres), and + * for older SDKs. Consumers must treat it as advisory and skip the check + * when it is missing. + */ + environment: z.string().optional(), + }) + .refine( + (value) => + value.dynamicWorkflowCode === undefined || + value.dynamicWorkflowCodeRef === undefined, + { + path: ['dynamicWorkflowCodeRef'], + message: + 'dynamicWorkflowCode and dynamicWorkflowCodeRef are mutually exclusive', + } + ) ); export type RunInput = z.infer; diff --git a/packages/world/src/runs.ts b/packages/world/src/runs.ts index 9b930d418e..8175d5c1f6 100644 --- a/packages/world/src/runs.ts +++ b/packages/world/src/runs.ts @@ -63,6 +63,13 @@ export const WorkflowRunBaseSchema = z.compile( executionContext: z.record(z.string(), z.any()).optional(), input: SerializedDataSchema.optional(), output: SerializedDataSchema.optional(), + /** + * A dynamic run's workflow VM code, serialized through the run-payload + * pipeline. Compression is conditional on protocol support and benefit; + * encryption is conditional on the World supplying run key material. + * Replay reads this opaque payload from the run rather than creation events. + */ + dynamicWorkflowCode: SerializedDataSchema.optional(), /** * The thrown value from a run_failed event, serialized via the workflow * serialization pipeline. To display the error to a user, hydrate it via @@ -163,12 +170,17 @@ export type WorkflowRun = z.infer; export type StartedWorkflowRun = WorkflowRun & { startedAt: Date }; /** - * WorkflowRun with input/output fields excluded (when resolveData='none'). + * WorkflowRun with its payload fields excluded (when resolveData='none'): + * input, output, and a dynamic run's stored workflow code. * Used for listing runs without fetching the full serialized data. */ -export type WorkflowRunWithoutData = Omit & { +export type WorkflowRunWithoutData = Omit< + WorkflowRun, + 'input' | 'output' | 'dynamicWorkflowCode' +> & { input: undefined; output: undefined; + dynamicWorkflowCode?: undefined; }; // Request types diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c5111f576b..5717f07a44 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -581,6 +581,9 @@ importers: '@workflow/world-vercel': specifier: workspace:* version: link:../world-vercel + acorn: + specifier: 8.15.0 + version: 8.15.0 debug: specifier: 4.4.3 version: 4.4.3(supports-color@8.1.1) diff --git a/workbench/example/workflows/99_e2e.ts b/workbench/example/workflows/99_e2e.ts index 1dcb014205..3c78756b0a 100644 --- a/workbench/example/workflows/99_e2e.ts +++ b/workbench/example/workflows/99_e2e.ts @@ -4136,3 +4136,108 @@ export async function lifecycleHookObserver(token: string) { using hook = createHook>({ token }); return await hook; } + +////////////////////////////////////////////////////////// +// Dynamic workflows (source generated by the app at runtime) +////////////////////////////////////////////////////////// + +/** + * Stands in for orchestration an app assembles at runtime from its own + * templates. Built here rather than inlined as a constant so the source + * really is assembled per call. + * + * Only `steps`, `sleep` and `createHook` are in scope inside it; `steps.add` + * resolves to the `add` step this file already exports and deploys. + */ +function generateDynamicSource(opts: { sleepFor?: string } = {}) { + const pause = opts.sleepFor + ? ` await sleep(${JSON.stringify(opts.sleepFor)});\n` + : ''; + return ` +async function workflow(input) { + "use workflow"; + const doubled = await steps.add(input.value, input.value); +${pause} const total = await steps.add(doubled, 1); + return { total }; +} +`; +} + +/** + * Starts a dynamic run from inside the deployment. + * + * This is the shape that matters for dynamic workflows and the reason it lives + * in the app rather than in the test runner: `experimental_dynamic.steps` is given the + * *imported* `add` function, so the `.stepId` the build-time transform stamped + * on it is what binds the source to a registered step. A caller outside the + * deployment cannot do that — it has no handle on the function — and would + * have to fall back to an explicit `{ stepId }` reference. + * + * In a step rather than in workflow code so the options object (which holds a + * function reference) is never serialized across a step boundary. + */ +async function startDynamicRun(value: number, sleepFor?: string) { + 'use step'; + const run = await start(generateDynamicSource({ sleepFor }), [{ value }], { + experimental_dynamic: { steps: { add } }, + }); + return { runId: run.runId }; +} + +/** + * Generates a workflow, starts it, and hands back the child's run ID. + * + * The child is deliberately NOT awaited here: `returnValue` polls from inside + * a step that holds a worker slot until the child finishes, which is the + * deadlock hazard documented on `fibonacciWorkflow`. The e2e runner awaits the + * child itself, which also lets it read the child's stored record. + */ +export async function dynamicWorkflowFromApp(value: number) { + 'use workflow'; + const child = await startDynamicRun(value); + return { parentInput: value, childRunId: child.runId }; +} + +/** + * Same, but the generated source suspends partway through. + * + * The suspension is the point: the delivery that resumes the child holds no + * in-memory copy of its code and no run input on the message, so it has to + * read the stored code back and decrypt it. That is the path a dynamic run + * dies on if the code is not durably stored (see the `world-local` fix in + * this PR). + */ +export async function dynamicWorkflowFromAppWithSleep(value: number) { + 'use workflow'; + const child = await startDynamicRun(value, '2s'); + return { parentInput: value, childRunId: child.runId }; +} + +async function startDisallowedDynamicRun(value: number) { + 'use step'; + const run = await start( + ` +async function workflow(input) { + "use workflow"; + return await steps.notAuthorized(input.value, 1); +} +`, + [{ value }], + { experimental_dynamic: { steps: { add } } } + ); + return { runId: run.runId }; +} + +/** + * Generates source that reaches for a step it was never given. + * + * `steps` is frozen and holds only the aliases passed through + * `experimental_dynamic.steps`, so the child must fail rather than dispatch a step the app + * did not authorize. Returns the child's run ID for the runner to inspect — + * the *child* failing is the expected outcome, so the parent must not. + */ +export async function dynamicWorkflowDisallowedStep(value: number) { + 'use workflow'; + const child = await startDisallowedDynamicRun(value); + return { childRunId: child.runId }; +}