fix(build): embed the declarative evaluator worker in compiled binaries

deno.json

@@ -1,6 +1,6 @@
 {
   "name": "veryfront",
-  "version": "0.1.1202",
+  "version": "0.1.1203",
   "license": "Apache-2.0",
   "nodeModulesDir": "auto",
   "minimumDependencyAge": {

scripts/build/compile-binary.ts

@@ -7,7 +7,30 @@ import { parseArgs } from "#std/flags";
 import { fromFileUrl, isAbsolute, join } from "#std/path.ts";
 
 const PROJECT_ROOT = fromFileUrl(new URL("../..", import.meta.url));
+
+/**
+ * Worker entrypoints `deno compile` cannot discover.
+ *
+ * These are spawned through a sibling URL whose extension is computed at
+ * runtime. Compile embeds a worker only when it can statically read the
+ * specifier, so these are invisible to it and must be listed explicitly.
+ *
+ * Omitting one fails nothing that runs from source -- the file resolves from
+ * disk, so the build and every test pass. The binary then starts, serves
+ * traffic, and dies on the first request that spawns the worker. The
+ * declarative config evaluator reached production this way and crash-looped it.
+ *
+ * See PROXY_INCLUDES for why the proxy profile does not carry these.
+ */
+export const UNTRACEABLE_WORKER_INCLUDES = [
+  "src/config/declarative-evaluator-worker-entry.ts",
+  "extensions/ext-react-ssr/src/worker-renderer.ts",
+  "extensions/ext-document-kreuzberg/src/upload-extraction-worker.ts",
+  "extensions/ext-document-kreuzberg/src/native-progress-extraction-worker.ts",
+];
+
 export const DEFAULT_INCLUDES = [
+  ...UNTRACEABLE_WORKER_INCLUDES,
   "src/platform/polyfills",
   "src/proxy/main.ts",
   "src/security/sandbox/worker-script.ts",
@@ -35,21 +58,31 @@ export const DEFAULT_INCLUDES = [
   "extensions/ext-parser-babel/src/index.ts",
   "extensions/ext-parser-babel/src/parser-only.ts",
   "extensions/ext-react-ssr/src/index.ts",
-  // Resolved through a computed sibling URL at runtime, so compile cannot
-  // discover either the worker entrypoint or its embedded renderer payload.
-  "extensions/ext-react-ssr/src/worker-renderer.ts",
+  // The renderer payload the worker loads. Not an entrypoint, so it is not in
+  // UNTRACEABLE_WORKER_INCLUDES, but compile cannot discover it either.
   "extensions/ext-react-ssr/src/worker-renderer-bundle.generated.ts",
   "extensions/ext-yaml/src/index.ts",
   "extensions/ext-sandbox-shell-tools/src/index.ts",
-  // Spawned via `new Worker(new URL(...))`, which deno compile does not trace.
-  "extensions/ext-document-kreuzberg/src/upload-extraction-worker.ts",
-  "extensions/ext-document-kreuzberg/src/native-progress-extraction-worker.ts",
   "src/rendering/rsc",
   "src/utils/clsx.ts",
   "dist/framework-src",
 ];
 
 export const PROXY_INCLUDES = [
+  // Deliberately omits UNTRACEABLE_WORKER_INCLUDES, on build grounds only.
+  // Adding them fails the compile outright: the worker entry's graph wants
+  // @babel/types@7.29.8 plus @babel/helper-string-parser and
+  // @babel/helper-validator-identifier, while proxy-deno.lock pins
+  // @babel/types@7.29.0, and --frozen refuses to update the lock.
+  //
+  // This is NOT evidence that the proxy is safe. The lock already carries a
+  // babel parse toolchain (parser, generator, traverse, types), so "the deps
+  // are absent, therefore the worker never runs here" does not follow --
+  // the conflict is a version skew, not an absence. `cli/proxy-main.ts` does
+  // pull the evaluator's runner into its graph, so whether the proxy can reach
+  // a spawn at runtime is an open question, tracked in veryfront-issue-inbox#382.
+  // If it can, this list and the proxy lock have to be regenerated together.
+  //
   // The proxy runtime is loaded after provider activation. Providers are
   // statically referenced by cli/proxy-main.ts so --include does not embed the
   // workspace file tree for each extension.
@@ -109,6 +142,101 @@ export function createCompileArgs(options: CompileBinaryOptions): string[] {
   return args;
 }
 
+/**
+ * Names the worker entries a compiled binary is missing.
+ *
+ * Every include-list check answers "did we ask for this?", which the binary
+ * that crash-looped production would have passed -- it was asked for by nobody
+ * and no test noticed. This answers "did it actually ship?", which is the only
+ * question whose answer differs between a working and a broken binary.
+ *
+ * Matches the embedded VFS entry (`"n":"<file>"`) rather than worker body
+ * symbols. `dist/framework-src` ships a renamed copy of the same source as a
+ * data asset (`...worker-entry.ts.src`), so body symbols appear in a broken
+ * binary too and read as a pass. The trailing quote keeps `.ts` from matching
+ * `.ts.src`.
+ */
+export function findMissingEmbeddedWorkers(
+  binaryContent: string,
+  workerIncludes: readonly string[],
+): string[] {
+  return workerIncludes.filter((include) => {
+    const fileName = include.slice(include.lastIndexOf("/") + 1);
+    return !binaryContent.includes(`"n":"${fileName}"`);
+  });
+}
+
+/**
+ * Streams the binary looking for each worker's VFS entry.
+ *
+ * Reads in chunks rather than decoding the file at once: these binaries embed
+ * hundreds of megabytes, and a single decode throws "buffer exceeds maximum
+ * length" well before it can check anything. Decodes as latin1 so bytes map to
+ * characters one-for-one and the ASCII markers survive arbitrary binary data,
+ * and carries the tail of each chunk forward so a marker split across a
+ * boundary is still found.
+ */
+async function findMissingEmbeddedWorkersInFile(
+  path: string,
+  workerIncludes: readonly string[],
+): Promise<string[]> {
+  const CHUNK_BYTES = 8 * 1024 * 1024;
+  const markerLength = (include: string) => include.length - include.lastIndexOf("/") + 6;
+  const carryLength = Math.max(...workerIncludes.map(markerLength));
+
+  const remaining = new Set(workerIncludes);
+  const decoder = new TextDecoder("latin1");
+  const buffer = new Uint8Array(CHUNK_BYTES);
+  const file = await Deno.open(path, { read: true });
+
+  try {
+    let carry = "";
+    while (remaining.size > 0) {
+      const bytesRead = await file.read(buffer);
+      if (bytesRead === null) break;
+
+      const text = carry + decoder.decode(buffer.subarray(0, bytesRead));
+      for (const include of [...remaining]) {
+        if (findMissingEmbeddedWorkers(text, [include]).length === 0) {
+          remaining.delete(include);
+        }
+      }
+      carry = text.slice(-carryLength);
+    }
+  } finally {
+    file.close();
+  }
+
+  return [...remaining];
+}
+
+async function assertWorkersEmbedded(
+  outputPath: string,
+  profile: CompileBinaryProfile,
+): Promise<void> {
+  // Expectations come from the declared constant, NOT from the resolved include
+  // list. Deriving them from the include list makes this tautological: dropping
+  // a worker from the list would also drop it from what is checked, so the one
+  // regression this exists to catch would pass. The proxy profile is the sole
+  // exemption, for the build reason documented on PROXY_INCLUDES.
+  const expected = profile === "proxy" ? [] : UNTRACEABLE_WORKER_INCLUDES;
+  if (expected.length === 0) {
+    return;
+  }
+
+  const missing = await findMissingEmbeddedWorkersInFile(
+    normalizeOutputPath(outputPath),
+    expected,
+  );
+  if (missing.length > 0) {
+    throw new Error(
+      `Compiled binary is missing ${missing.length} worker entrypoint(s): ${
+        missing.join(", ")
+      }.\nThe binary would start, serve traffic, and crash the first time one is spawned.`,
+    );
+  }
+}
+
 export async function compileBinary(options: CompileBinaryOptions): Promise<void> {
   const result = await new Deno.Command("deno", {
     args: createCompileArgs(options),
@@ -120,6 +248,8 @@ export async function compileBinary(options: CompileBinaryOptions): Promise<void
   if (!result.success) {
     throw new Error(`deno compile failed with exit code ${result.code}`);
   }
+
+  await assertWorkersEmbedded(options.output, options.profile ?? "full");
 }
 
 function normalizeOutputPath(path: string): string {

src/utils/version-constant.ts

@@ -1,4 +1,4 @@
 // Keep in sync with deno.json version.
 // scripts/release.ts updates this constant during releases.
 /** Shared version value. */
-export const VERSION = "0.1.1202";
+export const VERSION = "0.1.1203";

tests/unit/build/compile-binary-includes.test.ts

@@ -1,14 +1,19 @@
 import "../../_helpers/contract-init.ts";
 import { assertEquals } from "#veryfront/testing/assert.ts";
 import { describe, it } from "#veryfront/testing/bdd.ts";
-import { createCompileArgs } from "../../../scripts/build/compile-binary.ts";
+import {
+  createCompileArgs,
+  findMissingEmbeddedWorkers,
+  UNTRACEABLE_WORKER_INCLUDES,
+} from "../../../scripts/build/compile-binary.ts";
 
 describe("compile-binary includes", () => {
-  function getIncludeFlags(): string[] {
+  function getIncludeFlags(profile: "full" | "proxy" = "full"): string[] {
     const args = createCompileArgs({
-      entrypoint: "cli/main.ts",
+      entrypoint: profile === "proxy" ? "cli/proxy-main.ts" : "cli/main.ts",
       extraIncludes: [],
-      output: "/tmp/test-veryfront",
+      output: "test-veryfront",
+      profile,
     });
 
     const includeFlags: string[] = [];
@@ -22,6 +27,179 @@ describe("compile-binary includes", () => {
     return includeFlags;
   }
 
+  it("should detect a worker entry missing from a compiled binary", () => {
+    // The include-list tests all answer "did we ask for this?". The binary that
+    // crash-looped production would pass every one of them. This answers "did it
+    // ship?", which is the only question whose answer differed.
+    const embedded = '{"File":{"n":"declarative-evaluator-worker-entry.ts","o":[1,2]}}';
+    assertEquals(
+      findMissingEmbeddedWorkers(embedded, ["src/config/declarative-evaluator-worker-entry.ts"]),
+      [],
+    );
+    assertEquals(
+      findMissingEmbeddedWorkers("", ["src/config/declarative-evaluator-worker-entry.ts"]),
+      ["src/config/declarative-evaluator-worker-entry.ts"],
+    );
+  });
+
+  it("should not accept the framework-src data asset as an embedded worker", () => {
+    // dist/framework-src ships the same source renamed `.ts.src` so compile
+    // treats it as data, not a module. A broken binary therefore still contains
+    // the worker's body and its `.src` name. Only the exact VFS entry
+    // discriminates, so the trailing quote in the marker is load-bearing.
+    const brokenBinary =
+      '{"File":{"n":"declarative-evaluator-worker-entry.ts.src","o":[1,2]}} evaluateRequest';
+    assertEquals(
+      findMissingEmbeddedWorkers(brokenBinary, [
+        "src/config/declarative-evaluator-worker-entry.ts",
+      ]),
+      ["src/config/declarative-evaluator-worker-entry.ts"],
+    );
+  });
+
+  it("should embed the untraceable worker entrypoints in the full profile", () => {
+    // Wiring check on the `...UNTRACEABLE_WORKER_INCLUDES` spread, not the real
+    // protection -- iterating the constant makes an empty constant vacuously
+    // green. The two discovery tests below are what actually guard the class,
+    // so assert the constant is populated before trusting this.
+    assertEquals(
+      UNTRACEABLE_WORKER_INCLUDES.length > 0,
+      true,
+      "UNTRACEABLE_WORKER_INCLUDES is empty, which makes this test vacuous",
+    );
+    const includeFlags = getIncludeFlags("full");
+    for (const workerInclude of UNTRACEABLE_WORKER_INCLUDES) {
+      assertEquals(
+        includeFlags.includes(workerInclude),
+        true,
+        `${workerInclude} must be embedded in the full profile, got includes: ${
+          JSON.stringify(includeFlags)
+        }`,
+      );
+    }
+  });
+
+  it("should keep untraceable workers out of the frozen proxy profile", () => {
+    // Pins a build constraint, not a safety claim. Embedding these fails the
+    // proxy compile: the worker entry's graph wants a newer @babel/types than
+    // proxy-deno.lock pins, and --frozen refuses to update it. The lock does
+    // carry a babel toolchain already, so this says nothing about whether the
+    // proxy can reach a worker spawn -- that question is tracked separately.
+    // Regenerate the lock in the same change if the answer turns out to be yes.
+    assertEquals(
+      UNTRACEABLE_WORKER_INCLUDES.length > 0,
+      true,
+      "UNTRACEABLE_WORKER_INCLUDES is empty, which makes this test vacuous",
+    );
+    const includeFlags = getIncludeFlags("proxy");
+    for (const workerInclude of UNTRACEABLE_WORKER_INCLUDES) {
+      assertEquals(
+        includeFlags.includes(workerInclude),
+        false,
+        `${workerInclude} in the proxy profile breaks the frozen proxy lock; regenerate scripts/build/proxy-deno.lock in the same change`,
+      );
+    }
+  });
+
+  it("should include every worker entrypoint named as one", async () => {
+    // Complements the call-site scan below, which finds workers by how they are
+    // spawned and so misses an entrypoint that exists but is not yet wired to a
+    // `new Worker(...)` the scan recognises. This finds them by name instead, so
+    // the two together cover both a worker the code spawns and a worker the
+    // repo merely contains.
+    const projectRoot = new URL("../../../", import.meta.url);
+    const entrypointName = /-worker-entry\.ts$|(?:^|-)worker-script\.ts$/;
+
+    const workerEntrypoints: string[] = [];
+    async function collect(dir: string): Promise<void> {
+      for await (const entry of Deno.readDir(new URL(dir, projectRoot))) {
+        const path = `${dir}/${entry.name}`;
+        if (entry.isDirectory) {
+          if (entry.name === "node_modules" || entry.name.startsWith(".")) continue;
+          await collect(path);
+        } else if (entry.name.endsWith(".ts") && entrypointName.test(entry.name)) {
+          workerEntrypoints.push(path);
+        }
+      }
+    }
+    await collect("src");
+
+    assertEquals(
+      workerEntrypoints.length > 0,
+      true,
+      "expected to discover at least one worker entrypoint under src",
+    );
+
+    const includeFlags = getIncludeFlags("full");
+    for (const entrypoint of workerEntrypoints) {
+      assertEquals(
+        includeFlags.some((path) => entrypoint === path || entrypoint.startsWith(`${path}/`)),
+        true,
+        `${entrypoint} must be in compile-binary.ts DEFAULT_INCLUDES; deno compile cannot trace the computed worker URL, so the binary crashes when the worker is spawned`,
+      );
+    }
+  });
+
+  it("should include every worker entrypoint spawned from a sibling URL", async () => {
+    // `deno compile` embeds a worker only when it can statically read the
+    // specifier. Every worker here resolves one relative to `import.meta.url`,
+    // and two of them compute the extension, so none are traceable. A missing
+    // entry does not fail the build or any test that runs from source: the
+    // binary starts, serves traffic, and dies on the first request that spawns
+    // the worker. That is how the declarative config evaluator reached
+    // production and crash-looped it.
+    const projectRoot = new URL("../../../", import.meta.url);
+    const siblingWorkerSpecifier = /["'`](\.\/[^"'`\n]*[Ww]orker[^"'`\n]*)["'`]/g;
+
+    async function* sourceFiles(dir: string): AsyncGenerator<string> {
+      for await (const entry of Deno.readDir(new URL(dir, projectRoot))) {
+        const path = `${dir}/${entry.name}`;
+        if (entry.isDirectory) {
+          if (entry.name === "node_modules" || entry.name.startsWith(".")) continue;
+          yield* sourceFiles(path);
+        } else if (entry.name.endsWith(".ts") && !entry.name.endsWith(".test.ts")) {
+          yield path;
+        }
+      }
+    }
+
+    const includeFlags = getIncludeFlags();
+    const isCovered = (path: string) =>
+      includeFlags.some((include) => path === include || path.startsWith(`${include}/`));
+
+    const uncovered: string[] = [];
+    for (const root of ["src", "extensions"]) {
+      for await (const file of sourceFiles(root)) {
+        const source = await Deno.readTextFile(new URL(file, projectRoot));
+        // Only files that actually construct a Worker; a sibling URL elsewhere
+        // is not a worker entrypoint and does not need embedding.
+        if (!source.includes("new Worker(")) continue;
+
+        for (const match of source.matchAll(siblingWorkerSpecifier)) {
+          const specifier = match[1] ?? "";
+          // A static `import`/`export ... from` specifier is traced by compile
+          // and needs no include. Only a specifier reached some other way --
+          // assigned to a variable, or built inline for `new URL` -- is opaque.
+          if (/(?:from|import)\s*\(?\s*$/.test(source.slice(0, match.index))) continue;
+          // Normalise `./name${extension}` and `./name.js` to the `.ts` source.
+          const entry = specifier
+            .replace(/\$\{[^}]*\}/g, "")
+            .replace(/\.(?:ts|js)$/, "");
+          const resolved = `${file.slice(0, file.lastIndexOf("/"))}/${entry.slice(2)}.ts`;
+          if (!isCovered(resolved)) uncovered.push(`${file} -> ${resolved}`);
+        }
+      }
+    }
+
+    assertEquals(
+      uncovered,
+      [],
+      `Worker entrypoints spawned via a sibling URL must be listed in compile-binary.ts DEFAULT_INCLUDES, or they are absent from the compiled binary and crash it at runtime:\n${
+        uncovered.join("\n")
+      }`,
+    );
+  });
+
   it("should include src/rendering/rsc for client hydration scripts", () => {
     // Regression: client-boot.ts and client-dom.ts must be embedded in the
     // compiled binary, otherwise RSC hydration fails with