Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 62 additions & 5 deletions scripts/build/build-npm-dnt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -342,23 +342,80 @@ await buildExtensionPackages({
await verifyNpmRootImportLifecycle();

async function verifyNpmRootImportLifecycle(): Promise<void> {
const consumerDirectory = await Deno.makeTempDir({
prefix: "veryfront-npm-lifecycle-",
});
try {
await installBuiltNpmLifecycleConsumer(consumerDirectory);
await runNpmRootImportLifecycleProbe(consumerDirectory);
} finally {
await Deno.remove(consumerDirectory, { recursive: true }).catch(() => undefined);
}
}

async function installBuiltNpmLifecycleConsumer(consumerDirectory: string): Promise<void> {
const localPackageDirectories = await Promise.all([
Deno.realPath("./npm"),
Deno.realPath("./npm/extensions/ext-bundler-esbuild"),
Deno.realPath("./npm/extensions/ext-content-mdx"),
Deno.realPath("./npm/extensions/ext-css-tailwind"),
Deno.realPath("./npm/extensions/ext-parser-babel"),
Deno.realPath("./npm/extensions/ext-yaml"),
]);
await Deno.writeTextFile(
`${consumerDirectory}/package.json`,
JSON.stringify({ private: true, type: "module" }),
);
const install = await new Deno.Command("npm", {
args: [
"install",
"--ignore-scripts",
"--legacy-peer-deps",
"--no-audit",
"--no-fund",
"--no-package-lock",
"--install-links",
...localPackageDirectories,
],
cwd: consumerDirectory,
stdout: "piped",
stderr: "piped",
}).output();
if (!install.success) {
const stderr = new TextDecoder().decode(install.stderr).trim();
throw new Error(
`Built npm lifecycle consumer install failed with exit code ${install.code}.` +
(stderr ? `\n${stderr}` : ""),
);
}
}

async function runNpmRootImportLifecycleProbe(consumerDirectory: string): Promise<void> {
const timeoutMs = 10_000;
const probeSource = `
const root = await import("./esm/src/index.js");
const root = await import("veryfront");
if (typeof root.defineConfig !== "function") {
throw new Error("defineConfig export missing");
}

const agent = await import("veryfront/agent");
const metadata = agent.parseRuntimeSkillMetadata(
"---\\nname: public-api\\ndescription: Public API\\n---\\nBody",
);
if (metadata?.name !== "public-api") {
throw new Error("public runtime Skill parser default unavailable");
}

const { createEvalCliBuiltinExtensions } = await import(
"./esm/src/extensions/builtin-extensions.js"
"./node_modules/veryfront/esm/src/extensions/builtin-extensions.js"
);
const { getDeferredExtensionState } = await import(
"./esm/src/extensions/deferred-extension.js"
"./node_modules/veryfront/esm/src/extensions/deferred-extension.js"
);
const {
createEvalReportExporterRegistry,
EvalReportExporterRegistryName,
} = await import("./esm/src/extensions/eval/index.js");
} = await import("./node_modules/veryfront/esm/src/extensions/eval/index.js");

const registry = createEvalReportExporterRegistry();
const resolved = createEvalCliBuiltinExtensions(["mlflow"]).find(
Expand Down Expand Up @@ -407,7 +464,7 @@ if (registry.has("mlflow")) {
"--eval",
probeSource,
],
cwd: "./npm",
cwd: consumerDirectory,
env: {
MLFLOW_TRACKING_URI: "http://127.0.0.1:5000",
VF_DISABLE_LRU_INTERVAL: "0",
Expand Down
22 changes: 22 additions & 0 deletions scripts/build/npm-package-metadata.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,28 @@ Deno.test("root npm CLI package declares auto-loaded first-party extensions afte
}
});

Deno.test("npm lifecycle probe installs auto-loaded extensions in a real consumer layout", async () => {
const source = await Deno.readTextFile("scripts/build/build-npm-dnt.ts");

assertStringIncludes(source, '"--install-links"');
assertStringIncludes(source, 'const agent = await import("veryfront/agent")');
assertStringIncludes(source, "agent.parseRuntimeSkillMetadata(");
for (
const extensionDirectory of [
"ext-bundler-esbuild",
"ext-content-mdx",
"ext-css-tailwind",
"ext-parser-babel",
"ext-yaml",
]
) {
assertStringIncludes(
source,
`Deno.realPath("./npm/extensions/${extensionDirectory}")`,
);
}
});

Deno.test("npm publish version bump pins first-party extension dependencies to the publish version", async () => {
const packageDir = await Deno.makeTempDir();
const packagePath = `${packageDir}/package.json`;
Expand Down
9 changes: 6 additions & 3 deletions src/agent/factory-call-context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,11 @@ describe("agent/factory call context", () => {
});

assertStringIncludes(prompt, "<available_skills>");
assertStringIncludes(prompt, "- support-triage: Triage incoming support requests");
assertEquals(prompt.includes("(tools: create_file)"), false);
assertStringIncludes(
prompt,
'- {"skillId":"support-triage","description":"Triage incoming support requests","allowedTools":[]}',
);
assertEquals(prompt.includes("create_file"), false);
assertStringIncludes(prompt, "execute_skill_script: Call with");
});

Expand Down Expand Up @@ -148,7 +151,7 @@ describe("agent/factory call context", () => {

assertStringIncludes(
prompt,
"- support-triage: Triage incoming support requests (tools: create_file)",
'- {"skillId":"support-triage","description":"Triage incoming support requests","allowedTools":["create_file"]}',
);
});
});
6 changes: 3 additions & 3 deletions src/agent/factory.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ describe("agent factory", () => {
: effectiveSystem ?? "";
assertStringIncludes(
prompt,
"- support-triage: Triage incoming support requests",
'- {"skillId":"support-triage","description":"Triage incoming support requests"}',
);
assertEquals(prompt.includes("researcher--cite"), false);

Expand Down Expand Up @@ -174,8 +174,8 @@ describe("agent factory", () => {
? await allowlistedSystem()
: allowlistedSystem ?? "";

assertStringIncludes(prompt, "- writer--draft: Draft copy");
assertStringIncludes(prompt, "- global-plan: Plan the work");
assertStringIncludes(prompt, '- {"skillId":"writer--draft","description":"Draft copy"}');
assertStringIncludes(prompt, '- {"skillId":"global-plan","description":"Plan the work"}');
assertEquals(prompt.includes("global-review"), false);

if (!allowlisted.config.tools || allowlisted.config.tools === true) {
Expand Down
46 changes: 44 additions & 2 deletions src/agent/hosted/agent-project-steering.test.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,27 @@
import { assertEquals, assertRejects, assertThrows } from "@std/assert";
import { afterEach } from "#veryfront/testing/bdd.ts";
import { join, resolve } from "node:path";
import { createHostedAgentProjectSteering } from "./agent-project-steering.ts";
import { reset, tryResolve } from "../../extensions/contracts.ts";
import {
createHostedAgentProjectSteering as createHostedAgentProjectSteeringPublic,
type HostedAgentProjectSteeringOptions,
} from "./agent-project-steering.ts";
import { createStdYamlSkillDocumentParserProvider } from "../../../extensions/ext-yaml/src/adapter.ts";
import { register, reset, tryResolve } from "../../extensions/contracts.ts";
import { SkillDocumentParserProviderName } from "../../extensions/parser/skill-document-parser.ts";
import type { SchemaValidator } from "../../extensions/schema/index.ts";
import type { RuntimeProjectFilesFetch } from "../runtime/project-files-client.ts";

const skillDocumentParserProvider = createStdYamlSkillDocumentParserProvider();

function createHostedAgentProjectSteering(
options: Omit<HostedAgentProjectSteeringOptions, "skillDocumentParserProvider">,
) {
return createHostedAgentProjectSteeringPublic({
...options,
skillDocumentParserProvider,
});
}

function withTempDir(fn: (rootDir: string) => void | Promise<void>): Promise<void> {
const rootDir = Deno.makeTempDirSync();
return Promise.resolve(fn(rootDir)).finally(() => {
Expand Down Expand Up @@ -61,6 +77,32 @@ Deno.test("createHostedAgentProjectSteering registers the built-in schema valida
});
});

Deno.test("createHostedAgentProjectSteering keeps the parser provider optional for existing callers", async () => {
await withTempDir((rootDir) => {
const baseDir = writeAgentDefinition({ rootDir, agentId: "writer" });
const skillsDir = join(rootDir, "skills");
Deno.mkdirSync(skillsDir, { recursive: true });
Deno.writeTextFileSync(
join(skillsDir, "plan.md"),
`---
description: Plans
---
Plan carefully.`,
);
register(SkillDocumentParserProviderName, skillDocumentParserProvider);

const steering = createHostedAgentProjectSteeringPublic({
baseDir,
agentId: "writer",
skillsDir,
getApiUrl: () => "https://api.example.com",
});

assertEquals(steering.getAgentConfig().id, "writer");
assertEquals(steering.getProjectSteeringAdapter().listBuiltinSkillIds(), ["plan"]);
});
});

Deno.test("createHostedAgentProjectSteering loads and caches markdown agent definitions", async () => {
await withTempDir((rootDir) => {
const baseDir = writeAgentDefinition({ rootDir, agentId: "writer" });
Expand Down
5 changes: 5 additions & 0 deletions src/agent/hosted/agent-project-steering.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import type {
import type { RuntimeProjectSteeringLookup } from "../runtime/project-skill-catalog.ts";
import type { RuntimeLoadSkillToolContext } from "../runtime/load-skill-tool.ts";
import type { RuntimeSkillDefinition } from "../runtime/skill-metadata.ts";
import type { SkillDocumentParserProvider } from "#veryfront/extensions/parser/skill-document-parser.ts";

/** Public API contract for hosted agent project steering options data. */
export type HostedAgentProjectSteeringOptionsData = {
Expand Down Expand Up @@ -58,6 +59,7 @@ export type HostedAgentProjectSteeringOptions = HostedAgentProjectSteeringOption
logger?: HostedAgentProjectSteeringLogger;
trace?: RuntimeProjectFilesTrace;
fetch?: RuntimeProjectFilesFetch;
skillDocumentParserProvider?: SkillDocumentParserProvider;
};

/** Public API contract for hosted agent project steering. */
Expand Down Expand Up @@ -130,6 +132,9 @@ export function createHostedAgentProjectSteering(
logger: options.logger,
trace: options.trace,
fetch: options.fetch,
...(options.skillDocumentParserProvider === undefined
? {}
: { skillDocumentParserProvider: options.skillDocumentParserProvider }),
});

return cachedProjectSteeringAdapter;
Expand Down
46 changes: 45 additions & 1 deletion src/agent/hosted/child-fork-execution-runner.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { assertEquals } from "@std/assert";
import { assertEquals, assertRejects } from "@std/assert";
import type { HostToolSet } from "#veryfront/tool";
import {
DEFAULT_HOSTED_CHILD_FORK_STREAM_ACTIVE_TOOL_TIMEOUT_MS,
Expand All @@ -11,6 +11,7 @@ import {
} from "./child-fork-execution-runner.ts";
import { createHostedDurableChildForkRunContext } from "./child-fork-run-context.ts";
import type { AgentResponse } from "../schemas/index.ts";
import { UNCONFIRMED_AGENT_PROJECT_IDENTITY_MESSAGE } from "../project/context.ts";
import { getActiveModelCallRecorder } from "../../runtime/model-call-recorder-context.ts";

function createRuntimeEventStream(
Expand Down Expand Up @@ -439,6 +440,49 @@ Deno.test("executeHostedChildForkToolInput resolves runtime config and prepares
}
});

Deno.test("executeHostedChildForkToolInput rejects unconfirmed project identities before setup", async () => {
const callbacks: string[] = [];

await assertRejects(
() =>
executeHostedChildForkToolInput({
authToken: "token",
apiUrl: "https://api.example.com",
kind: "invoke_agent",
toolCallId: "tool-call-unconfirmed-project",
forkInput: {
description: "Review checkout",
prompt: "Review the checkout flow.",
context: {},
project_reference: "project-two",
},
defaultModel: "haiku",
defaultMaxSteps: 80,
resolveProjectReference: () => Promise.resolve({ projectId: "project-three" }),
onRequestedProjectId: () => {
callbacks.push("project");
},
resolveModelId: (modelId) => {
callbacks.push("model");
return modelId;
},
resolveProvider: () => "anthropic",
prepareToolAssembly: () => {
callbacks.push("tools");
return { ok: true, forkTools: {}, availableToolNames: [] };
},
startRuntime: () => {
callbacks.push("runtime");
throw new Error("unexpected runtime start");
},
}),
TypeError,
UNCONFIRMED_AGENT_PROJECT_IDENTITY_MESSAGE,
);

assertEquals(callbacks, []);
});

Deno.test("executeHostedChildForkToolInput honors full result mode", async () => {
const rawText =
'<function_calls><invoke name="run_bash">cat report.md</invoke></function_calls><function_result>Exact delegated output.</function_result>';
Expand Down
14 changes: 11 additions & 3 deletions src/agent/hosted/child-fork-execution-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ import {
import type { SourceIntegrationPolicyManifest } from "#veryfront/integrations/source-policy.ts";
import {
type HostedProjectReferenceResolver,
requireConfirmedHostedProjectReference,
resolveHostedProjectReference,
} from "./project-reference-resolver.ts";
import { runWithModelCallRecorder } from "../../runtime/model-call-recorder-context.ts";
Expand Down Expand Up @@ -227,7 +228,7 @@ export type ExecuteHostedChildForkToolInputOptions<
defaultModel: string;
defaultMaxSteps: number;
contextModel?: string;
onRequestedProjectId?: (projectId: string) => void | Promise<void>;
onRequestedProjectId?: (projectId: string, projectSlug?: string) => void | Promise<void>;
resolveProjectReference?: HostedProjectReferenceResolver;
prepareToolAssembly: (input: {
runtimeConfig: HostedChildForkRuntimeConfig;
Expand Down Expand Up @@ -260,13 +261,20 @@ export async function executeHostedChildForkToolInput<
const requestedProjectReference = input.forkInput.project_reference;
if (requestedProjectReference) {
const resolver = input.resolveProjectReference ?? resolveHostedProjectReference;
const resolvedProject = await resolver({
const resolution = await resolver({
projectReference: requestedProjectReference,
authToken: input.authToken,
apiUrl: input.apiUrl,
abortSignal: input.abortSignal,
});
await input.onRequestedProjectId?.(resolvedProject.projectId);
const resolvedProject = requireConfirmedHostedProjectReference(
resolution,
requestedProjectReference,
);
await input.onRequestedProjectId?.(
resolvedProject.projectId,
resolvedProject.projectSlug,
);
}

const forkInput = input.inputAlreadyHasInvocationContext
Expand Down
13 changes: 9 additions & 4 deletions src/agent/hosted/child-fork-tool-sources.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,8 @@ function createSandboxToolsResult(input: {

Deno.test("prepareDefaultHostedChildForkToolSources loads API, live Studio, and global tools", async () => {
const fixtures = createRemoteSourceFixtures();
const switchedProjects: string[] = [];
const switchedProjectIds: string[] = [];
const switchedProjects: Array<{ projectId: string; projectSlug?: string }> = [];

const result = await prepareDefaultHostedChildForkToolSources({
authToken: "token-1",
Expand All @@ -146,8 +147,11 @@ Deno.test("prepareDefaultHostedChildForkToolSources loads API, live Studio, and
execute: () => ({ ok: true }),
},
},
onConfirmedStudioProjectSwitch: (projectId) => {
switchedProjects.push(projectId);
onConfirmedStudioProjectSwitch: (projectId, confirmedProject) => {
switchedProjectIds.push(projectId);
if (confirmedProject) {
switchedProjects.push(confirmedProject);
}
},
createRemoteToolSource: fixtures.createRemoteToolSource,
});
Expand All @@ -168,7 +172,8 @@ Deno.test("prepareDefaultHostedChildForkToolSources loads API, live Studio, and

await result.forkTools.studio_open_project?.execute?.({ project_reference: "project-two" });

assertEquals(switchedProjects, ["project-2"]);
assertEquals(switchedProjectIds, ["project-2"]);
assertEquals(switchedProjects, [{ projectId: "project-2", projectSlug: "project-two" }]);
assertEquals(fixtures.executeCalls, [
{
sourceId: "veryfront-mcp-fork",
Expand Down
Loading