diff --git a/docs/api-reference/veryfront/agent.md b/docs/api-reference/veryfront/agent.md index ee1c32629e..9c08600d74 100644 --- a/docs/api-reference/veryfront/agent.md +++ b/docs/api-reference/veryfront/agent.md @@ -1040,7 +1040,7 @@ Input delivered to a hosted agent-service detached execution callback. | Name | Description | Source | | ------------------------------------ | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | -| `AgentRuntime` | Implement agent runtime. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/index.ts#L676) | +| `AgentRuntime` | Implement agent runtime. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/index.ts#L680) | | `AgentRuntimeMessageConversionError` | Error shape for agent runtime message conversion. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/message-adapter.ts#L138) | | `AgentServiceAuthError` | Error shape for hosted service auth. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/auth.ts#L14) | | `AppendConversationRunEventsError` | Error shape for append conversation run events. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/conversation/durable-append-errors.ts#L4) | diff --git a/docs/api-reference/veryfront/integrations.md b/docs/api-reference/veryfront/integrations.md index 81276d42a6..0195ba3cbb 100644 --- a/docs/api-reference/veryfront/integrations.md +++ b/docs/api-reference/veryfront/integrations.md @@ -13,7 +13,7 @@ import { getConnectorNames, getIcon, getRemoteIntegrationToolDefinitions, - isRemoteIntegrationTool, + getRemoteIntegrationToolDiscovery, } from "veryfront/integrations"; ``` @@ -24,12 +24,14 @@ import { getConnector, getIcon, getRemoteIntegrationToolDefinitions, + getRemoteIntegrationToolDiscovery, listConnectors, } from "veryfront/integrations"; const connectors = listConnectors(); const slack = getConnector("slack"); const slackIcon = getIcon("slack"); // raw SVG string +const discovery = await getRemoteIntegrationToolDiscovery(); const runtimeTools = await getRemoteIntegrationToolDefinitions(); ``` @@ -52,25 +54,27 @@ const runtimeTools = await getRemoteIntegrationToolDefinitions(); | Name | Description | Source | | ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | -| `executeRemoteIntegrationTool` | Execute a remote integration tool via the API. Called by the agent runtime when a tool isn't found in the local registry. The request, response, and caller-supplied cancellation signal remain bounded for the complete network and response-body lifecycle. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/remote-tools.ts#L665) | -| `getConnector` | Return connector. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/index.ts#L49) | -| `getConnectorNames` | Return connector names. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/index.ts#L60) | -| `getIcon` | Return icon. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/index.ts#L65) | -| `getRemoteIntegrationToolDefinitions` | Fetch integration tool definitions for the current request context. Returns ToolDefinition[] that the agent runtime merges into the model's available tools. Returns empty array if no API config or no tools. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/remote-tools.ts#L618) | -| `isRemoteIntegrationTool` | Check if a tool name looks like a remote integration tool. Integration tools use "integration__tool_id" format (double underscore separator). | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/remote-tools.ts#L655) | -| `listConnectors` | List connectors. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/index.ts#L55) | +| `executeRemoteIntegrationTool` | Execute a remote integration tool via the API. Called by the agent runtime when a tool isn't found in the local registry. The request, response, and caller-supplied cancellation signal remain bounded for the complete network and response-body lifecycle. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/remote-tools.ts#L791) | +| `getConnector` | Return connector. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/index.ts#L51) | +| `getConnectorNames` | Return connector names. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/index.ts#L62) | +| `getIcon` | Return icon. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/index.ts#L67) | +| `getRemoteIntegrationToolDefinitions` | Fetch integration tool definitions for the current request context. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/remote-tools.ts#L770) | +| `getRemoteIntegrationToolDiscovery` | Discover integration tools for the current request context. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/remote-tools.ts#L721) | +| `isRemoteIntegrationTool` | Check if a tool name looks like a remote integration tool. Integration tools use "integration__tool_id" format (double underscore separator). | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/remote-tools.ts#L781) | +| `listConnectors` | List connectors. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/index.ts#L57) | ### Types -| Name | Description | Source | -| -------------------------------------- | ------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| `EnvVarConfig` | Configuration used by env var. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L482) | -| `IntegrationConfig` | Configuration used by integration. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L496) | -| `IntegrationConnector` | Public API contract for integration connector. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/types.ts#L75) | -| `IntegrationEndpointHistoricalSummary` | Provider-declared summary contract for old tool outputs kept actionable across turns. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L490) | -| `IntegrationName` | Public API contract for integration name. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L480) | -| `IntegrationPrompt` | Public API contract for integration prompt. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L494) | -| `IntegrationTool` | Public API contract for integration tool. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/types.ts#L66) | -| `IntegrationToolMeta` | Public API contract for integration tool meta. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L488) | -| `OAuthConfig` | Configuration used by oauth. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L486) | -| `OAuthField` | Public API contract for oauth field. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L484) | +| Name | Description | Source | +| -------------------------------------- | ------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| `EnvVarConfig` | Configuration used by env var. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L482) | +| `IntegrationConfig` | Configuration used by integration. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L496) | +| `IntegrationConnector` | Public API contract for integration connector. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/types.ts#L75) | +| `IntegrationEndpointHistoricalSummary` | Provider-declared summary contract for old tool outputs kept actionable across turns. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L490) | +| `IntegrationName` | Public API contract for integration name. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L480) | +| `IntegrationPrompt` | Public API contract for integration prompt. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L494) | +| `IntegrationTool` | Public API contract for integration tool. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/types.ts#L66) | +| `IntegrationToolMeta` | Public API contract for integration tool meta. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L488) | +| `OAuthConfig` | Configuration used by oauth. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L486) | +| `OAuthField` | Public API contract for oauth field. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/schema.ts#L484) | +| `RemoteIntegrationToolDiscoveryResult` | Result of listing the integration tools available to the current run. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/integrations/remote-tools.ts#L70) | diff --git a/scripts/docs/generate-api-reference.ts b/scripts/docs/generate-api-reference.ts index b4762bcfb9..ff80655807 100644 --- a/scripts/docs/generate-api-reference.ts +++ b/scripts/docs/generate-api-reference.ts @@ -748,6 +748,10 @@ const DESCRIPTIONS: Record> = { getConnector: "Look up connector config by name from registry", getConnectorNames: "Return readonly array of all connector names", getIcon: "Return SVG icon string for integration by name", + getRemoteIntegrationToolDiscovery: + "List integration tools with a typed available or unavailable status", + getRemoteIntegrationToolDefinitions: + "List integration tool definitions, returning an empty list when discovery is unavailable", listConnectors: "Return readonly array of all connectors", registerIntegrationMCP: "Register integration tools into the MCP tool registry", @@ -758,6 +762,8 @@ const DESCRIPTIONS: Record> = { IntegrationMCPConfig: "Configuration for registering integrations into MCP", IntegrationName: "Union type of valid integration name literals", IntegrationPrompt: "Predefined prompt template for integration use", + RemoteIntegrationToolDiscoveryResult: + "Typed integration tool catalog status for the current run", IntegrationTool: "Integration tool with endpoint execution spec", IntegrationToolMeta: "Tool metadata: name, description, write requirements", OAuthConfig: "OAuth/API key authentication type and parameters", diff --git a/src/agent/runtime/agent-runtime-step.test.ts b/src/agent/runtime/agent-runtime-step.test.ts index 7d77038f68..7d5ad07f05 100644 --- a/src/agent/runtime/agent-runtime-step.test.ts +++ b/src/agent/runtime/agent-runtime-step.test.ts @@ -3,8 +3,11 @@ import { assertEquals, assertStrictEquals } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; import type { RemoteToolSource, ToolDefinition, ToolExecutionContext } from "#veryfront/tool"; import type { AgentConfig, Message } from "../types.ts"; -import type { AgentRuntimeStepState } from "./agent-runtime-step.ts"; -import { prepareAgentRuntimeStep } from "./agent-runtime-step.ts"; +import type { AgentRuntimeStepState, RuntimeStepToolLoader } from "./agent-runtime-step.ts"; +import { + prepareAgentRuntimeStep, + withIntegrationToolDiscoveryStatus, +} from "./agent-runtime-step.ts"; import { createToolExposureState } from "./tool-exposure.ts"; function toolDefinition(name: string): ToolDefinition { @@ -91,6 +94,152 @@ describe("agent/runtime-step", () => { assertEquals(prepared.toolExposurePlan.authorized.map((tool) => tool.name), ["create_release"]); }); + it("preserves a typed empty integration catalog without adding a warning", async () => { + const prepared = await prepareAgentRuntimeStep({ + agentId: "agent_1", + activeSkillToolAvailability: undefined, + allowedRemoteToolNames: undefined, + config: { model: "auto", system: "Base", tools: true } as AgentConfig, + forwardedRemoteToolDefinitions: undefined, + getAvailableTools: async (_toolsConfig, options) => { + options?.onIntegrationToolDiscovery?.({ status: "ok", tools: [] }); + return []; + }, + supportsToolCalling: true, + messages: [], + mode: "generate", + remoteToolSources: undefined, + resolveRuntimeState: async () => ({ systemPrompt: "Base" }), + runtimeContext: undefined, + step: 0, + systemPrompt: "Base", + toolContextBase: undefined, + }); + + assertEquals(prepared.integrationToolDiscovery, { status: "ok", tools: [] }); + const systemPrompt = withIntegrationToolDiscoveryStatus( + prepared.systemPrompt, + prepared.integrationToolDiscovery, + ); + assertEquals(systemPrompt, "Base"); + }); + + it("tells the model once when integration discovery is unavailable", async () => { + const getAvailableTools: RuntimeStepToolLoader = async (_toolsConfig, options) => { + options?.onIntegrationToolDiscovery?.({ + status: "unavailable", + reason: "request_failed", + }); + return []; + }; + const input = { + agentId: "agent_1", + activeSkillToolAvailability: undefined, + allowedRemoteToolNames: undefined, + config: { model: "auto", system: "Base", tools: true } as AgentConfig, + forwardedRemoteToolDefinitions: undefined, + getAvailableTools, + supportsToolCalling: true, + messages: [], + mode: "generate" as const, + remoteToolSources: undefined, + runtimeContext: undefined, + step: 0, + systemPrompt: "Base", + toolContextBase: undefined, + }; + + const first = await prepareAgentRuntimeStep({ + ...input, + resolveRuntimeState: async () => ({ systemPrompt: "Base" }), + }); + const firstSystemPrompt = withIntegrationToolDiscoveryStatus( + first.systemPrompt, + first.integrationToolDiscovery, + ); + const second = await prepareAgentRuntimeStep({ + ...input, + systemPrompt: firstSystemPrompt, + resolveRuntimeState: async () => ({ systemPrompt: firstSystemPrompt }), + }); + const secondSystemPrompt = withIntegrationToolDiscoveryStatus( + second.systemPrompt, + second.integrationToolDiscovery, + ); + + assertEquals(second.integrationToolDiscovery, { + status: "unavailable", + reason: "request_failed", + }); + assertEquals( + secondSystemPrompt.includes( + "You must not treat this failure as an empty integration catalog", + ), + true, + ); + assertEquals( + secondSystemPrompt.split("Integration tool discovery status:").length - 1, + 1, + ); + }); + + it("replaces an integration discovery status block before prompt suffix content", () => { + const unavailable = { + status: "unavailable" as const, + reason: "request_failed" as const, + }; + const firstSystemPrompt = withIntegrationToolDiscoveryStatus("Base", unavailable); + const secondSystemPrompt = withIntegrationToolDiscoveryStatus( + `${firstSystemPrompt}\n\nSuffix`, + unavailable, + ); + + assertEquals( + secondSystemPrompt.split("Integration tool discovery status:").length - 1, + 1, + ); + assertEquals(secondSystemPrompt.startsWith("Base\n\nSuffix\n\n"), true); + }); + + it("does not report unavailable discovery when forwarded integration tools remain usable", async () => { + const forwardedTool = toolDefinition("gmail__list_emails"); + const prepared = await prepareAgentRuntimeStep({ + agentId: "agent_1", + activeSkillToolAvailability: undefined, + allowedRemoteToolNames: [forwardedTool.name], + config: { model: "auto", system: "Base", tools: true } as AgentConfig, + forwardedRemoteToolDefinitions: [forwardedTool], + getAvailableTools: async (_toolsConfig, options) => { + options?.onIntegrationToolDiscovery?.({ + status: "unavailable", + reason: "request_failed", + }); + return [forwardedTool]; + }, + supportsToolCalling: true, + messages: [], + mode: "generate", + remoteToolSources: undefined, + resolveRuntimeState: async () => ({ systemPrompt: "Base" }), + runtimeContext: undefined, + step: 0, + systemPrompt: "Base", + toolContextBase: undefined, + }); + + assertEquals(prepared.integrationToolDiscovery, { + status: "ok", + tools: [forwardedTool], + }); + assertEquals( + withIntegrationToolDiscoveryStatus( + prepared.systemPrompt, + prepared.integrationToolDiscovery, + ), + "Base", + ); + }); + it("does not let runtime context shadow the trusted abort signal", async () => { const trustedAbort = new AbortController(); const shadowAbort = new AbortController(); diff --git a/src/agent/runtime/agent-runtime-step.ts b/src/agent/runtime/agent-runtime-step.ts index 3dd2dd1514..fab5a1b2f0 100644 --- a/src/agent/runtime/agent-runtime-step.ts +++ b/src/agent/runtime/agent-runtime-step.ts @@ -4,6 +4,7 @@ import { filterToolsForSkill, type SkillToolAvailability } from "#veryfront/skil import type { ToolConfigEntry } from "./tool-helpers.ts"; import { filterToolsAfterSubmittedFormInput } from "./skill-policy-enforcement.ts"; import type { SourceIntegrationPolicyManifest } from "#veryfront/integrations/source-policy.ts"; +import type { RemoteIntegrationToolDiscoveryResult } from "#veryfront/integrations/remote-tools.ts"; import { resolveRuntimeToolLoading, SOURCE_INTEGRATION_POLICY_CONTEXT_KEY, @@ -34,6 +35,7 @@ export type RuntimeStepToolLoader = ( forwardedRemoteToolDefinitions?: ToolDefinition[]; remoteToolSources?: RemoteToolSource[]; remoteToolContext?: ToolExecutionContext; + onIntegrationToolDiscovery?: (result: RemoteIntegrationToolDiscoveryResult) => void; sourceIntegrationPolicy?: SourceIntegrationPolicyManifest; strictConfiguredToolsOnly?: boolean; callerAgentId?: string; @@ -80,6 +82,7 @@ export interface PrepareAgentRuntimeStepInput { } export interface PreparedAgentRuntimeStep { + integrationToolDiscovery?: RemoteIntegrationToolDiscoveryResult; runtimeContext: Record | undefined; systemPrompt: string; toolContext: ToolExecutionContext; @@ -87,6 +90,46 @@ export interface PreparedAgentRuntimeStep { toolExposurePlan: ToolExposurePlan; } +const INTEGRATION_TOOL_DISCOVERY_STATUS_HEADER = "Integration tool discovery status:"; +const INTEGRATION_TOOL_DISCOVERY_STATUS_FOOTER = "End integration tool discovery status."; + +function removeIntegrationToolDiscoveryStatus(systemPrompt: string): string { + let result = systemPrompt; + while (true) { + const headerIndex = result.indexOf(INTEGRATION_TOOL_DISCOVERY_STATUS_HEADER); + if (headerIndex < 0) return result; + const footerIndex = result.indexOf( + INTEGRATION_TOOL_DISCOVERY_STATUS_FOOTER, + headerIndex + INTEGRATION_TOOL_DISCOVERY_STATUS_HEADER.length, + ); + if (footerIndex < 0) return result; + + result = [ + result.slice(0, headerIndex).trimEnd(), + result.slice( + footerIndex + INTEGRATION_TOOL_DISCOVERY_STATUS_FOOTER.length, + ).trimStart(), + ].filter(Boolean).join("\n\n"); + } +} + +export function withIntegrationToolDiscoveryStatus( + systemPrompt: string, + discovery: RemoteIntegrationToolDiscoveryResult | undefined, +): string { + const basePrompt = removeIntegrationToolDiscoveryStatus(systemPrompt); + let message: string | undefined; + if (discovery?.status === "unavailable") { + message = + "Integration tool discovery is temporarily unavailable for this run. You must not treat this failure as an empty integration catalog. If the user needs an integration tool, explain that discovery is temporarily unavailable and ask the user to retry."; + } + + if (!message) return basePrompt; + const statusBlock = + `${INTEGRATION_TOOL_DISCOVERY_STATUS_HEADER}\n\n${message}\n\n${INTEGRATION_TOOL_DISCOVERY_STATUS_FOOTER}`; + return basePrompt.length > 0 ? `${basePrompt}\n\n${statusBlock}` : statusBlock; +} + function shouldIncludeSkillTools(config: AgentConfig): boolean { return config.skills !== false && (!Array.isArray(config.skills) || config.skills.length > 0); } @@ -130,6 +173,7 @@ export async function prepareAgentRuntimeStep( toolContext.activeSkillToolAvailability = input.activeSkillToolAvailability; } + let integrationToolDiscovery: RemoteIntegrationToolDiscoveryResult | undefined; let tools = input.supportsToolCalling ? await input.getAvailableTools(input.config.tools, { callerAgentId: input.agentId, @@ -138,6 +182,9 @@ export async function prepareAgentRuntimeStep( forwardedRemoteToolDefinitions: input.forwardedRemoteToolDefinitions, remoteToolSources: input.remoteToolSources, remoteToolContext: toolContext, + onIntegrationToolDiscovery: (result) => { + integrationToolDiscovery = result; + }, sourceIntegrationPolicy: input.sourceIntegrationPolicy, strictConfiguredToolsOnly: input.strictConfiguredToolsOnly, }) @@ -159,6 +206,18 @@ export async function prepareAgentRuntimeStep( if (excludedToolNames !== undefined) { tools = tools.filter((tool) => !excludedToolNames.has(tool.name)); } + if ( + integrationToolDiscovery?.status === "unavailable" && + input.forwardedRemoteToolDefinitions?.length + ) { + const forwardedToolNames = new Set( + input.forwardedRemoteToolDefinitions.map((tool) => tool.name), + ); + const usableForwardedTools = tools.filter((tool) => forwardedToolNames.has(tool.name)); + if (usableForwardedTools.length > 0) { + integrationToolDiscovery = { status: "ok", tools: usableForwardedTools }; + } + } const toolExposureState = input.toolExposureState ?? createToolExposureState(); if (input.toolExposureCheckpoint) { const restoredState = restoreToolExposureState(input.toolExposureCheckpoint, tools); @@ -173,10 +232,11 @@ export async function prepareAgentRuntimeStep( state: toolExposureState, maxVisibleTools: getProviderToolProfile(input.effectiveModel ?? input.config.model).maxTools, }); - const systemPrompt = hasRuntimeToolInventory(runtimeState.systemPrompt) + const baseSystemPrompt = removeIntegrationToolDiscoveryStatus(runtimeState.systemPrompt); + const systemPrompt = hasRuntimeToolInventory(baseSystemPrompt) ? flattenSystemInstructions( withRuntimeToolInventory( - runtimeState.systemPrompt, + baseSystemPrompt, [...toolExposurePlan.visible.map((tool) => tool.name), ...(input.providerToolNames ?? [])] .filter((name, index, names) => names.indexOf(name) === index) .sort(), @@ -188,9 +248,10 @@ export async function prepareAgentRuntimeStep( })), ), ) - : runtimeState.systemPrompt; + : baseSystemPrompt; return { + integrationToolDiscovery, runtimeContext: trustedAllowedSkillIds === undefined ? runtimeState.context : { ...runtimeState.context, allowedSkillIds: [...trustedAllowedSkillIds] }, diff --git a/src/agent/runtime/index.ts b/src/agent/runtime/index.ts index 3ddfd85a98..8ff222ebf1 100644 --- a/src/agent/runtime/index.ts +++ b/src/agent/runtime/index.ts @@ -96,7 +96,11 @@ import { applySourceIntegrationPolicy, type SourceIntegrationPolicyManifest, } from "#veryfront/integrations/source-policy.ts"; -import { prepareAgentRuntimeStep } from "./agent-runtime-step.ts"; +import { runWithRemoteIntegrationToolDiscoveryScope } from "#veryfront/integrations/remote-tools.ts"; +import { + prepareAgentRuntimeStep, + withIntegrationToolDiscoveryStatus, +} from "./agent-runtime-step.ts"; import { buildStreamedAssistantMessage } from "./streamed-assistant-message.ts"; import { flattenSystemInstructions, @@ -819,27 +823,29 @@ export class AgentRuntime { return chain.execute( agentContext, () => - this.executeAgentLoop( - systemPrompt, - messages, - { - agentId: this.id, - projectId: tryGetCacheKeyContext()?.projectId, - }, - context, - supportsToolCalling, - resolvedModelString, - transport.languageModel, - transport.headers, - transport.providerOptions, - transport.reasoning, - maxOutputTokensOverride, - requestedModel, - this.createGenerateReplacementTools( - options?.toolReplacements, - options?.retainSkillLoaderTools, - ), - abortSignal, + runWithRemoteIntegrationToolDiscoveryScope(() => + this.executeAgentLoop( + systemPrompt, + messages, + { + agentId: this.id, + projectId: tryGetCacheKeyContext()?.projectId, + }, + context, + supportsToolCalling, + resolvedModelString, + transport.languageModel, + transport.headers, + transport.providerOptions, + transport.reasoning, + maxOutputTokensOverride, + requestedModel, + this.createGenerateReplacementTools( + options?.toolReplacements, + options?.retainSkillLoaderTools, + ), + abortSignal, + ) ), ); }); @@ -946,24 +952,26 @@ export class AgentRuntime { inFlight = chain.execute( agentContext, () => - this.executeAgentLoopStreaming( - systemPrompt, - memoryMessages, - controller, - encoder, - callbacks, - textPartId, - toolContext, - context, - supportsToolCalling, - resolvedModelString, - languageModel, - transport.headers, - transport.providerOptions, - transport.reasoning, - maxOutputTokensOverride, - streamAbortSignal, - requestedModel, + runWithRemoteIntegrationToolDiscoveryScope(() => + this.executeAgentLoopStreaming( + systemPrompt, + memoryMessages, + controller, + encoder, + callbacks, + textPartId, + toolContext, + context, + supportsToolCalling, + resolvedModelString, + languageModel, + transport.headers, + transport.providerOptions, + transport.reasoning, + maxOutputTokensOverride, + streamAbortSignal, + requestedModel, + ) ), ); const response = await inFlight; @@ -1164,7 +1172,10 @@ export class AgentRuntime { model: effectiveModel, providerTools: stepProviderTools, }); - currentSystemPrompt = synchronizeRuntimeToolInventory(currentSystemPrompt, runtimeTools); + currentSystemPrompt = withIntegrationToolDiscoveryStatus( + synchronizeRuntimeToolInventory(currentSystemPrompt, runtimeTools), + preparedStep.integrationToolDiscovery, + ); const response = await withSpan("agent.generate_text", async (span) => { setSpanAttributes(span, { "model.id": effectiveModel, @@ -1749,7 +1760,10 @@ export class AgentRuntime { model: effectiveModel, providerTools: stepProviderTools, }); - currentSystemPrompt = synchronizeRuntimeToolInventory(currentSystemPrompt, runtimeTools); + currentSystemPrompt = withIntegrationToolDiscoveryStatus( + synchronizeRuntimeToolInventory(currentSystemPrompt, runtimeTools), + preparedStep.integrationToolDiscovery, + ); const runtimeToolNames = Object.keys(runtimeTools ?? {}).sort(); const temperature = this.resolveTemperature( diff --git a/src/agent/runtime/tool-helpers.ts b/src/agent/runtime/tool-helpers.ts index c18a5df5e2..dcc9038446 100644 --- a/src/agent/runtime/tool-helpers.ts +++ b/src/agent/runtime/tool-helpers.ts @@ -17,6 +17,7 @@ import { executeRemoteIntegrationTool, isRemoteIntegrationTool, } from "#veryfront/integrations/remote-tools.ts"; +import type { RemoteIntegrationToolDiscoveryResult } from "#veryfront/integrations/remote-tools.ts"; import { isIntegrationToolAllowedBySourcePolicy, type SourceIntegrationPolicyManifest, @@ -172,6 +173,7 @@ async function getRemoteToolDefinitions(options?: { allowedRemoteToolNames?: string[]; remoteToolSources?: RemoteToolSource[]; remoteToolContext?: ToolExecutionContext; + onIntegrationToolDiscovery?: (result: RemoteIntegrationToolDiscoveryResult) => void; }): Promise { const remoteToolContext = options?.remoteToolContext; const definitions: ToolDefinition[] = []; @@ -210,11 +212,15 @@ async function getRemoteToolDefinitions(options?: { } try { - const { getRemoteIntegrationToolDefinitions } = await import( + const { getRemoteIntegrationToolDiscovery } = await import( "#veryfront/integrations/remote-tools.ts" ); - for (const def of await getRemoteIntegrationToolDefinitions(remoteToolContext)) { - addDefinition(def); + const discovery = await getRemoteIntegrationToolDiscovery(remoteToolContext); + options?.onIntegrationToolDiscovery?.(discovery); + if (discovery.status === "ok") { + for (const def of discovery.tools) { + addDefinition(def); + } } } catch { return definitions; @@ -428,6 +434,7 @@ export async function getAvailableTools( forwardedRemoteToolDefinitions?: ToolDefinition[]; remoteToolSources?: RemoteToolSource[]; remoteToolContext?: ToolExecutionContext; + onIntegrationToolDiscovery?: (result: RemoteIntegrationToolDiscoveryResult) => void; sourceIntegrationPolicy?: SourceIntegrationPolicyManifest; strictConfiguredToolsOnly?: boolean; /** Calling agent id for owner-aware tool visibility. */ diff --git a/src/integrations/index.ts b/src/integrations/index.ts index e111529daf..d7e33851ec 100644 --- a/src/integrations/index.ts +++ b/src/integrations/index.ts @@ -7,6 +7,7 @@ * import { * getConnector, * getIcon, + * getRemoteIntegrationToolDiscovery, * getRemoteIntegrationToolDefinitions, * listConnectors, * } from "veryfront/integrations"; @@ -14,6 +15,7 @@ * const connectors = listConnectors(); * const slack = getConnector("slack"); * const slackIcon = getIcon("slack"); // raw SVG string + * const discovery = await getRemoteIntegrationToolDiscovery(); * const runtimeTools = await getRemoteIntegrationToolDefinitions(); * ``` */ @@ -71,6 +73,8 @@ export function getIcon(name: IntegrationName | string): string | undefined { export { executeRemoteIntegrationTool, getRemoteIntegrationToolDefinitions, + getRemoteIntegrationToolDiscovery, isRemoteIntegrationTool, } from "./remote-tools.ts"; +export type { RemoteIntegrationToolDiscoveryResult } from "./remote-tools.ts"; export type { IntegrationConnector, IntegrationTool } from "./types.ts"; diff --git a/src/integrations/remote-tools.test.ts b/src/integrations/remote-tools.test.ts index 900316f2bf..27a655709c 100644 --- a/src/integrations/remote-tools.test.ts +++ b/src/integrations/remote-tools.test.ts @@ -10,7 +10,9 @@ import { withMockFetch } from "#veryfront/testing/mock-fetch.ts"; import { executeRemoteIntegrationTool, getRemoteIntegrationToolDefinitions, + getRemoteIntegrationToolDiscovery, isRemoteIntegrationTool, + runWithRemoteIntegrationToolDiscoveryScope, } from "./remote-tools.ts"; const ENV_KEYS = [ @@ -76,6 +78,59 @@ describe("integrations/remote-tools", () => { assertEquals(definitions, []); }); + it("memoizes a typed empty integration catalog for the current run", async () => { + setRemoteToolEnv({ + VERYFRONT_API_BASE_URL: "https://api.test", + VERYFRONT_API_TOKEN: "env-token", + }); + + let fetchCalls = 0; + const results = await withMockFetch(async () => { + fetchCalls++; + return Response.json({ tools: [] }); + }, () => + runWithRemoteIntegrationToolDiscoveryScope(async () => [ + await getRemoteIntegrationToolDiscovery(), + await getRemoteIntegrationToolDiscovery(), + ])); + + assertEquals(fetchCalls, 1); + assertEquals(results, [ + { status: "ok", tools: [] }, + { status: "ok", tools: [] }, + ]); + }); + + it("caches a transient failure for the current run and retries the next run", async () => { + setRemoteToolEnv({ + VERYFRONT_API_BASE_URL: "https://api.test", + VERYFRONT_API_TOKEN: "env-token", + }); + + let fetchCalls = 0; + const outcome = await withMockFetch(async () => { + fetchCalls++; + return fetchCalls === 1 + ? new Response(undefined, { status: 503, statusText: "Service Unavailable" }) + : Response.json({ tools: [] }); + }, async () => ({ + currentRun: await runWithRemoteIntegrationToolDiscoveryScope(async () => [ + await getRemoteIntegrationToolDiscovery(), + await getRemoteIntegrationToolDiscovery(), + ]), + nextRun: await runWithRemoteIntegrationToolDiscoveryScope(() => + getRemoteIntegrationToolDiscovery() + ), + })); + + assertEquals(fetchCalls, 2); + assertEquals(outcome.currentRun, [ + { status: "unavailable", reason: "request_failed" }, + { status: "unavailable", reason: "request_failed" }, + ]); + assertEquals(outcome.nextRun, { status: "ok", tools: [] }); + }); + it("prefers the request-scoped token and normalizes empty input schemas", async () => { setRemoteToolEnv({ PROXY_MODE: "1", diff --git a/src/integrations/remote-tools.ts b/src/integrations/remote-tools.ts index c2c059e1dc..99c0088a21 100644 --- a/src/integrations/remote-tools.ts +++ b/src/integrations/remote-tools.ts @@ -11,6 +11,7 @@ import { getApiBaseUrlEnv, getApiTokenEnv } from "#veryfront/config/env.ts"; import { getEnvironmentConfig } from "#veryfront/config/environment-config.ts"; +import { AsyncLocalStorage } from "#veryfront/platform/compat/async-context.ts"; import { getActiveSourceIntegrationPolicy } from "#veryfront/integrations/source-policy-context.ts"; import { isIntegrationToolAllowedBySourcePolicy, @@ -65,6 +66,34 @@ interface RemoteIntegrationExecutionContext { readonly abortSignal: AbortSignal | undefined; } +/** Result of listing the integration tools available to the current run. */ +export type RemoteIntegrationToolDiscoveryResult = + | { readonly status: "ok"; readonly tools: ToolDefinition[] } + | { readonly status: "unavailable"; readonly reason: "request_failed" }; + +type RemoteIntegrationToolCatalogResult = + | { readonly status: "ok"; readonly tools: RemoteToolDefinition[] } + | { readonly status: "unavailable"; readonly reason: "request_failed" }; + +interface RemoteIntegrationToolDiscoveryCacheEntry { + readonly baseUrl: string; + readonly token: string; + readonly projectSlug: string | undefined; + readonly result: Promise; +} + +interface RemoteIntegrationToolDiscoveryScope { + entry?: RemoteIntegrationToolDiscoveryCacheEntry; +} + +const remoteIntegrationToolDiscoveryStorage = new AsyncLocalStorage< + RemoteIntegrationToolDiscoveryScope +>(); +const requestIntegrationToolDiscoveryScopes = new WeakMap< + object, + RemoteIntegrationToolDiscoveryScope +>(); + const utf8Encoder = new TextEncoder(); const EMPTY_REMOTE_INTEGRATION_CONTEXT: RemoteIntegrationExecutionContext = Object.freeze({ hasExplicitCredential: false, @@ -75,6 +104,30 @@ const EMPTY_REMOTE_INTEGRATION_CONTEXT: RemoteIntegrationExecutionContext = Obje abortSignal: undefined, }); +/** Run a callback with one integration-tool discovery result shared by all continuations. */ +export function runWithRemoteIntegrationToolDiscoveryScope( + callback: () => Promise, +): Promise { + return remoteIntegrationToolDiscoveryStorage.run({}, callback); +} + +function getRemoteIntegrationToolDiscoveryScope(): + | RemoteIntegrationToolDiscoveryScope + | undefined { + const runScope = remoteIntegrationToolDiscoveryStorage.getStore(); + if (runScope) return runScope; + + const requestContext = getCurrentRequestContext(); + if (!requestContext) return undefined; + + let requestScope = requestIntegrationToolDiscoveryScopes.get(requestContext); + if (!requestScope) { + requestScope = {}; + requestIntegrationToolDiscoveryScopes.set(requestContext, requestScope); + } + return requestScope; +} + function snapshotToolExecutionContext( context: ToolExecutionContext | undefined, includeCallMetadata: boolean, @@ -526,6 +579,57 @@ async function fetchToolList( } } +async function discoverRemoteIntegrationToolCatalog( + baseUrl: string, + token: string, + context: RemoteIntegrationExecutionContext, +): Promise { + try { + return { + status: "ok", + tools: await fetchToolList(baseUrl, token, context), + }; + } catch (err) { + context.abortSignal?.throwIfAborted(); + logger.error("Failed to fetch remote integration tool definitions", { + error: err instanceof Error ? err.message : String(err), + }); + return { status: "unavailable", reason: "request_failed" }; + } +} + +function getRemoteIntegrationToolCatalog( + baseUrl: string, + token: string, + projectSlug: string | undefined, + context: RemoteIntegrationExecutionContext, +): Promise { + const scope = getRemoteIntegrationToolDiscoveryScope(); + const cached = scope?.entry; + if ( + cached?.baseUrl === baseUrl && + cached.token === token && + cached.projectSlug === projectSlug + ) { + return cached.result; + } + + const result = discoverRemoteIntegrationToolCatalog(baseUrl, token, context); + if (scope) { + const entry: RemoteIntegrationToolDiscoveryCacheEntry = { + baseUrl, + token, + projectSlug, + result, + }; + scope.entry = entry; + void result.catch(() => { + if (scope.entry === entry) scope.entry = undefined; + }); + } + return result; +} + async function callRemoteTool( baseUrl: string, token: string, @@ -606,48 +710,70 @@ async function callRemoteTool( // --------------------------------------------------------------------------- /** - * Fetch integration tool definitions for the current request context. - * Returns ToolDefinition[] that the agent runtime merges into the model's - * available tools. Returns empty array if no API config or no tools. + * Discover integration tools for the current request context. * - * Called per agent loop iteration — results are scoped to the current - * project's authorized integration tools via the per-request API token. - * Caller cancellation is propagated; remote failures and malformed or - * over-limit catalogs fail closed to an empty list. + * A successful empty catalog returns `status: "ok"`. Request, protocol, and + * response failures return `status: "unavailable"` so callers do not mistake + * a failed lookup for a project with no integration tools. The agent runtime + * memoizes both outcomes for the current run. Direct callers inside a + * Veryfront request receive the same request-scoped behavior. */ -export async function getRemoteIntegrationToolDefinitions( +export async function getRemoteIntegrationToolDiscovery( context?: ToolExecutionContext, -): Promise< - ToolDefinition[] -> { +): Promise { const requestContext = snapshotToolExecutionContext(context, false); requestContext.abortSignal?.throwIfAborted(); const baseUrl = getApiBaseUrlEnv(); const token = resolveRequestToken(requestContext); - if (!baseUrl || !token) return []; + if (!baseUrl || !token) return { status: "ok", tools: [] }; try { - const remoteDefs = await fetchToolList(baseUrl, token, requestContext); + const projectSlug = resolveRequestProjectSlug(requestContext); + const catalog = await getRemoteIntegrationToolCatalog( + baseUrl, + token, + projectSlug, + requestContext, + ); + if (catalog.status === "unavailable") return catalog; + const sourceIntegrationPolicy = getActiveSourceIntegrationPolicy(); - return remoteDefs.filter((def) => - sourceIntegrationPolicy === undefined || - isIntegrationToolAllowedBySourcePolicy(def.name, sourceIntegrationPolicy) - ).map((def) => ({ - name: def.name, - description: def.description, - parameters: def.inputSchema && Object.keys(def.inputSchema).length > 0 - ? def.inputSchema - : { type: "object", properties: {} }, - })); + return { + status: "ok", + tools: catalog.tools.filter((def) => + sourceIntegrationPolicy === undefined || + isIntegrationToolAllowedBySourcePolicy(def.name, sourceIntegrationPolicy) + ).map((def) => ({ + name: def.name, + description: def.description, + parameters: def.inputSchema && Object.keys(def.inputSchema).length > 0 + ? def.inputSchema + : { type: "object", properties: {} }, + })), + }; } catch (err) { requestContext.abortSignal?.throwIfAborted(); logger.error("Failed to fetch remote integration tool definitions", { error: err instanceof Error ? err.message : String(err), }); - return []; + return { status: "unavailable", reason: "request_failed" }; } } +/** + * Fetch integration tool definitions for the current request context. + * + * This compatibility helper returns an empty array for unavailable catalogs. + * Use `getRemoteIntegrationToolDiscovery` when the caller must distinguish a + * successful empty catalog from a discovery failure. + */ +export async function getRemoteIntegrationToolDefinitions( + context?: ToolExecutionContext, +): Promise { + const discovery = await getRemoteIntegrationToolDiscovery(context); + return discovery.status === "ok" ? discovery.tools : []; +} + /** * Check if a tool name looks like a remote integration tool. * Integration tools use "integration__tool_id" format (double underscore separator).