test(server-utils): Cover the Flue instrumentation

dev-packages/node-integration-tests/suites/tracing/flue/instrument.mjs

@@ -0,0 +1,11 @@
+import * as Sentry from '@sentry/node';
+import { loggingTransport } from '@sentry-internal/node-integration-tests';
+
+Sentry.init({
+  traceLifecycle: 'stream',
+  dsn: 'https://public@dsn.ingest.sentry.io/1337',
+  release: '1.0',
+  tracesSampleRate: 1.0,
+  dataCollection: { genAI: { inputs: false, outputs: false } },
+  transport: loggingTransport,
+});

dev-packages/node-integration-tests/suites/tracing/flue/scenario.mjs

@@ -0,0 +1,42 @@
+import * as Sentry from '@sentry/node';
+import { __flueBindAgentModule, init, instrument, useModel, useTool } from '@flue/runtime';
+import { start } from '@flue/runtime/node';
+import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux';
+import * as v from 'valibot';
+
+// `pi-ai`'s faux provider scripts model responses in-process, so the run is deterministic and needs
+// no provider key or mock server. Two steps: a tool call, then the final answer.
+instrument(Sentry.createFlueInstrumentation());
+
+const faux = fauxProvider({
+  provider: 'faux',
+  models: [{ id: 'faux-model', cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 } }],
+});
+faux.setResponses([
+  fauxAssistantMessage(fauxToolCall('get_weather', { city: 'Berlin' }, { id: 'call_1' }), { stopReason: 'toolUse' }),
+  fauxAssistantMessage('It is 21 degrees and sunny in Berlin.'),
+]);
+
+function Hello() {
+  useModel('faux/faux-model');
+  useTool({
+    name: 'get_weather',
+    description: 'Get the current weather for a city.',
+    // Without an `input` schema Flue validates the call against an empty one and rejects the
+    // model's arguments, so the tool never runs and its span settles as an error.
+    input: v.object({ city: v.string() }),
+    run: ({ data }) => `It is 21 degrees and sunny in ${data.city}.`,
+  });
+  return 'You are a helpful assistant.';
+}
+__flueBindAgentModule(Hello, { identity: 'Hello' });
+
+await Sentry.startSpan({ name: 'flue-test', op: 'function' }, async () => {
+  const flue = await start({ agents: [Hello], providers: [faux.provider] });
+  const agent = init(Hello, { id: 'e2e' });
+  const receipt = await agent.dispatch('What is the weather in Berlin?');
+  await agent.read(receipt);
+  await flue[Symbol.asyncDispose]?.();
+});
+
+await Sentry.flush(2000);

dev-packages/node-integration-tests/suites/tracing/flue/test.ts

@@ -0,0 +1,114 @@
+import {
+  GEN_AI_AGENT_NAME,
+  GEN_AI_CONVERSATION_ID,
+  GEN_AI_COST_TOTAL_TOKENS,
+  GEN_AI_OPERATION_NAME,
+  GEN_AI_RESPONSE_FINISH_REASONS,
+  GEN_AI_TOOL_NAME,
+  GEN_AI_USAGE_INPUT_TOKENS,
+  GEN_AI_USAGE_OUTPUT_TOKENS,
+  GEN_AI_USAGE_TOTAL_TOKENS,
+} from '@sentry/conventions/attributes';
+import { afterAll, expect } from 'vitest';
+import { conditionalTest } from '../../../utils';
+import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner';
+
+// `@flue/runtime` declares `engines.node >= 22.19`, so it can't live in the package's root
+// `devDependencies` (that would break `yarn install` on the 20.19 CI matrix). Install it per-suite
+// instead, guarded by the `min: 22` skip below.
+const FLUE_DEPENDENCIES = {
+  additionalDependencies: {
+    '@flue/runtime': '2.0.3',
+    '@earendil-works/pi-ai': '0.85.1',
+    valibot: '1.1.0',
+  },
+};
+
+conditionalTest({ min: 22 })('Flue integration', () => {
+  afterAll(() => {
+    cleanupChildProcesses();
+  });
+
+  createEsmAndCjsTests(
+    __dirname,
+    'scenario.mjs',
+    'instrument.mjs',
+    (createRunner, test, mode) => {
+      // `@flue/runtime` is ESM-only — its `exports` map has no `require` condition, so there is no
+      // CJS variant of this scenario to run.
+      if (mode === 'cjs') {
+        return;
+      }
+
+      test('creates the invoke_agent / chat / execute_tool hierarchy', async () => {
+        await createRunner()
+          .expect({
+            span: container => {
+              const spans = container.items;
+
+              const root = spans.find(span => span.name === 'flue-test')!;
+              expect(root.is_segment).toBe(true);
+
+              // Counted rather than looked up: the interceptor skips the submission wrapper
+              // operation, so one dispatch opens exactly one agent span, and each turn and tool call
+              // is spanned once. `find` passes just as happily on a duplicate.
+              expect(spans.filter(span => span.attributes['sentry.origin']?.value === 'auto.ai.flue')).toHaveLength(4);
+
+              const agents = spans.filter(span => span.name === 'invoke_agent Hello');
+              const chats = spans.filter(span => span.name === 'chat faux-model');
+              const tools = spans.filter(span => span.name === 'execute_tool get_weather');
+              expect(agents).toHaveLength(1);
+              expect(tools).toHaveLength(1);
+              // One turn asks for the tool, the second answers with its result.
+              expect(chats).toHaveLength(2);
+
+              const agent = agents[0]!;
+              expect(agent.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent');
+              expect(agent.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('invoke_agent');
+              expect(agent.attributes[GEN_AI_AGENT_NAME]?.value).toBe('Hello');
+              expect(agent.parent_span_id).toBe(root.span_id);
+
+              const conversationId = agent.attributes[GEN_AI_CONVERSATION_ID]?.value;
+              expect(conversationId).toEqual(expect.any(String));
+
+              // Both turns, not just the first: they leave the provider by different paths (a tool
+              // call, then a final answer) and resolve their parent through separate tracker lookups.
+              for (const chat of chats) {
+                expect(chat.attributes['sentry.op']?.value).toBe('gen_ai.chat');
+                expect(chat.attributes['sentry.origin']?.value).toBe('auto.ai.flue');
+                expect(chat.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('chat');
+                expect(chat.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(conversationId);
+                expect(chat.parent_span_id).toBe(agent.span_id);
+                expect(chat.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBeGreaterThan(0);
+                expect(chat.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBeGreaterThan(0);
+                expect(chat.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBeGreaterThan(0);
+                // Flue computes cost itself; no provider SDK reports it. The faux provider prices
+                // every model at zero, so this only proves the attribute is mapped.
+                expect(chat.attributes[GEN_AI_COST_TOTAL_TOKENS]?.value).toEqual(expect.any(Number));
+              }
+
+              expect(chats.map(chat => chat.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value).sort()).toEqual([
+                '["stop"]',
+                '["toolUse"]',
+              ]);
+
+              const tool = tools[0]!;
+              // The tool has to actually run: a schema mismatch still produces a correctly named and
+              // parented span, so only the status separates a real call from a rejected one.
+              expect(tool.status).toBe('ok');
+              expect(tool.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool');
+              expect(tool.attributes['sentry.origin']?.value).toBe('auto.ai.flue');
+              expect(tool.attributes[GEN_AI_TOOL_NAME]?.value).toBe('get_weather');
+
+              // Tool spans are siblings of `chat` under the agent invocation, matching how Flue's
+              // own OpenTelemetry adapter projects them.
+              expect(tool.parent_span_id).toBe(agent.span_id);
+            },
+          })
+          .start()
+          .completed();
+      });
+    },
+    FLUE_DEPENDENCIES,
+  );
+});