Skip to content

Commit 8c6f644

Browse files
RulaKhaledRulaKhaled
committed
test(node): enable additionalDependencies in integration runner
1 parent ccc7d32 commit 8c6f644

File tree

3 files changed

+245
-20
lines changed

3 files changed

+245
-20
lines changed

‎dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-v5.mjs‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
import * as Sentry from '@sentry/node';
2+
import { generateText } from 'ai';
3+
import { MockLanguageModelV2 } from 'ai/test';
4+
import { z } from 'zod';
5+
6+
async function run() {
7+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
8+
await generateText({
9+
model: new MockLanguageModelV2({
10+
doGenerate: async () => ({
11+
finishReason: 'stop',
12+
usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
13+
content: [{ type: 'text', text: 'First span here!' }],
14+
}),
15+
}),
16+
prompt: 'Where is the first span?',
17+
});
18+
19+
// This span should have input and output prompts attached because telemetry is explicitly enabled.
20+
await generateText({
21+
experimental_telemetry: { isEnabled: true },
22+
model: new MockLanguageModelV2({
23+
doGenerate: async () => ({
24+
finishReason: 'stop',
25+
usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
26+
content: [{ type: 'text', text: 'Second span here!' }],
27+
}),
28+
}),
29+
prompt: 'Where is the second span?',
30+
});
31+
32+
// This span should include tool calls and tool results
33+
await generateText({
34+
model: new MockLanguageModelV2({
35+
doGenerate: async () => ({
36+
finishReason: 'tool-calls',
37+
usage: { inputTokens: 15, outputTokens: 25, totalTokens: 40 },
38+
content: [{ type: 'text', text: 'Tool call completed!' }],
39+
toolCalls: [
40+
{
41+
toolCallType: 'function',
42+
toolCallId: 'call-1',
43+
toolName: 'getWeather',
44+
args: '{ "location": "San Francisco" }',
45+
},
46+
],
47+
}),
48+
}),
49+
tools: {
50+
getWeather: {
51+
parameters: z.object({ location: z.string() }),
52+
execute: async args => {
53+
return `Weather in ${args.location}: Sunny, 72°F`;
54+
},
55+
},
56+
},
57+
prompt: 'What is the weather in San Francisco?',
58+
});
59+
60+
// This span should not be captured because we've disabled telemetry
61+
await generateText({
62+
experimental_telemetry: { isEnabled: false },
63+
model: new MockLanguageModelV2({
64+
doGenerate: async () => ({
65+
finishReason: 'stop',
66+
usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
67+
content: [{ type: 'text', text: 'Third span here!' }],
68+
}),
69+
}),
70+
prompt: 'Where is the third span?',
71+
});
72+
});
73+
}
74+
75+
run();

‎dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,73 @@ describe('Vercel AI integration', () => {
197197
]),
198198
};
199199

200+
// Todo: Add missing attribute spans for v5
201+
// Right now only second span is recorded as it's manually opted in via explicit telemetry option
202+
const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 = {
203+
transaction: 'main',
204+
spans: expect.arrayContaining([
205+
expect.objectContaining({
206+
data: {
207+
'vercel.ai.model.id': 'mock-model-id',
208+
'vercel.ai.model.provider': 'mock-provider',
209+
'vercel.ai.operationId': 'ai.generateText',
210+
'vercel.ai.pipeline.name': 'generateText',
211+
'vercel.ai.prompt': '{"prompt":"Where is the second span?"}',
212+
'vercel.ai.response.finishReason': 'stop',
213+
'gen_ai.response.text': expect.any(String),
214+
'vercel.ai.settings.maxRetries': 2,
215+
// 'vercel.ai.settings.maxSteps': 1,
216+
'vercel.ai.streaming': false,
217+
'gen_ai.prompt': '{"prompt":"Where is the second span?"}',
218+
'gen_ai.response.model': 'mock-model-id',
219+
'gen_ai.usage.input_tokens': 10,
220+
'gen_ai.usage.output_tokens': 20,
221+
'gen_ai.usage.total_tokens': 30,
222+
'operation.name': 'ai.generateText',
223+
'sentry.op': 'gen_ai.invoke_agent',
224+
'sentry.origin': 'auto.vercelai.otel',
225+
},
226+
description: 'generateText',
227+
op: 'gen_ai.invoke_agent',
228+
origin: 'auto.vercelai.otel',
229+
status: 'ok',
230+
}),
231+
// doGenerate
232+
expect.objectContaining({
233+
data: {
234+
'sentry.origin': 'auto.vercelai.otel',
235+
'sentry.op': 'gen_ai.generate_text',
236+
'operation.name': 'ai.generateText.doGenerate',
237+
'vercel.ai.operationId': 'ai.generateText.doGenerate',
238+
'vercel.ai.model.provider': 'mock-provider',
239+
'vercel.ai.model.id': 'mock-model-id',
240+
'vercel.ai.settings.maxRetries': 2,
241+
'gen_ai.system': 'mock-provider',
242+
'gen_ai.request.model': 'mock-model-id',
243+
'vercel.ai.pipeline.name': 'generateText.doGenerate',
244+
'vercel.ai.streaming': false,
245+
'vercel.ai.response.finishReason': 'stop',
246+
'vercel.ai.response.model': 'mock-model-id',
247+
'vercel.ai.response.id': expect.any(String),
248+
'gen_ai.response.text': 'Second span here!',
249+
'vercel.ai.response.timestamp': expect.any(String),
250+
// 'vercel.ai.prompt.format': expect.any(String),
251+
'gen_ai.request.messages': expect.any(String),
252+
'gen_ai.response.finish_reasons': ['stop'],
253+
'gen_ai.usage.input_tokens': 10,
254+
'gen_ai.usage.output_tokens': 20,
255+
'gen_ai.response.id': expect.any(String),
256+
'gen_ai.response.model': 'mock-model-id',
257+
'gen_ai.usage.total_tokens': 30,
258+
},
259+
description: 'generate_text mock-model-id',
260+
op: 'gen_ai.generate_text',
261+
origin: 'auto.vercelai.otel',
262+
status: 'ok',
263+
}),
264+
]),
265+
};
266+
200267
const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = {
201268
transaction: 'main',
202269
spans: expect.arrayContaining([
@@ -538,6 +605,23 @@ describe('Vercel AI integration', () => {
538605
});
539606
});
540607

608+
// Test with specific Vercel AI v5 version
609+
createEsmAndCjsTests(
610+
__dirname,
611+
'scenario-v5.mjs',
612+
'instrument.mjs',
613+
(createRunner, test) => {
614+
test('creates ai related spans with v5', async () => {
615+
await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 }).start().completed();
616+
});
617+
},
618+
{
619+
additionalDependencies: {
620+
ai: '^5.0.0',
621+
},
622+
},
623+
);
624+
541625
createEsmAndCjsTests(__dirname, 'scenario-error-in-tool-express.mjs', 'instrument.mjs', (createRunner, test) => {
542626
test('captures error in tool in express server', async () => {
543627
const expectedTransaction = {

‎dev-packages/node-integration-tests/utils/runner.ts‎

Lines changed: 86 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ import type {
1414
import { normalize } from '@sentry/core';
1515
import { createBasicSentryServer } from '@sentry-internal/test-utils';
1616
import { execSync, spawn, spawnSync } from 'child_process';
17-
import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'fs';
18-
import { join } from 'path';
17+
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
18+
import { basename, join } from 'path';
1919
import { inspect } from 'util';
20-
import { afterAll, beforeAll, describe, test } from 'vitest';
20+
import { afterAll, describe, test } from 'vitest';
2121
import {
2222
assertEnvelopeHeader,
2323
assertSentryCheckIn,
@@ -174,7 +174,7 @@ export function createEsmAndCjsTests(
174174
testFn: typeof test | typeof test.fails,
175175
mode: 'esm' | 'cjs',
176176
) => void,
177-
options?: { failsOnCjs?: boolean; failsOnEsm?: boolean },
177+
options?: { failsOnCjs?: boolean; failsOnEsm?: boolean; additionalDependencies?: Record<string, string> },
178178
): void {
179179
const mjsScenarioPath = join(cwd, scenarioPath);
180180
const mjsInstrumentPath = join(cwd, instrumentPath);
@@ -187,31 +187,97 @@ export function createEsmAndCjsTests(
187187
throw new Error(`Instrument file not found: ${mjsInstrumentPath}`);
188188
}
189189

190-
const cjsScenarioPath = join(cwd, `tmp_${scenarioPath.replace('.mjs', '.cjs')}`);
191-
const cjsInstrumentPath = join(cwd, `tmp_${instrumentPath.replace('.mjs', '.cjs')}`);
190+
// Create a dedicated tmp directory that includes copied ESM & CJS scenario/instrument files.
191+
// If additionalDependencies are provided, we also create a nested package.json and install them there.
192+
const uniqueId = `${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
193+
const tmpDirPath = join(cwd, `tmp_${uniqueId}`);
194+
mkdirSync(tmpDirPath);
195+
196+
// Copy ESM files as-is into tmp dir
197+
const esmScenarioBasename = basename(scenarioPath);
198+
const esmInstrumentBasename = basename(instrumentPath);
199+
const esmScenarioPathForRun = join(tmpDirPath, esmScenarioBasename);
200+
const esmInstrumentPathForRun = join(tmpDirPath, esmInstrumentBasename);
201+
writeFileSync(esmScenarioPathForRun, readFileSync(mjsScenarioPath, 'utf8'));
202+
writeFileSync(esmInstrumentPathForRun, readFileSync(mjsInstrumentPath, 'utf8'));
203+
204+
// Pre-create CJS converted files inside tmp dir
205+
const cjsScenarioPath = join(tmpDirPath, esmScenarioBasename.replace('.mjs', '.cjs'));
206+
const cjsInstrumentPath = join(tmpDirPath, esmInstrumentBasename.replace('.mjs', '.cjs'));
207+
convertEsmFileToCjs(esmScenarioPathForRun, cjsScenarioPath);
208+
convertEsmFileToCjs(esmInstrumentPathForRun, cjsInstrumentPath);
209+
210+
// Create a minimal package.json with requested dependencies (if any) and install them
211+
const additionalDependencies = options?.additionalDependencies ?? {};
212+
if (Object.keys(additionalDependencies).length > 0) {
213+
const packageJson = {
214+
name: 'tmp-integration-test',
215+
private: true,
216+
version: '0.0.0',
217+
dependencies: additionalDependencies,
218+
} as const;
219+
220+
writeFileSync(join(tmpDirPath, 'package.json'), JSON.stringify(packageJson, null, 2));
221+
222+
try {
223+
const deps = Object.entries(additionalDependencies).map(([name, range]) => {
224+
if (!range || typeof range !== 'string') {
225+
throw new Error(`Invalid version range for "${name}": ${String(range)}`);
226+
}
227+
return `${name}@${range}`;
228+
});
229+
230+
if (deps.length > 0) {
231+
// --ignore-engines is needed to avoid engine mismatches when installing deps in the tmp dir
232+
// (e.g. Vercel AI v5 requires a package that requires Node >= 20 while the system Node is 18)
233+
// https://github.com/vercel/ai/issues/7777
234+
const result = spawnSync('yarn', ['add', '--non-interactive', '--ignore-engines', ...deps], {
235+
cwd: tmpDirPath,
236+
encoding: 'utf8',
237+
});
238+
239+
if (process.env.DEBUG) {
240+
// eslint-disable-next-line no-console
241+
console.log('[additionalDependencies]', deps.join(' '));
242+
// eslint-disable-next-line no-console
243+
console.log('[yarn stdout]', result.stdout);
244+
// eslint-disable-next-line no-console
245+
console.log('[yarn stderr]', result.stderr);
246+
}
247+
248+
if (result.error) {
249+
throw new Error(`Failed to install additionalDependencies in tmp dir ${tmpDirPath}: ${result.error.message}`);
250+
}
251+
if (typeof result.status === 'number' && result.status !== 0) {
252+
throw new Error(
253+
`Failed to install additionalDependencies in tmp dir ${tmpDirPath} (exit ${result.status}):\n${
254+
result.stderr || result.stdout || '(no output)'
255+
}`,
256+
);
257+
}
258+
}
259+
} catch (e) {
260+
// eslint-disable-next-line no-console
261+
console.error('Failed to install additionalDependencies:', e);
262+
throw e;
263+
}
264+
}
192265

193266
describe('esm', () => {
194267
const testFn = options?.failsOnEsm ? test.fails : test;
195-
callback(() => createRunner(mjsScenarioPath).withFlags('--import', mjsInstrumentPath), testFn, 'esm');
268+
callback(() => createRunner(esmScenarioPathForRun).withFlags('--import', esmInstrumentPathForRun), testFn, 'esm');
196269
});
197270

198271
describe('cjs', () => {
199-
beforeAll(() => {
200-
// For the CJS runner, we create some temporary files...
201-
convertEsmFileToCjs(mjsScenarioPath, cjsScenarioPath);
202-
convertEsmFileToCjs(mjsInstrumentPath, cjsInstrumentPath);
203-
});
204-
272+
// Clean up the tmp directory once CJS tests are finished
205273
afterAll(() => {
206274
try {
207-
unlinkSync(cjsInstrumentPath);
275+
rmSync(tmpDirPath, { recursive: true, force: true });
208276
} catch {
209-
// Ignore errors here
210-
}
211-
try {
212-
unlinkSync(cjsScenarioPath);
213-
} catch {
214-
// Ignore errors here
277+
if (process.env.DEBUG) {
278+
// eslint-disable-next-line no-console
279+
console.error(`Failed to remove tmp dir: ${tmpDirPath}`);
280+
}
215281
}
216282
});
217283

0 commit comments

Comments
 (0)