Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit f307a22

Browse files
RulaKhaledmydea
andauthored
test(node): Enable additionalDependencies in integration runner (#17361)
This update enhances the Node integration test runner to support per-scenario dependency overrides via a temporary folder that contains package.json. When additionalDependencies are provided, the runner now: 1. Creates a unique temp directory with a package.json containing the requested dependencies. 2. Copies the ESM and CJS versions of the scenario and instrument files into the temp directory. 3. Installs the specified dependency versions. 4. Runs tests ESM and CJS test modes continue to run normally using the files from the temp workspace. Also adds: - Minimal test scenario for vercel AI test using ai@^5.0.0. (Adjusted expectations to match the current v5 output format) --------- Co-authored-by: Francesco Novy <francesco.novy@sentry.io>
1 parent 06105b6 commit f307a22

File tree

4 files changed

+259
-26
lines changed

4 files changed

+259
-26
lines changed

‎dev-packages/node-integration-tests/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
"build:dev": "yarn build",
1515
"build:transpile": "rollup -c rollup.npm.config.mjs",
1616
"build:types": "tsc -p tsconfig.types.json",
17-
"clean": "rimraf -g **/node_modules && run-p clean:script",
17+
"clean": "rimraf -g suites/**/node_modules suites/**/tmp_* && run-p clean:script",
1818
"clean:script": "node scripts/clean.js",
1919
"lint": "eslint . --format stylish",
2020
"fix": "eslint . --format stylish --fix",
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
import * as Sentry from '@sentry/node';
2+
import { generateText } from 'ai';
3+
import { MockLanguageModelV2 } from 'ai/test';
4+
import { z } from 'zod';
5+
6+
async function run() {
7+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
8+
await generateText({
9+
model: new MockLanguageModelV2({
10+
doGenerate: async () => ({
11+
finishReason: 'stop',
12+
usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
13+
content: [{ type: 'text', text: 'First span here!' }],
14+
}),
15+
}),
16+
prompt: 'Where is the first span?',
17+
});
18+
19+
// This span should have input and output prompts attached because telemetry is explicitly enabled.
20+
await generateText({
21+
experimental_telemetry: { isEnabled: true },
22+
model: new MockLanguageModelV2({
23+
doGenerate: async () => ({
24+
finishReason: 'stop',
25+
usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
26+
content: [{ type: 'text', text: 'Second span here!' }],
27+
}),
28+
}),
29+
prompt: 'Where is the second span?',
30+
});
31+
32+
// This span should include tool calls and tool results
33+
await generateText({
34+
model: new MockLanguageModelV2({
35+
doGenerate: async () => ({
36+
finishReason: 'tool-calls',
37+
usage: { inputTokens: 15, outputTokens: 25, totalTokens: 40 },
38+
content: [{ type: 'text', text: 'Tool call completed!' }],
39+
toolCalls: [
40+
{
41+
toolCallType: 'function',
42+
toolCallId: 'call-1',
43+
toolName: 'getWeather',
44+
args: '{ "location": "San Francisco" }',
45+
},
46+
],
47+
}),
48+
}),
49+
tools: {
50+
getWeather: {
51+
parameters: z.object({ location: z.string() }),
52+
execute: async args => {
53+
return `Weather in ${args.location}: Sunny, 72°F`;
54+
},
55+
},
56+
},
57+
prompt: 'What is the weather in San Francisco?',
58+
});
59+
60+
// This span should not be captured because we've disabled telemetry
61+
await generateText({
62+
experimental_telemetry: { isEnabled: false },
63+
model: new MockLanguageModelV2({
64+
doGenerate: async () => ({
65+
finishReason: 'stop',
66+
usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
67+
content: [{ type: 'text', text: 'Third span here!' }],
68+
}),
69+
}),
70+
prompt: 'Where is the third span?',
71+
});
72+
});
73+
}
74+
75+
run();

‎dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,73 @@ describe('Vercel AI integration', () => {
197197
]),
198198
};
199199

200+
// Todo: Add missing attribute spans for v5
201+
// Right now only second span is recorded as it's manually opted in via explicit telemetry option
202+
const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 = {
203+
transaction: 'main',
204+
spans: expect.arrayContaining([
205+
expect.objectContaining({
206+
data: {
207+
'vercel.ai.model.id': 'mock-model-id',
208+
'vercel.ai.model.provider': 'mock-provider',
209+
'vercel.ai.operationId': 'ai.generateText',
210+
'vercel.ai.pipeline.name': 'generateText',
211+
'vercel.ai.prompt': '{"prompt":"Where is the second span?"}',
212+
'vercel.ai.response.finishReason': 'stop',
213+
'gen_ai.response.text': expect.any(String),
214+
'vercel.ai.settings.maxRetries': 2,
215+
// 'vercel.ai.settings.maxSteps': 1,
216+
'vercel.ai.streaming': false,
217+
'gen_ai.prompt': '{"prompt":"Where is the second span?"}',
218+
'gen_ai.response.model': 'mock-model-id',
219+
'gen_ai.usage.input_tokens': 10,
220+
'gen_ai.usage.output_tokens': 20,
221+
'gen_ai.usage.total_tokens': 30,
222+
'operation.name': 'ai.generateText',
223+
'sentry.op': 'gen_ai.invoke_agent',
224+
'sentry.origin': 'auto.vercelai.otel',
225+
},
226+
description: 'generateText',
227+
op: 'gen_ai.invoke_agent',
228+
origin: 'auto.vercelai.otel',
229+
status: 'ok',
230+
}),
231+
// doGenerate
232+
expect.objectContaining({
233+
data: {
234+
'sentry.origin': 'auto.vercelai.otel',
235+
'sentry.op': 'gen_ai.generate_text',
236+
'operation.name': 'ai.generateText.doGenerate',
237+
'vercel.ai.operationId': 'ai.generateText.doGenerate',
238+
'vercel.ai.model.provider': 'mock-provider',
239+
'vercel.ai.model.id': 'mock-model-id',
240+
'vercel.ai.settings.maxRetries': 2,
241+
'gen_ai.system': 'mock-provider',
242+
'gen_ai.request.model': 'mock-model-id',
243+
'vercel.ai.pipeline.name': 'generateText.doGenerate',
244+
'vercel.ai.streaming': false,
245+
'vercel.ai.response.finishReason': 'stop',
246+
'vercel.ai.response.model': 'mock-model-id',
247+
'vercel.ai.response.id': expect.any(String),
248+
'gen_ai.response.text': 'Second span here!',
249+
'vercel.ai.response.timestamp': expect.any(String),
250+
// 'vercel.ai.prompt.format': expect.any(String),
251+
'gen_ai.request.messages': expect.any(String),
252+
'gen_ai.response.finish_reasons': ['stop'],
253+
'gen_ai.usage.input_tokens': 10,
254+
'gen_ai.usage.output_tokens': 20,
255+
'gen_ai.response.id': expect.any(String),
256+
'gen_ai.response.model': 'mock-model-id',
257+
'gen_ai.usage.total_tokens': 30,
258+
},
259+
description: 'generate_text mock-model-id',
260+
op: 'gen_ai.generate_text',
261+
origin: 'auto.vercelai.otel',
262+
status: 'ok',
263+
}),
264+
]),
265+
};
266+
200267
const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = {
201268
transaction: 'main',
202269
spans: expect.arrayContaining([
@@ -538,6 +605,23 @@ describe('Vercel AI integration', () => {
538605
});
539606
});
540607

608+
// Test with specific Vercel AI v5 version
609+
createEsmAndCjsTests(
610+
__dirname,
611+
'scenario-v5.mjs',
612+
'instrument.mjs',
613+
(createRunner, test) => {
614+
test('creates ai related spans with v5', async () => {
615+
await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 }).start().completed();
616+
});
617+
},
618+
{
619+
additionalDependencies: {
620+
ai: '^5.0.0',
621+
},
622+
},
623+
);
624+
541625
createEsmAndCjsTests(__dirname, 'scenario-error-in-tool-express.mjs', 'instrument.mjs', (createRunner, test) => {
542626
test('captures error in tool in express server', async () => {
543627
const expectedTransaction = {

‎dev-packages/node-integration-tests/utils/runner.ts

Lines changed: 99 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ import type {
1414
import { normalize } from '@sentry/core';
1515
import { createBasicSentryServer } from '@sentry-internal/test-utils';
1616
import { execSync, spawn, spawnSync } from 'child_process';
17-
import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'fs';
18-
import { join } from 'path';
17+
import { existsSync, mkdirSync,readFileSync, rmSync, writeFileSync } from 'fs';
18+
import { basename,join } from 'path';
1919
import { inspect } from 'util';
20-
import { afterAll, beforeAll,describe, test } from 'vitest';
20+
import { afterAll, describe, test } from 'vitest';
2121
import {
2222
assertEnvelopeHeader,
2323
assertSentryCheckIn,
@@ -174,7 +174,10 @@ export function createEsmAndCjsTests(
174174
testFn: typeof test | typeof test.fails,
175175
mode: 'esm' | 'cjs',
176176
) => void,
177-
options?: { failsOnCjs?: boolean; failsOnEsm?: boolean },
177+
// `additionalDependencies` to install in a tmp dir for the esm and cjs tests
178+
// This could be used to override packages that live in the parent package.json for the specific run of the test
179+
// e.g. `{ ai: '^5.0.0' }` to test Vercel AI v5
180+
options?: { failsOnCjs?: boolean; failsOnEsm?: boolean; additionalDependencies?: Record<string, string> },
178181
): void {
179182
const mjsScenarioPath = join(cwd, scenarioPath);
180183
const mjsInstrumentPath = join(cwd, instrumentPath);
@@ -187,36 +190,107 @@ export function createEsmAndCjsTests(
187190
throw new Error(`Instrument file not found: ${mjsInstrumentPath}`);
188191
}
189192

190-
const cjsScenarioPath = join(cwd, `tmp_${scenarioPath.replace('.mjs', '.cjs')}`);
191-
const cjsInstrumentPath = join(cwd, `tmp_${instrumentPath.replace('.mjs', '.cjs')}`);
193+
// Create a dedicated tmp directory that includes copied ESM & CJS scenario/instrument files.
194+
// If additionalDependencies are provided, we also create a nested package.json and install them there.
195+
const uniqueId = `${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
196+
const tmpDirPath = join(cwd, `tmp_${uniqueId}`);
197+
mkdirSync(tmpDirPath);
198+
199+
// Copy ESM files as-is into tmp dir
200+
const esmScenarioBasename = basename(scenarioPath);
201+
const esmInstrumentBasename = basename(instrumentPath);
202+
const esmScenarioPathForRun = join(tmpDirPath, esmScenarioBasename);
203+
const esmInstrumentPathForRun = join(tmpDirPath, esmInstrumentBasename);
204+
writeFileSync(esmScenarioPathForRun, readFileSync(mjsScenarioPath, 'utf8'));
205+
writeFileSync(esmInstrumentPathForRun, readFileSync(mjsInstrumentPath, 'utf8'));
206+
207+
// Pre-create CJS converted files inside tmp dir
208+
const cjsScenarioPath = join(tmpDirPath, esmScenarioBasename.replace('.mjs', '.cjs'));
209+
const cjsInstrumentPath = join(tmpDirPath, esmInstrumentBasename.replace('.mjs', '.cjs'));
210+
convertEsmFileToCjs(esmScenarioPathForRun, cjsScenarioPath);
211+
convertEsmFileToCjs(esmInstrumentPathForRun, cjsInstrumentPath);
212+
213+
// Create a minimal package.json with requested dependencies (if any) and install them
214+
const additionalDependencies = options?.additionalDependencies ?? {};
215+
if (Object.keys(additionalDependencies).length > 0) {
216+
const packageJson = {
217+
name: 'tmp-integration-test',
218+
private: true,
219+
version: '0.0.0',
220+
dependencies: additionalDependencies,
221+
} as const;
222+
223+
writeFileSync(join(tmpDirPath, 'package.json'), JSON.stringify(packageJson, null, 2));
224+
225+
try {
226+
const deps = Object.entries(additionalDependencies).map(([name, range]) => {
227+
if (!range || typeof range !== 'string') {
228+
throw new Error(`Invalid version range for "${name}": ${String(range)}`);
229+
}
230+
return `${name}@${range}`;
231+
});
192232

193-
describe('esm', () => {
194-
const testFn = options?.failsOnEsm ? test.fails : test;
195-
callback(() => createRunner(mjsScenarioPath).withFlags('--import', mjsInstrumentPath), testFn, 'esm');
196-
});
233+
if (deps.length > 0) {
234+
// Prefer npm for temp installs to avoid Yarn engine strictness; see https://github.com/vercel/ai/issues/7777
235+
// We rely on the generated package.json dependencies and run a plain install.
236+
const result = spawnSync('npm', ['install', '--silent', '--no-audit', '--no-fund'], {
237+
cwd: tmpDirPath,
238+
encoding: 'utf8',
239+
});
240+
241+
if (process.env.DEBUG) {
242+
// eslint-disable-next-line no-console
243+
console.log('[additionalDependencies via npm]', deps.join(' '));
244+
// eslint-disable-next-line no-console
245+
console.log('[npm stdout]', result.stdout);
246+
// eslint-disable-next-line no-console
247+
console.log('[npm stderr]', result.stderr);
248+
}
197249

198-
describe('cjs', () => {
199-
beforeAll(() => {
200-
// For the CJS runner, we create some temporary files...
201-
convertEsmFileToCjs(mjsScenarioPath, cjsScenarioPath);
202-
convertEsmFileToCjs(mjsInstrumentPath, cjsInstrumentPath);
250+
if (result.error) {
251+
throw new Error(`Failed to install additionalDependencies in tmp dir ${tmpDirPath}: ${result.error.message}`);
252+
}
253+
if (typeof result.status === 'number' && result.status !== 0) {
254+
throw new Error(
255+
`Failed to install additionalDependencies in tmp dir ${tmpDirPath} (exit ${result.status}):\n${
256+
result.stderr || result.stdout || '(no output)'
257+
}`,
258+
);
259+
}
260+
}
261+
} catch (e) {
262+
// eslint-disable-next-line no-console
263+
console.error('Failed to install additionalDependencies:', e);
264+
throw e;
265+
}
266+
}
267+
268+
describe('esm/cjs', () => {
269+
const esmTestFn = options?.failsOnEsm ? test.fails : test;
270+
describe('esm', () => {
271+
callback(
272+
() => createRunner(esmScenarioPathForRun).withFlags('--import', esmInstrumentPathForRun),
273+
esmTestFn,
274+
'esm',
275+
);
276+
});
277+
278+
const cjsTestFn = options?.failsOnCjs ? test.fails : test;
279+
describe('cjs', () => {
280+
callback(() => createRunner(cjsScenarioPath).withFlags('--require', cjsInstrumentPath), cjsTestFn, 'cjs');
203281
});
204282

283+
// Clean up the tmp directory after both esm and cjs suites have run
205284
afterAll(() => {
206285
try {
207-
unlinkSync(cjsInstrumentPath);
208-
} catch {
209-
// Ignore errors here
210-
}
211-
try {
212-
unlinkSync(cjsScenarioPath);
286+
rmSync(tmpDirPath, { recursive: true, force: true });
213287
} catch {
214-
// Ignore errors here
288+
if (process.env.DEBUG) {
289+
// eslint-disable-next-line no-console
290+
console.error(`Failed to remove tmp dir: ${tmpDirPath}`);
291+
}
215292
}
216293
});
217-
218-
const testFn = options?.failsOnCjs ? test.fails : test;
219-
callback(() => createRunner(cjsScenarioPath).withFlags('--require', cjsInstrumentPath), testFn, 'cjs');
220294
});
221295
}
222296

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /