Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 9e70a5a

Browse files
RulaKhaledandreiborza
andauthored
feat(core): Add tool calls attributes for Anthropic AI (#17478)
This PR adds missing tool call attributes, we addgen_ai.response.tool_calls attribute for Anthropic AI, supporting both streaming and non-streaming requests. Core changes: Request Side - Capture available tools: - Extract tools extract from request params - Set gen_ai.request.available_tools attribute Response Side - Capture actual tool calls: - Extract from response.tool_calls - Set gen_ai.response.tool_calls attribute for both Streaming Support (in streaming.ts): - Accumulation of tool calls during streaming - Respects recordOutputs privacy setting --------- Co-authored-by: Andrei <168741329+andreiborza@users.noreply.github.com>
1 parent 9a37660 commit 9e70a5a

File tree

7 files changed

+380
-16
lines changed

7 files changed

+380
-16
lines changed
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import { instrumentAnthropicAiClient } from '@sentry/core';
2+
import * as Sentry from '@sentry/node';
3+
4+
function createMockStreamEvents(model = 'claude-3-haiku-20240307') {
5+
async function* generator() {
6+
// initial message metadata with id/model and input tokens
7+
yield {
8+
type: 'content_block_start',
9+
message: {
10+
id: 'msg_stream_tool_1',
11+
type: 'message',
12+
role: 'assistant',
13+
model,
14+
content: [],
15+
stop_reason: 'end_turn',
16+
usage: { input_tokens: 11 },
17+
},
18+
};
19+
20+
// streamed text
21+
yield { type: 'content_block_delta', delta: { text: 'Starting tool...' } };
22+
23+
// tool_use streamed via partial json
24+
yield {
25+
type: 'content_block_start',
26+
index: 0,
27+
content_block: { type: 'tool_use', id: 'tool_weather_2', name: 'weather' },
28+
};
29+
yield { type: 'content_block_delta', index: 0, delta: { partial_json: '{"city":' } };
30+
yield { type: 'content_block_delta', index: 0, delta: { partial_json: '"Paris"}' } };
31+
yield { type: 'content_block_stop', index: 0 };
32+
33+
// more text
34+
yield { type: 'content_block_delta', delta: { text: 'Done.' } };
35+
36+
// final usage
37+
yield { type: 'message_delta', usage: { output_tokens: 9 } };
38+
}
39+
return generator();
40+
}
41+
42+
class MockAnthropic {
43+
constructor(config) {
44+
this.apiKey = config.apiKey;
45+
this.messages = {
46+
create: this._messagesCreate.bind(this),
47+
stream: this._messagesStream.bind(this),
48+
};
49+
}
50+
51+
async _messagesCreate(params) {
52+
await new Promise(resolve => setTimeout(resolve, 5));
53+
if (params?.stream) {
54+
return createMockStreamEvents(params.model);
55+
}
56+
return {
57+
id: 'msg_mock_no_stream',
58+
type: 'message',
59+
model: params.model,
60+
role: 'assistant',
61+
content: [{ type: 'text', text: 'No stream' }],
62+
usage: { input_tokens: 2, output_tokens: 3 },
63+
};
64+
}
65+
66+
async _messagesStream(params) {
67+
await new Promise(resolve => setTimeout(resolve, 5));
68+
return createMockStreamEvents(params?.model);
69+
}
70+
}
71+
72+
async function run() {
73+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
74+
const mockClient = new MockAnthropic({ apiKey: 'mock-api-key' });
75+
const client = instrumentAnthropicAiClient(mockClient);
76+
77+
// stream via create(stream:true)
78+
const stream1 = await client.messages.create({
79+
model: 'claude-3-haiku-20240307',
80+
messages: [{ role: 'user', content: 'Need the weather' }],
81+
tools: [
82+
{
83+
name: 'weather',
84+
description: 'Get weather',
85+
input_schema: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] },
86+
},
87+
],
88+
stream: true,
89+
});
90+
for await (const _ of stream1) {
91+
void _;
92+
}
93+
94+
// stream via messages.stream
95+
const stream2 = await client.messages.stream({
96+
model: 'claude-3-haiku-20240307',
97+
messages: [{ role: 'user', content: 'Need the weather' }],
98+
tools: [
99+
{
100+
name: 'weather',
101+
description: 'Get weather',
102+
input_schema: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] },
103+
},
104+
],
105+
});
106+
for await (const _ of stream2) {
107+
void _;
108+
}
109+
});
110+
}
111+
112+
run();
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import { instrumentAnthropicAiClient } from '@sentry/core';
2+
import * as Sentry from '@sentry/node';
3+
4+
class MockAnthropic {
5+
constructor(config) {
6+
this.apiKey = config.apiKey;
7+
8+
this.messages = {
9+
create: this._messagesCreate.bind(this),
10+
};
11+
}
12+
13+
async _messagesCreate(params) {
14+
await new Promise(resolve => setTimeout(resolve, 5));
15+
16+
return {
17+
id: 'msg_mock_tool_1',
18+
type: 'message',
19+
model: params.model,
20+
role: 'assistant',
21+
content: [
22+
{ type: 'text', text: 'Let me check the weather.' },
23+
{
24+
type: 'tool_use',
25+
id: 'tool_weather_1',
26+
name: 'weather',
27+
input: { city: 'Paris' },
28+
},
29+
{ type: 'text', text: 'It is sunny.' },
30+
],
31+
stop_reason: 'end_turn',
32+
stop_sequence: null,
33+
usage: {
34+
input_tokens: 5,
35+
output_tokens: 7,
36+
},
37+
};
38+
}
39+
}
40+
41+
async function run() {
42+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
43+
const mockClient = new MockAnthropic({ apiKey: 'mock-api-key' });
44+
const client = instrumentAnthropicAiClient(mockClient);
45+
46+
await client.messages.create({
47+
model: 'claude-3-haiku-20240307',
48+
messages: [{ role: 'user', content: 'What is the weather?' }],
49+
tools: [
50+
{
51+
name: 'weather',
52+
description: 'Get the weather by city',
53+
input_schema: {
54+
type: 'object',
55+
properties: { city: { type: 'string' } },
56+
required: ['city'],
57+
},
58+
},
59+
],
60+
});
61+
});
62+
}
63+
64+
run();

‎dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,4 +293,59 @@ describe('Anthropic integration', () => {
293293
await createRunner().ignore('event').expect({ transaction: EXPECTED_STREAM_SPANS_PII_TRUE }).start().completed();
294294
});
295295
});
296+
297+
// Non-streaming tool calls + available tools (PII true)
298+
createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
299+
test('non-streaming sets available tools and tool calls with PII', async () => {
300+
const EXPECTED_TOOLS_JSON =
301+
'[{"name":"weather","description":"Get the weather by city","input_schema":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}]';
302+
const EXPECTED_TOOL_CALLS_JSON =
303+
'[{"type":"tool_use","id":"tool_weather_1","name":"weather","input":{"city":"Paris"}}]';
304+
await createRunner()
305+
.ignore('event')
306+
.expect({
307+
transaction: {
308+
spans: expect.arrayContaining([
309+
expect.objectContaining({
310+
op: 'gen_ai.messages',
311+
data: expect.objectContaining({
312+
'gen_ai.request.available_tools': EXPECTED_TOOLS_JSON,
313+
'gen_ai.response.tool_calls': EXPECTED_TOOL_CALLS_JSON,
314+
}),
315+
}),
316+
]),
317+
},
318+
})
319+
.start()
320+
.completed();
321+
});
322+
});
323+
324+
// Streaming tool calls + available tools (PII true)
325+
createEsmAndCjsTests(__dirname, 'scenario-stream-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
326+
test('streaming sets available tools and tool calls with PII', async () => {
327+
const EXPECTED_TOOLS_JSON =
328+
'[{"name":"weather","description":"Get weather","input_schema":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}]';
329+
const EXPECTED_TOOL_CALLS_JSON =
330+
'[{"type":"tool_use","id":"tool_weather_2","name":"weather","input":{"city":"Paris"}}]';
331+
await createRunner()
332+
.ignore('event')
333+
.expect({
334+
transaction: {
335+
spans: expect.arrayContaining([
336+
expect.objectContaining({
337+
description: expect.stringContaining('stream-response'),
338+
op: 'gen_ai.messages',
339+
data: expect.objectContaining({
340+
'gen_ai.request.available_tools': EXPECTED_TOOLS_JSON,
341+
'gen_ai.response.tool_calls': EXPECTED_TOOL_CALLS_JSON,
342+
}),
343+
}),
344+
]),
345+
},
346+
})
347+
.start()
348+
.completed();
349+
});
350+
});
296351
});

‎packages/core/src/utils/anthropic-ai/constants.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@ export const ANTHROPIC_AI_INSTRUMENTED_METHODS = [
99
'models.get',
1010
'completions.create',
1111
'models.retrieve',
12+
'beta.messages.create',
1213
] as const;

‎packages/core/src/utils/anthropic-ai/index.ts

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
ANTHROPIC_AI_RESPONSE_TIMESTAMP_ATTRIBUTE,
99
GEN_AI_OPERATION_NAME_ATTRIBUTE,
1010
GEN_AI_PROMPT_ATTRIBUTE,
11+
GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE,
1112
GEN_AI_REQUEST_FREQUENCY_PENALTY_ATTRIBUTE,
1213
GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE,
1314
GEN_AI_REQUEST_MESSAGES_ATTRIBUTE,
@@ -19,6 +20,7 @@ import {
1920
GEN_AI_RESPONSE_ID_ATTRIBUTE,
2021
GEN_AI_RESPONSE_MODEL_ATTRIBUTE,
2122
GEN_AI_RESPONSE_TEXT_ATTRIBUTE,
23+
GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE,
2224
GEN_AI_SYSTEM_ATTRIBUTE,
2325
} from '../ai/gen-ai-attributes';
2426
import { buildMethodPath, getFinalOperationName, getSpanOperation, setTokenUsageAttributes } from '../ai/utils';
@@ -31,6 +33,7 @@ import type {
3133
AnthropicAiOptions,
3234
AnthropicAiResponse,
3335
AnthropicAiStreamingEvent,
36+
ContentBlock,
3437
} from './types';
3538
import { shouldInstrument } from './utils';
3639

@@ -46,6 +49,9 @@ function extractRequestAttributes(args: unknown[], methodPath: string): Record<s
4649

4750
if (args.length > 0 && typeof args[0] === 'object' && args[0] !== null) {
4851
const params = args[0] as Record<string, unknown>;
52+
if (params.tools && Array.isArray(params.tools)) {
53+
attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE] = JSON.stringify(params.tools);
54+
}
4955

5056
attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE] = params.model ?? 'unknown';
5157
if ('temperature' in params) attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE] = params.temperature;
@@ -96,10 +102,21 @@ function addResponseAttributes(span: Span, response: AnthropicAiResponse, record
96102
if (Array.isArray(response.content)) {
97103
span.setAttributes({
98104
[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: response.content
99-
.map((item: {text: string|undefined}) => item.text)
100-
.filter((text): text is string=> text!==undefined)
105+
.map((item: ContentBlock) => item.text)
106+
.filter(text=> !!text)
101107
.join(''),
102108
});
109+
110+
const toolCalls: Array<ContentBlock> = [];
111+
112+
for (const item of response.content) {
113+
if (item.type === 'tool_use' || item.type === 'server_tool_use') {
114+
toolCalls.push(item);
115+
}
116+
}
117+
if (toolCalls.length > 0) {
118+
span.setAttributes({ [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: JSON.stringify(toolCalls) });
119+
}
103120
}
104121
}
105122
// Completions.create

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /