diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/agent-scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/agent-scenario.mjs new file mode 100644 index 000000000000..cef786b8988f --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/agent-scenario.mjs @@ -0,0 +1,70 @@ +import { ChatAnthropic } from '@langchain/anthropic'; +import { HumanMessage, SystemMessage } from '@langchain/core/messages'; +import { createReactAgent } from '@langchain/langgraph/prebuilt'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + const model = req.body.model; + + // Simulate basic response + res.json({ + id: 'msg_react_agent_123', + type: 'message', + role: 'assistant', + content: [ + { + type: 'text', + text: 'Mock response from Anthropic!', + }, + ], + model: model, + stop_reason: 'end_turn', + stop_sequence: null, + usage: { + input_tokens: 10, + output_tokens: 15, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + // Create mocked LLM instance + const llm = new ChatAnthropic({ + model: 'claude-3-5-sonnet-20241022', + apiKey: 'mock-api-key', + clientOptions: { + baseURL: baseUrl, + }, + }); + + // Create a simple react agent with no tools + const agent = createReactAgent({ llm, tools: [] }); + + // Test: basic invocation + await agent.invoke({ + messages: [new SystemMessage('You are a helpful assistant.'), new HumanMessage('What is the weather today?')], + }); + }); + + await Sentry.flush(2000); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts index 6a67b5cd1e86..bc1646db5468 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts @@ -205,4 +205,44 @@ describe('LangGraph integration', () => { await createRunner().ignore('event').expect({ transaction: EXPECTED_TRANSACTION_WITH_TOOLS }).start().completed(); }); }); + + const EXPECTED_TRANSACTION_REACT_AGENT = { + transaction: 'main', + spans: expect.arrayContaining([ + // create_agent span + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'create_agent', + 'sentry.op': 'gen_ai.create_agent', + 'sentry.origin': 'auto.ai.langgraph', + }), + description: expect.stringContaining('create_agent'), + op: 'gen_ai.create_agent', + origin: 'auto.ai.langgraph', + status: 'ok', + }), + // invoke_agent span + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'invoke_agent', + 'sentry.op': 'gen_ai.invoke_agent', + 'sentry.origin': 'auto.ai.langgraph', + }), + description: expect.stringContaining('invoke_agent'), + op: 'gen_ai.invoke_agent', + origin: 'auto.ai.langgraph', + status: 'ok', + }), + ]), + }; + + createEsmAndCjsTests(__dirname, 'agent-scenario.mjs', 'instrument.mjs', (createRunner, test) => { + test('should instrument LangGraph createReactAgent with default PII settings', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_REACT_AGENT }) + .start() + .completed(); + }); + }); }); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e18ea294f182..1a8ca0b05fbd 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -153,7 +153,7 @@ export type { GoogleGenAIResponse } from './tracing/google-genai/types'; export { createLangChainCallbackHandler } from './tracing/langchain'; export { LANGCHAIN_INTEGRATION_NAME } from './tracing/langchain/constants'; export type { LangChainOptions, LangChainIntegration } from './tracing/langchain/types'; -export { instrumentStateGraphCompile, instrumentLangGraph } from './tracing/langgraph'; +export { instrumentStateGraphCompile, instrumentCreateReactAgent, instrumentLangGraph } from './tracing/langgraph'; export { LANGGRAPH_INTEGRATION_NAME } from './tracing/langgraph/constants'; export type { LangGraphOptions, LangGraphIntegration, CompiledGraph } from './tracing/langgraph/types'; export type { OpenAiClient, OpenAiOptions, InstrumentedMethod } from './tracing/openai/types'; diff --git a/packages/core/src/tracing/langchain/types.ts b/packages/core/src/tracing/langchain/types.ts index 7379de764817..e2aa08f4631c 100644 --- a/packages/core/src/tracing/langchain/types.ts +++ b/packages/core/src/tracing/langchain/types.ts @@ -30,6 +30,14 @@ export interface LangChainSerialized { kwargs?: Record; } +/** + * Subset of the 'llm' param passed to createReactAgent + */ +export interface BaseChatModel { + lc_namespace: string[]; + modelName: string; +} + /** * LangChain message structure * Supports both regular messages and LangChain serialized format diff --git a/packages/core/src/tracing/langgraph/index.ts b/packages/core/src/tracing/langgraph/index.ts index 5601cddf458b..67f90357d713 100644 --- a/packages/core/src/tracing/langgraph/index.ts +++ b/packages/core/src/tracing/langgraph/index.ts @@ -8,14 +8,15 @@ import { GEN_AI_PIPELINE_NAME_ATTRIBUTE, GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, GEN_AI_REQUEST_MESSAGES_ATTRIBUTE, + GEN_AI_REQUEST_MODEL_ATTRIBUTE, } from '../ai/gen-ai-attributes'; import { truncateGenAiMessages } from '../ai/messageTruncation'; -import type { LangChainMessage } from '../langchain/types'; +import type { BaseChatModel, LangChainMessage } from '../langchain/types'; import { normalizeLangChainMessages } from '../langchain/utils'; import { startSpan } from '../trace'; import { LANGGRAPH_ORIGIN } from './constants'; import type { CompiledGraph, LangGraphOptions } from './types'; -import { extractToolsFromCompiledGraph, setResponseAttributes } from './utils'; +import { extractLLMFromParams, extractToolsFromCompiledGraph, setResponseAttributes } from './utils'; /** * Instruments StateGraph's compile method to create spans for agent creation and invocation @@ -90,9 +91,11 @@ function instrumentCompiledGraphInvoke( graphInstance: CompiledGraph, compileOptions: Record, options: LangGraphOptions, + llm?: BaseChatModel | null, ): (...args: unknown[]) => Promise { return new Proxy(originalInvoke, { apply(target, thisArg, args: unknown[]): Promise { + const modelName = llm?.modelName; return startSpan( { op: 'gen_ai.invoke_agent', @@ -112,6 +115,9 @@ function instrumentCompiledGraphInvoke( span.setAttribute(GEN_AI_AGENT_NAME_ATTRIBUTE, graphName); span.updateName(`invoke_agent ${graphName}`); } + if (modelName) { + span.setAttribute(GEN_AI_REQUEST_MODEL_ATTRIBUTE, modelName); + } // Extract available tools from the graph instance const tools = extractToolsFromCompiledGraph(graphInstance); @@ -156,6 +162,60 @@ function instrumentCompiledGraphInvoke( }) as (...args: unknown[]) => Promise; } +/** + * Instruments createReactAgent to create spans for agent creation and invocation + * + * Creates a `gen_ai.create_agent` span when createReactAgent() is called + */ +export function instrumentCreateReactAgent( + originalCreateReactAgent: (...args: unknown[]) => CompiledGraph, + options: LangGraphOptions, +): (...args: unknown[]) => CompiledGraph { + return new Proxy(originalCreateReactAgent, { + apply(target, thisArg, args: unknown[]): CompiledGraph { + const llm = extractLLMFromParams(args); + return startSpan( + { + op: 'gen_ai.create_agent', + name: 'create_agent', + attributes: { + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: LANGGRAPH_ORIGIN, + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.create_agent', + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'create_agent', + }, + }, + span => { + try { + const compiledGraph = Reflect.apply(target, thisArg, args); + const compiledOptions = args.length > 0 ? (args[0] as Record) : {}; + const originalInvoke = compiledGraph.invoke; + if (originalInvoke && typeof originalInvoke === 'function') { + compiledGraph.invoke = instrumentCompiledGraphInvoke( + originalInvoke.bind(compiledGraph) as (...args: unknown[]) => Promise, + compiledGraph, + compiledOptions, + options, + llm, + ) as typeof originalInvoke; + } + + return compiledGraph; + } catch (error) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + captureException(error, { + mechanism: { + handled: false, + type: 'auto.ai.langgraph.error', + }, + }); + throw error; + } + }, + ); + }, + }) as (...args: unknown[]) => CompiledGraph; +} + /** * Directly instruments a StateGraph instance to add tracing spans * diff --git a/packages/core/src/tracing/langgraph/utils.ts b/packages/core/src/tracing/langgraph/utils.ts index 4b1990058924..f2c64052c071 100644 --- a/packages/core/src/tracing/langgraph/utils.ts +++ b/packages/core/src/tracing/langgraph/utils.ts @@ -8,10 +8,25 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../ai/gen-ai-attributes'; -import type { LangChainMessage } from '../langchain/types'; +import type { BaseChatModel, LangChainMessage } from '../langchain/types'; import { normalizeLangChainMessages } from '../langchain/utils'; import type { CompiledGraph, LangGraphTool } from './types'; +/** + * Extract LLM model object from createReactAgent params + */ +export function extractLLMFromParams(args: unknown[]): null | BaseChatModel { + const arg = args[0]; + return typeof arg === 'object' && + !!arg && + 'llm' in arg && + !!arg.llm && + typeof arg.llm === 'object' && + typeof (arg.llm as BaseChatModel).modelName === 'string' + ? (arg.llm as BaseChatModel) + : null; +} + /** * Extract tool calls from messages */ @@ -139,7 +154,9 @@ export function setResponseAttributes(span: Span, inputMessages: LangChainMessag } // Get new messages (delta between input and output) + /* v8 ignore start - coverage gets confused by this somehow */ const inputCount = inputMessages?.length ?? 0; + /* v8 ignore stop */ const newMessages = outputMessages.length > inputCount ? outputMessages.slice(inputCount) : []; if (newMessages.length === 0) { diff --git a/packages/core/test/lib/utils/langraph-utils.test.ts b/packages/core/test/lib/utils/langraph-utils.test.ts new file mode 100644 index 000000000000..bd8d5c7193ec --- /dev/null +++ b/packages/core/test/lib/utils/langraph-utils.test.ts @@ -0,0 +1,255 @@ +import { beforeEach, describe, expect, it } from 'vitest'; +import type { Span } from '../../../src'; +import { + extractLLMFromParams, + extractModelMetadata, + extractTokenUsageFromMessage, + extractToolCalls, + extractToolsFromCompiledGraph, + setResponseAttributes, +} from '../../../src/tracing/langgraph/utils'; + +describe('extractLLMFromParams', () => { + it('handles invalid args or missing llm object', () => { + // @ts-expect-error should be arguments array, at least. + expect(extractLLMFromParams({})).toBe(null); + expect(extractLLMFromParams([])).toBe(null); + expect(extractLLMFromParams([null])).toBe(null); + expect(extractLLMFromParams([{}])).toBe(null); + expect(extractLLMFromParams([{ llm: false }])).toBe(null); + expect(extractLLMFromParams([{ llm: 123 }])).toBe(null); + expect(extractLLMFromParams([{ llm: {} }])).toBe(null); + }); + it('extracts llm object if found', () => { + expect(extractLLMFromParams([{ llm: { modelName: 'model-name-1' } }])).toStrictEqual({ modelName: 'model-name-1' }); + }); +}); + +describe('extractToolCalls', () => { + it('returns null for missing/empty messages', () => { + expect(extractToolCalls(null)).toBe(null); + expect(extractToolCalls([])).toBe(null); + expect(extractToolCalls([{}])).toBe(null); + expect(extractToolCalls([{ tool_calls: null }])).toBe(null); + expect(extractToolCalls([{ tool_calls: [] }])).toBe(null); + }); + it('extracts tool call from messages array', () => { + expect( + extractToolCalls([ + { tool_calls: [{ name: 'tool a' }] }, + { tool_calls: [{ name: 'tool b' }, { name: 'tool c' }] }, + ]), + ).toStrictEqual([{ name: 'tool a' }, { name: 'tool b' }, { name: 'tool c' }]); + }); +}); + +describe('extractTokenUsageFromMessage', () => { + it('extracts from usage_metadata', () => { + const inputs = [{}, { input_tokens: 10 }]; + const outputs = [{}, { output_tokens: 20 }]; + const totals = [{}, { total_tokens: 30 }]; + for (const i of inputs) { + for (const o of outputs) { + for (const t of totals) { + expect( + extractTokenUsageFromMessage({ + usage_metadata: { + ...i, + ...o, + ...t, + }, + }), + ).toStrictEqual({ + inputTokens: i?.input_tokens ?? 0, + outputTokens: o?.output_tokens ?? 0, + totalTokens: t?.total_tokens ?? 0, + }); + } + } + } + }); + it('falls back to response_metadata', () => { + const inputs = [{}, { promptTokens: 10 }]; + const outputs = [{}, { completionTokens: 20 }]; + const totals = [{}, { totalTokens: 30 }]; + for (const i of inputs) { + for (const o of outputs) { + for (const t of totals) { + expect( + extractTokenUsageFromMessage({ + response_metadata: { + // @ts-expect-error using old tokenUsage field + tokenUsage: { + ...i, + ...o, + ...t, + }, + }, + }), + ).toStrictEqual({ + inputTokens: i?.promptTokens ?? 0, + outputTokens: o?.completionTokens ?? 0, + totalTokens: t?.totalTokens ?? 0, + }); + } + } + } + }); +}); + +describe('extractModelMetadata', () => { + let attributes: Record = {}; + const span = { + setAttribute(key: string, value: unknown) { + attributes[key] = value; + }, + } as unknown as Span; + beforeEach(() => (attributes = {})); + + it('handles lacking metadata ok', () => { + extractModelMetadata(span, {}); + expect(attributes).toStrictEqual({}); + }); + + it('extracts response model name from metadata', () => { + extractModelMetadata(span, { + response_metadata: { + model_name: 'model-name', + finish_reason: 'stop', + }, + }); + expect(attributes).toStrictEqual({ + 'gen_ai.response.model': 'model-name', + 'gen_ai.response.finish_reasons': ['stop'], + }); + }); +}); + +describe('extractToolsFromCompiledGraph', () => { + it('returns null if no tools found', () => { + expect(extractToolsFromCompiledGraph({})).toBe(null); + expect( + extractToolsFromCompiledGraph({ + builder: { nodes: { tools: { runnable: {} } } }, + }), + ).toBe(null); + expect( + extractToolsFromCompiledGraph({ + // @ts-expect-error Wants LangGraphTool[] + builder: { nodes: { tools: { runnable: { tools: 'not an array' } } } }, + }), + ).toBe(null); + expect( + extractToolsFromCompiledGraph({ + builder: { nodes: { tools: { runnable: { tools: [] } } } }, + }), + ).toBe(null); + }); + it('returns the tools found', () => { + expect( + extractToolsFromCompiledGraph({ + builder: { + nodes: { + tools: { + runnable: { + tools: [ + {}, + { lc_kwargs: { name: 'name' } }, + { lc_kwargs: { name: 'name', description: 'desc' } }, + { lc_kwargs: { name: 'name', description: 'desc', schema: 'schema' } }, + ], + }, + }, + }, + }, + }), + ).toStrictEqual([ + { name: undefined, description: undefined, schema: undefined }, + { name: 'name', description: undefined, schema: undefined }, + { name: 'name', description: 'desc', schema: undefined }, + { name: 'name', description: 'desc', schema: 'schema' }, + ]); + }); +}); + +describe('setResponseAttribute', () => { + let attributes: Record = {}; + const span = { + setAttribute(key: string, value: unknown) { + attributes[key] = value; + }, + } as unknown as Span; + beforeEach(() => (attributes = {})); + + it('handles lack of messages', () => { + setResponseAttributes(span, [], undefined); + expect(attributes).toStrictEqual({}); + + setResponseAttributes(span, null, undefined); + expect(attributes).toStrictEqual({}); + + setResponseAttributes(span, null, {}); + expect(attributes).toStrictEqual({}); + + setResponseAttributes(span, null, { messages: null }); + expect(attributes).toStrictEqual({}); + + // no new messages + setResponseAttributes(span, [{}], { messages: [{}] }); + expect(attributes).toStrictEqual({}); + setResponseAttributes(span, [], { messages: [] }); + expect(attributes).toStrictEqual({}); + + // @ts-expect-error cover excessive type safety case + setResponseAttributes(span, { length: undefined }, []); + expect(attributes).toStrictEqual({}); + }); + + it('extracts tool calls', () => { + setResponseAttributes(span, [], { + messages: [{ tool_calls: [{ name: 'tool a' }] }], + }); + expect(attributes).toStrictEqual({ + 'gen_ai.response.text': '[{"role":"user"}]', + 'gen_ai.response.tool_calls': JSON.stringify([{ name: 'tool a' }]), + }); + }); + + it('extracts token usage', () => { + setResponseAttributes(span, [], { + messages: [ + { + usage_metadata: { + input_tokens: 1, + output_tokens: 2, + total_tokens: 3, + }, + }, + ], + }); + expect(attributes).toStrictEqual({ + 'gen_ai.response.text': '[{"role":"user"}]', + 'gen_ai.usage.input_tokens': 1, + 'gen_ai.usage.output_tokens': 2, + 'gen_ai.usage.total_tokens': 3, + }); + }); + + it('extracts model metadata', () => { + setResponseAttributes(span, [], { + messages: [ + { + response_metadata: { + model_name: 'model-name-1', + finish_reason: 'stop', + }, + }, + ], + }); + expect(attributes).toStrictEqual({ + 'gen_ai.response.text': '[{"role":"user"}]', + 'gen_ai.response.model': 'model-name-1', + 'gen_ai.response.finish_reasons': ['stop'], + }); + }); +}); diff --git a/packages/node/src/integrations/tracing/langgraph/instrumentation.ts b/packages/node/src/integrations/tracing/langgraph/instrumentation.ts index d275e1b9d39b..b3f897e70363 100644 --- a/packages/node/src/integrations/tracing/langgraph/instrumentation.ts +++ b/packages/node/src/integrations/tracing/langgraph/instrumentation.ts @@ -6,7 +6,7 @@ import { InstrumentationNodeModuleFile, } from '@opentelemetry/instrumentation'; import type { CompiledGraph, LangGraphOptions } from '@sentry/core'; -import { getClient, instrumentStateGraphCompile, SDK_VERSION } from '@sentry/core'; +import { getClient, instrumentCreateReactAgent, instrumentStateGraphCompile, SDK_VERSION } from '@sentry/core'; const supportedVersions = ['>=0.0.0 <2.0.0']; @@ -18,6 +18,7 @@ type LangGraphInstrumentationOptions = InstrumentationConfig & LangGraphOptions; interface PatchedModuleExports { [key: string]: unknown; StateGraph?: abstract new (...args: unknown[]) => unknown; + createReactAgent?: (...args: unknown[]) => CompiledGraph; } /** @@ -31,28 +32,72 @@ export class SentryLangGraphInstrumentation extends InstrumentationBase exports, - [ - new InstrumentationNodeModuleFile( - /** - * In CJS, LangGraph packages re-export from dist/index.cjs files. - * Patching only the root module sometimes misses the real implementation or - * gets overwritten when that file is loaded. We add a file-level patch so that - * _patch runs again on the concrete implementation - */ - '@langchain/langgraph/dist/index.cjs', - supportedVersions, - this._patch.bind(this), - exports => exports, - ), - ], - ); - return module; + public init(): InstrumentationModuleDefinition[] { + return [ + new InstrumentationNodeModuleDefinition( + '@langchain/langgraph', + supportedVersions, + this._patch.bind(this), + exports => exports, + [ + new InstrumentationNodeModuleFile( + /** + * In CJS, LangGraph packages re-export from dist/index.cjs files. + * Patching only the root module sometimes misses the real implementation or + * gets overwritten when that file is loaded. We add a file-level patch so that + * _patch runs again on the concrete implementation + */ + '@langchain/langgraph/dist/index.cjs', + supportedVersions, + this._patch.bind(this), + exports => exports, + ), + new InstrumentationNodeModuleFile( + /** + * In CJS, LangGraph packages re-export from dist/prebuilt/index.cjs files. + * Patching only the root module sometimes misses the real implementation or + * gets overwritten when that file is loaded. We add a file-level patch so that + * _patch runs again on the concrete implementation + */ + '@langchain/langgraph/dist/prebuilt/index.cjs', + supportedVersions, + this._patch.bind(this), + exports => exports, + ), + ], + ), + new InstrumentationNodeModuleDefinition( + '@langchain/langgraph/prebuilt', + supportedVersions, + this._patch.bind(this), + exports => exports, + [ + new InstrumentationNodeModuleFile( + /** + * ESM builds use dist/prebuilt/index.js (without .cjs extension) + * This catches ESM imports that resolve through the main package, + * using the package.json submodule export + */ + '@langchain/langgraph/prebuilt', + supportedVersions, + this._patch.bind(this), + exports => exports, + ), + new InstrumentationNodeModuleFile( + /** + * In CJS, LangGraph packages re-export from dist/prebuilt/index.cjs files. + * Patching only the root module sometimes misses the real implementation or + * gets overwritten when that file is loaded. We add a file-level patch so that + * _patch runs again on the concrete implementation + */ + '@langchain/langgraph/dist/prebuilt/index.cjs', + supportedVersions, + this._patch.bind(this), + exports => exports, + ), + ], + ), + ]; } /** @@ -83,6 +128,17 @@ export class SentryLangGraphInstrumentation extends InstrumentationBase CompiledGraph, options), + writable: true, + enumerable: true, + configurable: true, + }); + } + return exports; } }