LeadMagnet/leadchat
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat: Instrument captain (#12949)

Browse Source 
Co-authored-by: aakashb95 <aakash@chatwoot.com>
This commit is contained in:
Aakash Bakhle
2025-11-28 15:12:55 +05:30
committed by GitHub
parent 92fc286ecb
commit 1ef945de7b
12 changed files with 346 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
lib/integrations/llm_instrumentation.rb
View File

@@ -1,28 +1,10 @@
# frozen_string_literal: true
require 'opentelemetry_config'
require_relative 'llm_instrumentation_constants'
module Integrations::LlmInstrumentation
# OpenTelemetry attribute names following GenAI semantic conventions
# https://opentelemetry.io/docs/specs/semconv/gen-ai/
ATTR_GEN_AI_PROVIDER = 'gen_ai.provider.name'
ATTR_GEN_AI_REQUEST_MODEL = 'gen_ai.request.model'
ATTR_GEN_AI_REQUEST_TEMPERATURE = 'gen_ai.request.temperature'
ATTR_GEN_AI_PROMPT_ROLE = 'gen_ai.prompt.%d.role'
ATTR_GEN_AI_PROMPT_CONTENT = 'gen_ai.prompt.%d.content'
ATTR_GEN_AI_COMPLETION_ROLE = 'gen_ai.completion.0.role'
ATTR_GEN_AI_COMPLETION_CONTENT = 'gen_ai.completion.0.content'
ATTR_GEN_AI_USAGE_INPUT_TOKENS = 'gen_ai.usage.input_tokens'
ATTR_GEN_AI_USAGE_OUTPUT_TOKENS = 'gen_ai.usage.output_tokens'
ATTR_GEN_AI_USAGE_TOTAL_TOKENS = 'gen_ai.usage.total_tokens'
ATTR_GEN_AI_RESPONSE_ERROR = 'gen_ai.response.error'
ATTR_GEN_AI_RESPONSE_ERROR_CODE = 'gen_ai.response.error_code'
# Langfuse-specific attributes
# https://langfuse.com/integrations/native/opentelemetry#property-mapping
ATTR_LANGFUSE_USER_ID = 'langfuse.user.id'
ATTR_LANGFUSE_SESSION_ID = 'langfuse.session.id'
ATTR_LANGFUSE_TAGS = 'langfuse.trace.tags'
include Integrations::LlmInstrumentationConstants
def tracer
@tracer ||= OpentelemetryConfig.tracer
@@ -42,6 +24,37 @@ module Integrations::LlmInstrumentation
yield
end
def instrument_agent_session(params)
return yield unless ChatwootApp.otel_enabled?
tracer.in_span(params[:span_name]) do |span|
set_metadata_attributes(span, params)
# By default, the input and output of a trace are set from the root observation
span.set_attribute(ATTR_LANGFUSE_OBSERVATION_INPUT, params[:messages].to_json)
result = yield
span.set_attribute(ATTR_LANGFUSE_OBSERVATION_OUTPUT, result.to_json)
result
end
rescue StandardError => e
ChatwootExceptionTracker.new(e, account: params[:account]).capture_exception
yield
end
def instrument_tool_call(tool_name, arguments)
# There is no error handling because tools can fail and LLMs should be
# aware of those failures and factor them into their response.
return yield unless ChatwootApp.otel_enabled?
tracer.in_span(format(TOOL_SPAN_NAME, tool_name)) do |span|
span.set_attribute(ATTR_LANGFUSE_OBSERVATION_INPUT, arguments.to_json)
result = yield
span.set_attribute(ATTR_LANGFUSE_OBSERVATION_OUTPUT, result.to_json)
result
end
end
private
def setup_span_attributes(span, params)
@@ -62,8 +75,11 @@ module Integrations::LlmInstrumentation
def set_prompt_messages(span, messages)
messages.each_with_index do |msg, idx|
span.set_attribute(format(ATTR_GEN_AI_PROMPT_ROLE, idx), msg['role'])
span.set_attribute(format(ATTR_GEN_AI_PROMPT_CONTENT, idx), msg['content'])
role = msg[:role] || msg['role']
content = msg[:content] || msg['content']
span.set_attribute(format(ATTR_GEN_AI_PROMPT_ROLE, idx), role)
span.set_attribute(format(ATTR_GEN_AI_PROMPT_CONTENT, idx), content.to_s)
end
end
@@ -72,6 +88,12 @@ module Integrations::LlmInstrumentation
span.set_attribute(ATTR_LANGFUSE_USER_ID, params[:account_id].to_s) if params[:account_id]
span.set_attribute(ATTR_LANGFUSE_SESSION_ID, session_id) if session_id.present?
span.set_attribute(ATTR_LANGFUSE_TAGS, [params[:feature_name]].to_json)
return unless params[:metadata].is_a?(Hash)
params[:metadata].each do |key, value|
span.set_attribute(format(ATTR_LANGFUSE_METADATA, key), value.to_s)
end
end
def set_completion_attributes(span, result)
@@ -81,26 +103,29 @@ module Integrations::LlmInstrumentation
end
def set_completion_message(span, result)
return if result[:message].blank?
message = result[:message] || result.dig('choices', 0, 'message', 'content')
return if message.blank?
span.set_attribute(ATTR_GEN_AI_COMPLETION_ROLE, 'assistant')
span.set_attribute(ATTR_GEN_AI_COMPLETION_CONTENT, result[:message])
span.set_attribute(ATTR_GEN_AI_COMPLETION_CONTENT, message)
end
def set_usage_metrics(span, result)
return if result[:usage].blank?
usage = result[:usage] || result['usage']
return if usage.blank?
usage = result[:usage]
span.set_attribute(ATTR_GEN_AI_USAGE_INPUT_TOKENS, usage['prompt_tokens']) if usage['prompt_tokens']
span.set_attribute(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, usage['completion_tokens']) if usage['completion_tokens']
span.set_attribute(ATTR_GEN_AI_USAGE_TOTAL_TOKENS, usage['total_tokens']) if usage['total_tokens']
end
def set_error_attributes(span, result)
return if result[:error].blank?
error = result[:error] || result['error']
return if error.blank?
span.set_attribute(ATTR_GEN_AI_RESPONSE_ERROR, result[:error].to_json)
span.set_attribute(ATTR_GEN_AI_RESPONSE_ERROR_CODE, result[:error_code]) if result[:error_code]
span.status = OpenTelemetry::Trace::Status.error("API Error: #{result[:error_code]}")
error_code = result[:error_code] || result['error_code']
span.set_attribute(ATTR_GEN_AI_RESPONSE_ERROR, error.to_json)
span.set_attribute(ATTR_GEN_AI_RESPONSE_ERROR_CODE, error_code) if error_code
span.status = OpenTelemetry::Trace::Status.error("API Error: #{error_code}")
end
end

31
lib/integrations/llm_instrumentation_constants.rb Normal file
View File

@@ -0,0 +1,31 @@
# frozen_string_literal: true
module Integrations::LlmInstrumentationConstants
# OpenTelemetry attribute names following GenAI semantic conventions
# https://opentelemetry.io/docs/specs/semconv/gen-ai/
ATTR_GEN_AI_PROVIDER = 'gen_ai.provider.name'
ATTR_GEN_AI_REQUEST_MODEL = 'gen_ai.request.model'
ATTR_GEN_AI_REQUEST_TEMPERATURE = 'gen_ai.request.temperature'
ATTR_GEN_AI_PROMPT_ROLE = 'gen_ai.prompt.%d.role'
ATTR_GEN_AI_PROMPT_CONTENT = 'gen_ai.prompt.%d.content'
ATTR_GEN_AI_COMPLETION_ROLE = 'gen_ai.completion.0.role'
ATTR_GEN_AI_COMPLETION_CONTENT = 'gen_ai.completion.0.content'
ATTR_GEN_AI_USAGE_INPUT_TOKENS = 'gen_ai.usage.input_tokens'
ATTR_GEN_AI_USAGE_OUTPUT_TOKENS = 'gen_ai.usage.output_tokens'
ATTR_GEN_AI_USAGE_TOTAL_TOKENS = 'gen_ai.usage.total_tokens'
ATTR_GEN_AI_RESPONSE_ERROR = 'gen_ai.response.error'
ATTR_GEN_AI_RESPONSE_ERROR_CODE = 'gen_ai.response.error_code'
TOOL_SPAN_NAME = 'tool.%s'
# Langfuse-specific attributes
# https://langfuse.com/integrations/native/opentelemetry#property-mapping
ATTR_LANGFUSE_USER_ID = 'langfuse.user.id'
ATTR_LANGFUSE_SESSION_ID = 'langfuse.session.id'
ATTR_LANGFUSE_TAGS = 'langfuse.trace.tags'
ATTR_LANGFUSE_METADATA = 'langfuse.trace.metadata.%s'
ATTR_LANGFUSE_TRACE_INPUT = 'langfuse.trace.input'
ATTR_LANGFUSE_TRACE_OUTPUT = 'langfuse.trace.output'
ATTR_LANGFUSE_OBSERVATION_INPUT = 'langfuse.observation.input'
ATTR_LANGFUSE_OBSERVATION_OUTPUT = 'langfuse.observation.output'
end