diff --git a/enterprise/app/helpers/captain/chat_response_helper.rb b/enterprise/app/helpers/captain/chat_response_helper.rb index e0323996f..bfb8adc11 100644 --- a/enterprise/app/helpers/captain/chat_response_helper.rb +++ b/enterprise/app/helpers/captain/chat_response_helper.rb @@ -1,10 +1,13 @@ module Captain::ChatResponseHelper + include Integrations::LlmInstrumentationConstants + private def build_response(response) Rails.logger.debug { "#{self.class.name} Assistant: #{@assistant.id}, Received response #{response}" } parsed = parse_json_response(response.content) + apply_credit_usage_metadata(parsed) persist_message(parsed, 'assistant') parsed @@ -19,6 +22,26 @@ module Captain::ChatResponseHelper { 'content' => content } end + def apply_credit_usage_metadata(parsed_response) + return unless captain_v1_assistant? + + OpenTelemetry::Trace.current_span.set_attribute( + format(ATTR_LANGFUSE_METADATA, 'credit_used'), + credit_used_for_response?(parsed_response).to_s + ) + rescue StandardError => e + Rails.logger.warn "#{self.class.name} Assistant: #{@assistant.id}, Failed to set credit usage metadata: #{e.message}" + end + + def credit_used_for_response?(parsed_response) + response = parsed_response['response'] + response.present? && response != 'conversation_handoff' + end + + def captain_v1_assistant? + feature_name == 'assistant' && !@assistant.account.feature_enabled?('captain_integration_v2') + end + def persist_thinking_message(tool_call) return if @copilot_thread.blank? diff --git a/lib/captain/tool_instrumentation.rb b/lib/captain/tool_instrumentation.rb index af79a3fca..157aab829 100644 --- a/lib/captain/tool_instrumentation.rb +++ b/lib/captain/tool_instrumentation.rb @@ -15,6 +15,7 @@ module Captain::ToolInstrumentation response = yield executed = true span.set_attribute(ATTR_LANGFUSE_OBSERVATION_OUTPUT, response[:message] || response.to_json) + set_tool_session_error_attributes(span, response) if response.is_a?(Hash) end response rescue StandardError => e @@ -29,6 +30,14 @@ module Captain::ToolInstrumentation span.set_attribute(ATTR_LANGFUSE_OBSERVATION_INPUT, params[:messages].to_json) end + def set_tool_session_error_attributes(span, response) + error = response[:error] || response['error'] + return if error.blank? + + span.set_attribute(ATTR_GEN_AI_RESPONSE_ERROR, error.to_json) + span.status = OpenTelemetry::Trace::Status.error(error.to_s.truncate(1000)) + end + def record_generation(chat, message, model) return unless ChatwootApp.otel_enabled? return unless message.respond_to?(:role) && message.role.to_s == 'assistant' diff --git a/lib/integrations/llm_instrumentation.rb b/lib/integrations/llm_instrumentation.rb index c3baf291e..326bb901e 100644 --- a/lib/integrations/llm_instrumentation.rb +++ b/lib/integrations/llm_instrumentation.rb @@ -37,6 +37,7 @@ module Integrations::LlmInstrumentation result = yield executed = true span.set_attribute(ATTR_LANGFUSE_OBSERVATION_OUTPUT, result.to_json) + set_error_attributes(span, result) if result.is_a?(Hash) result end rescue StandardError => e @@ -50,9 +51,11 @@ module Integrations::LlmInstrumentation return yield unless ChatwootApp.otel_enabled? tracer.in_span(format(TOOL_SPAN_NAME, tool_name)) do |span| + span.set_attribute(ATTR_LANGFUSE_OBSERVATION_TYPE, 'tool') span.set_attribute(ATTR_LANGFUSE_OBSERVATION_INPUT, arguments.to_json) result = yield span.set_attribute(ATTR_LANGFUSE_OBSERVATION_OUTPUT, result.to_json) + set_error_attributes(span, result) if result.is_a?(Hash) result end end diff --git a/lib/integrations/llm_instrumentation_constants.rb b/lib/integrations/llm_instrumentation_constants.rb index 6ce296ee9..dfe1e7704 100644 --- a/lib/integrations/llm_instrumentation_constants.rb +++ b/lib/integrations/llm_instrumentation_constants.rb @@ -26,6 +26,7 @@ module Integrations::LlmInstrumentationConstants ATTR_LANGFUSE_METADATA = 'langfuse.trace.metadata.%s' ATTR_LANGFUSE_TRACE_INPUT = 'langfuse.trace.input' ATTR_LANGFUSE_TRACE_OUTPUT = 'langfuse.trace.output' + ATTR_LANGFUSE_OBSERVATION_TYPE = 'langfuse.observation.type' ATTR_LANGFUSE_OBSERVATION_INPUT = 'langfuse.observation.input' ATTR_LANGFUSE_OBSERVATION_OUTPUT = 'langfuse.observation.output' end diff --git a/lib/integrations/llm_instrumentation_spans.rb b/lib/integrations/llm_instrumentation_spans.rb index 824b6aa4a..85ea599f8 100644 --- a/lib/integrations/llm_instrumentation_spans.rb +++ b/lib/integrations/llm_instrumentation_spans.rb @@ -39,6 +39,7 @@ module Integrations::LlmInstrumentationSpans tool_name = tool_call.name.to_s span = tracer.start_span(format(TOOL_SPAN_NAME, tool_name)) + span.set_attribute(ATTR_LANGFUSE_OBSERVATION_TYPE, 'tool') span.set_attribute(ATTR_LANGFUSE_OBSERVATION_INPUT, tool_call.arguments.to_json) @pending_tool_spans ||= []