feat: insrument captain v2 (#13439)

# Pull Request Template

## Description

Instruments captain v2

## Type of change

- [x] New feature (non-breaking change which adds functionality)

## How Has This Been Tested?

Please describe the tests that you ran to verify your changes. Provide
instructions so we can reproduce. Please also list any relevant details
for your test configuration.

Local testing:
<img width="864" height="510" alt="image"
src="https://github.com/user-attachments/assets/855ebce5-e8b8-4d22-b0bb-0d413769a6ab"
/>



## Checklist:

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my code
- [x] I have commented on my code, particularly in hard-to-understand
areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [x] Any dependent changes have been merged and published in downstream
modules

---------

Co-authored-by: Shivam Mishra <scm.mymail@gmail.com>

Gemfile

@@ -191,7 +191,7 @@ gem 'reverse_markdown'
 
 gem 'iso-639'
 gem 'ruby-openai'
-gem 'ai-agents', '>= 0.7.0'
+gem 'ai-agents'
 
 # TODO: Move this gem as a dependency of ai-agents
 gem 'ruby_llm', '>= 1.8.2'

Gemfile.lock

@@ -126,8 +126,8 @@ GEM
       jbuilder (~> 2)
       rails (>= 4.2, < 7.2)
       selectize-rails (~> 0.6)
-    ai-agents (0.7.0)
+    ai-agents (0.9.0)
-      ruby_llm (~> 1.8.2)
+      ruby_llm (~> 1.9.1)
     annotaterb (4.20.0)
       activerecord (>= 6.0.0)
       activesupport (>= 6.0.0)
@@ -314,7 +314,7 @@ GEM
     faraday-net_http_persistent (2.1.0)
       faraday (~> 2.5)
       net-http-persistent (~> 4.0)
-    faraday-retry (2.2.1)
+    faraday-retry (2.4.0)
       faraday (~> 2.0)
     faraday_middleware-aws-sigv4 (1.0.1)
       aws-sigv4 (~> 1.0)
@@ -540,7 +540,7 @@ GEM
       net-imap
       net-pop
       net-smtp
-    marcel (1.0.4)
+    marcel (1.1.0)
     maxminddb (0.1.22)
     meta_request (0.8.5)
       rack-contrib (>= 1.1, < 3)
@@ -559,7 +559,7 @@ GEM
     multi_json (1.15.0)
     multi_xml (0.8.0)
       bigdecimal (>= 3.1, < 5)
-    multipart-post (2.3.0)
+    multipart-post (2.4.1)
     mutex_m (0.3.0)
     neighbor (0.2.3)
       activerecord (>= 5.2)
@@ -825,7 +825,7 @@ GEM
     ruby2ruby (2.5.0)
       ruby_parser (~> 3.1)
       sexp_processor (~> 4.6)
-    ruby_llm (1.8.2)
+    ruby_llm (1.9.2)
       base64
       event_stream_parser (~> 1)
       faraday (>= 1.10.0)
@@ -1004,7 +1004,7 @@ GEM
     working_hours (1.4.1)
       activesupport (>= 3.2)
       tzinfo
-    zeitwerk (2.6.17)
+    zeitwerk (2.7.4)
 
 PLATFORMS
   arm64-darwin-20
@@ -1024,7 +1024,7 @@ DEPENDENCIES
   administrate (>= 0.20.1)
   administrate-field-active_storage (>= 1.0.3)
   administrate-field-belongs_to_search (>= 0.9.0)
-  ai-agents (>= 0.7.0)
+  ai-agents
   annotaterb
   attr_extras
   audited (~> 5.4, >= 5.4.1)

enterprise/app/services/captain/assistant/agent_runner_service.rb

@@ -1,6 +1,9 @@
 require 'agents'
+require 'agents/instrumentation'
 
 class Captain::Assistant::AgentRunnerService
+  include Integrations::LlmInstrumentationConstants
+
   CONVERSATION_STATE_ATTRIBUTES = %i[
     id display_id inbox_id contact_id status priority
     label_list custom_attributes additional_attributes
@@ -22,7 +25,9 @@ class Captain::Assistant::AgentRunnerService
     context = build_context(message_history)
     message_to_process = extract_last_user_message(message_history)
     runner = Agents::Runner.with_agents(*agents)
+    runner = add_usage_metadata_callback(runner)
     runner = add_callbacks_to_runner(runner) if @callbacks.any?
+    install_instrumentation(runner)
     result = runner.run(message_to_process, context: context, max_turns: 100)
 
     process_agent_result(result)
@@ -50,6 +55,7 @@ class Captain::Assistant::AgentRunnerService
     end
 
     {
+      session_id: "#{@assistant.account_id}_#{@conversation&.display_id}",
       conversation_history: conversation_history,
       state: build_state
     }
@@ -124,6 +130,31 @@ class Captain::Assistant::AgentRunnerService
     [assistant_agent] + scenario_agents
   end
 
+  def install_instrumentation(runner)
+    return unless ChatwootApp.otel_enabled?
+
+    Agents::Instrumentation.install(
+      runner,
+      tracer: OpentelemetryConfig.tracer,
+      trace_name: 'llm.captain_v2',
+      span_attributes: {
+        ATTR_LANGFUSE_TAGS => ['captain_v2'].to_json
+      },
+      attribute_provider: ->(context_wrapper) { dynamic_trace_attributes(context_wrapper) }
+    )
+  end
+
+  def dynamic_trace_attributes(context_wrapper)
+    state = context_wrapper&.context&.dig(:state) || {}
+    conversation = state[:conversation] || {}
+    {
+      ATTR_LANGFUSE_USER_ID => state[:account_id],
+      format(ATTR_LANGFUSE_METADATA, 'assistant_id') => state[:assistant_id],
+      format(ATTR_LANGFUSE_METADATA, 'conversation_id') => conversation[:id],
+      format(ATTR_LANGFUSE_METADATA, 'conversation_display_id') => conversation[:display_id]
+    }.compact.transform_values(&:to_s)
+  end
+
   def add_callbacks_to_runner(runner)
     runner = add_agent_thinking_callback(runner) if @callbacks[:on_agent_thinking]
     runner = add_tool_start_callback(runner) if @callbacks[:on_tool_start]
@@ -132,6 +163,36 @@ class Captain::Assistant::AgentRunnerService
     runner
   end
 
+  def add_usage_metadata_callback(runner)
+    return runner unless ChatwootApp.otel_enabled?
+
+    handoff_tool_name = Captain::Tools::HandoffTool.new(@assistant).name
+
+    runner.on_tool_complete do |tool_name, _tool_result, context_wrapper|
+      track_handoff_usage(tool_name, handoff_tool_name, context_wrapper)
+    end
+
+    runner.on_run_complete do |_agent_name, _result, context_wrapper|
+      write_credits_used_metadata(context_wrapper)
+    end
+    runner
+  end
+
+  def track_handoff_usage(tool_name, handoff_tool_name, context_wrapper)
+    return unless context_wrapper&.context
+    return unless tool_name.to_s == handoff_tool_name
+
+    context_wrapper.context[:captain_v2_handoff_tool_called] = true
+  end
+
+  def write_credits_used_metadata(context_wrapper)
+    root_span = context_wrapper&.context&.dig(:__otel_tracing, :root_span)
+    return unless root_span
+
+    credits_used = !context_wrapper.context[:captain_v2_handoff_tool_called]
+    root_span.set_attribute(format(ATTR_LANGFUSE_METADATA, 'credits_used'), credits_used)
+  end
+
   def add_agent_thinking_callback(runner)
     runner.on_agent_thinking do |*args|
       @callbacks[:on_agent_thinking].call(*args)

spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb

@@ -75,6 +75,7 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
 
     it 'runs agent with extracted user message and context' do
       expected_context = {
+        session_id: "#{account.id}_#{conversation.display_id}",
         conversation_history: [
           { role: :user, content: 'Hello there', agent_name: nil },
           { role: :assistant, content: 'Hi! How can I help you?', agent_name: 'Assistant' },
@@ -306,6 +307,60 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
     end
   end
 
+  describe '#add_usage_metadata_callback' do
+    it 'sets credits_used=false when handoff tool is used' do
+      service = described_class.new(assistant: assistant, conversation: conversation)
+      runner = instance_double(Agents::AgentRunner)
+      tool_complete_callback = nil
+      run_complete_callback = nil
+      span_class = Class.new do
+        def set_attribute(*); end
+      end
+      root_span = instance_double(span_class)
+      context_wrapper = Struct.new(:context).new({ __otel_tracing: { root_span: root_span } })
+
+      allow(ChatwootApp).to receive(:otel_enabled?).and_return(true)
+      allow(runner).to receive(:on_tool_complete) do |&block|
+        tool_complete_callback = block
+        runner
+      end
+      allow(runner).to receive(:on_run_complete) do |&block|
+        run_complete_callback = block
+        runner
+      end
+
+      service.send(:add_usage_metadata_callback, runner)
+
+      tool_complete_callback.call(Captain::Tools::HandoffTool.new(assistant).name, 'ok', context_wrapper)
+
+      expect(root_span).to receive(:set_attribute).with('langfuse.trace.metadata.credits_used', false)
+      run_complete_callback.call('assistant', nil, context_wrapper)
+    end
+
+    it 'sets credits_used=true when handoff tool is not used' do
+      service = described_class.new(assistant: assistant, conversation: conversation)
+      runner = instance_double(Agents::AgentRunner)
+      run_complete_callback = nil
+      span_class = Class.new do
+        def set_attribute(*); end
+      end
+      root_span = instance_double(span_class)
+      context_wrapper = Struct.new(:context).new({ __otel_tracing: { root_span: root_span } })
+
+      allow(ChatwootApp).to receive(:otel_enabled?).and_return(true)
+      allow(runner).to receive(:on_tool_complete).and_return(runner)
+      allow(runner).to receive(:on_run_complete) do |&block|
+        run_complete_callback = block
+        runner
+      end
+
+      service.send(:add_usage_metadata_callback, runner)
+
+      expect(root_span).to receive(:set_attribute).with('langfuse.trace.metadata.credits_used', true)
+      run_complete_callback.call('assistant', nil, context_wrapper)
+    end
+  end
+
   describe 'constants' do
     it 'defines conversation state attributes' do
       expect(described_class::CONVERSATION_STATE_ATTRIBUTES).to include(