From 38743836987a218aeec0ee2599df1151bc0e2de6 Mon Sep 17 00:00:00 2001
From: Aakash Bakhle <48802744+aakashb95@users.noreply.github.com>
Date: Tue, 17 Feb 2026 13:28:26 +0530
Subject: [PATCH] feat: insrument captain v2 (#13439)
# Pull Request Template
## Description
Instruments captain v2
## Type of change
- [x] New feature (non-breaking change which adds functionality)
## How Has This Been Tested?
Please describe the tests that you ran to verify your changes. Provide
instructions so we can reproduce. Please also list any relevant details
for your test configuration.
Local testing:
## Checklist:
- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my code
- [x] I have commented on my code, particularly in hard-to-understand
areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [x] Any dependent changes have been merged and published in downstream
modules
---------
Co-authored-by: Shivam Mishra
---
Gemfile | 2 +-
Gemfile.lock | 16 ++---
.../captain/assistant/agent_runner_service.rb | 61 +++++++++++++++++++
.../assistant/agent_runner_service_spec.rb | 55 +++++++++++++++++
4 files changed, 125 insertions(+), 9 deletions(-)
diff --git a/Gemfile b/Gemfile
index 2023c32b1..e6c5a5250 100644
--- a/Gemfile
+++ b/Gemfile
@@ -191,7 +191,7 @@ gem 'reverse_markdown'
gem 'iso-639'
gem 'ruby-openai'
-gem 'ai-agents', '>= 0.7.0'
+gem 'ai-agents'
# TODO: Move this gem as a dependency of ai-agents
gem 'ruby_llm', '>= 1.8.2'
diff --git a/Gemfile.lock b/Gemfile.lock
index db9b59c66..cc1f9e253 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -126,8 +126,8 @@ GEM
jbuilder (~> 2)
rails (>= 4.2, < 7.2)
selectize-rails (~> 0.6)
- ai-agents (0.7.0)
- ruby_llm (~> 1.8.2)
+ ai-agents (0.9.0)
+ ruby_llm (~> 1.9.1)
annotaterb (4.20.0)
activerecord (>= 6.0.0)
activesupport (>= 6.0.0)
@@ -314,7 +314,7 @@ GEM
faraday-net_http_persistent (2.1.0)
faraday (~> 2.5)
net-http-persistent (~> 4.0)
- faraday-retry (2.2.1)
+ faraday-retry (2.4.0)
faraday (~> 2.0)
faraday_middleware-aws-sigv4 (1.0.1)
aws-sigv4 (~> 1.0)
@@ -540,7 +540,7 @@ GEM
net-imap
net-pop
net-smtp
- marcel (1.0.4)
+ marcel (1.1.0)
maxminddb (0.1.22)
meta_request (0.8.5)
rack-contrib (>= 1.1, < 3)
@@ -559,7 +559,7 @@ GEM
multi_json (1.15.0)
multi_xml (0.8.0)
bigdecimal (>= 3.1, < 5)
- multipart-post (2.3.0)
+ multipart-post (2.4.1)
mutex_m (0.3.0)
neighbor (0.2.3)
activerecord (>= 5.2)
@@ -825,7 +825,7 @@ GEM
ruby2ruby (2.5.0)
ruby_parser (~> 3.1)
sexp_processor (~> 4.6)
- ruby_llm (1.8.2)
+ ruby_llm (1.9.2)
base64
event_stream_parser (~> 1)
faraday (>= 1.10.0)
@@ -1004,7 +1004,7 @@ GEM
working_hours (1.4.1)
activesupport (>= 3.2)
tzinfo
- zeitwerk (2.6.17)
+ zeitwerk (2.7.4)
PLATFORMS
arm64-darwin-20
@@ -1024,7 +1024,7 @@ DEPENDENCIES
administrate (>= 0.20.1)
administrate-field-active_storage (>= 1.0.3)
administrate-field-belongs_to_search (>= 0.9.0)
- ai-agents (>= 0.7.0)
+ ai-agents
annotaterb
attr_extras
audited (~> 5.4, >= 5.4.1)
diff --git a/enterprise/app/services/captain/assistant/agent_runner_service.rb b/enterprise/app/services/captain/assistant/agent_runner_service.rb
index 9c4e56841..a40e096e6 100644
--- a/enterprise/app/services/captain/assistant/agent_runner_service.rb
+++ b/enterprise/app/services/captain/assistant/agent_runner_service.rb
@@ -1,6 +1,9 @@
require 'agents'
+require 'agents/instrumentation'
class Captain::Assistant::AgentRunnerService
+ include Integrations::LlmInstrumentationConstants
+
CONVERSATION_STATE_ATTRIBUTES = %i[
id display_id inbox_id contact_id status priority
label_list custom_attributes additional_attributes
@@ -22,7 +25,9 @@ class Captain::Assistant::AgentRunnerService
context = build_context(message_history)
message_to_process = extract_last_user_message(message_history)
runner = Agents::Runner.with_agents(*agents)
+ runner = add_usage_metadata_callback(runner)
runner = add_callbacks_to_runner(runner) if @callbacks.any?
+ install_instrumentation(runner)
result = runner.run(message_to_process, context: context, max_turns: 100)
process_agent_result(result)
@@ -50,6 +55,7 @@ class Captain::Assistant::AgentRunnerService
end
{
+ session_id: "#{@assistant.account_id}_#{@conversation&.display_id}",
conversation_history: conversation_history,
state: build_state
}
@@ -124,6 +130,31 @@ class Captain::Assistant::AgentRunnerService
[assistant_agent] + scenario_agents
end
+ def install_instrumentation(runner)
+ return unless ChatwootApp.otel_enabled?
+
+ Agents::Instrumentation.install(
+ runner,
+ tracer: OpentelemetryConfig.tracer,
+ trace_name: 'llm.captain_v2',
+ span_attributes: {
+ ATTR_LANGFUSE_TAGS => ['captain_v2'].to_json
+ },
+ attribute_provider: ->(context_wrapper) { dynamic_trace_attributes(context_wrapper) }
+ )
+ end
+
+ def dynamic_trace_attributes(context_wrapper)
+ state = context_wrapper&.context&.dig(:state) || {}
+ conversation = state[:conversation] || {}
+ {
+ ATTR_LANGFUSE_USER_ID => state[:account_id],
+ format(ATTR_LANGFUSE_METADATA, 'assistant_id') => state[:assistant_id],
+ format(ATTR_LANGFUSE_METADATA, 'conversation_id') => conversation[:id],
+ format(ATTR_LANGFUSE_METADATA, 'conversation_display_id') => conversation[:display_id]
+ }.compact.transform_values(&:to_s)
+ end
+
def add_callbacks_to_runner(runner)
runner = add_agent_thinking_callback(runner) if @callbacks[:on_agent_thinking]
runner = add_tool_start_callback(runner) if @callbacks[:on_tool_start]
@@ -132,6 +163,36 @@ class Captain::Assistant::AgentRunnerService
runner
end
+ def add_usage_metadata_callback(runner)
+ return runner unless ChatwootApp.otel_enabled?
+
+ handoff_tool_name = Captain::Tools::HandoffTool.new(@assistant).name
+
+ runner.on_tool_complete do |tool_name, _tool_result, context_wrapper|
+ track_handoff_usage(tool_name, handoff_tool_name, context_wrapper)
+ end
+
+ runner.on_run_complete do |_agent_name, _result, context_wrapper|
+ write_credits_used_metadata(context_wrapper)
+ end
+ runner
+ end
+
+ def track_handoff_usage(tool_name, handoff_tool_name, context_wrapper)
+ return unless context_wrapper&.context
+ return unless tool_name.to_s == handoff_tool_name
+
+ context_wrapper.context[:captain_v2_handoff_tool_called] = true
+ end
+
+ def write_credits_used_metadata(context_wrapper)
+ root_span = context_wrapper&.context&.dig(:__otel_tracing, :root_span)
+ return unless root_span
+
+ credits_used = !context_wrapper.context[:captain_v2_handoff_tool_called]
+ root_span.set_attribute(format(ATTR_LANGFUSE_METADATA, 'credits_used'), credits_used)
+ end
+
def add_agent_thinking_callback(runner)
runner.on_agent_thinking do |*args|
@callbacks[:on_agent_thinking].call(*args)
diff --git a/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb b/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb
index 2c05860e2..04fa0f967 100644
--- a/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb
+++ b/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb
@@ -75,6 +75,7 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
it 'runs agent with extracted user message and context' do
expected_context = {
+ session_id: "#{account.id}_#{conversation.display_id}",
conversation_history: [
{ role: :user, content: 'Hello there', agent_name: nil },
{ role: :assistant, content: 'Hi! How can I help you?', agent_name: 'Assistant' },
@@ -306,6 +307,60 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
end
end
+ describe '#add_usage_metadata_callback' do
+ it 'sets credits_used=false when handoff tool is used' do
+ service = described_class.new(assistant: assistant, conversation: conversation)
+ runner = instance_double(Agents::AgentRunner)
+ tool_complete_callback = nil
+ run_complete_callback = nil
+ span_class = Class.new do
+ def set_attribute(*); end
+ end
+ root_span = instance_double(span_class)
+ context_wrapper = Struct.new(:context).new({ __otel_tracing: { root_span: root_span } })
+
+ allow(ChatwootApp).to receive(:otel_enabled?).and_return(true)
+ allow(runner).to receive(:on_tool_complete) do |&block|
+ tool_complete_callback = block
+ runner
+ end
+ allow(runner).to receive(:on_run_complete) do |&block|
+ run_complete_callback = block
+ runner
+ end
+
+ service.send(:add_usage_metadata_callback, runner)
+
+ tool_complete_callback.call(Captain::Tools::HandoffTool.new(assistant).name, 'ok', context_wrapper)
+
+ expect(root_span).to receive(:set_attribute).with('langfuse.trace.metadata.credits_used', false)
+ run_complete_callback.call('assistant', nil, context_wrapper)
+ end
+
+ it 'sets credits_used=true when handoff tool is not used' do
+ service = described_class.new(assistant: assistant, conversation: conversation)
+ runner = instance_double(Agents::AgentRunner)
+ run_complete_callback = nil
+ span_class = Class.new do
+ def set_attribute(*); end
+ end
+ root_span = instance_double(span_class)
+ context_wrapper = Struct.new(:context).new({ __otel_tracing: { root_span: root_span } })
+
+ allow(ChatwootApp).to receive(:otel_enabled?).and_return(true)
+ allow(runner).to receive(:on_tool_complete).and_return(runner)
+ allow(runner).to receive(:on_run_complete) do |&block|
+ run_complete_callback = block
+ runner
+ end
+
+ service.send(:add_usage_metadata_callback, runner)
+
+ expect(root_span).to receive(:set_attribute).with('langfuse.trace.metadata.credits_used', true)
+ run_complete_callback.call('assistant', nil, context_wrapper)
+ end
+ end
+
describe 'constants' do
it 'defines conversation state attributes' do
expect(described_class::CONVERSATION_STATE_ATTRIBUTES).to include(