feat: scenario agents & runner (#11944)

Co-authored-by: Muhsin Keloth <muhsinkeramam@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Sojan Jose <sojan@pepalo.com>
Co-authored-by: Pranav <pranav@chatwoot.com>
Co-authored-by: Sivin Varghese <64252451+iamsivin@users.noreply.github.com>
This commit is contained in:
Shivam Mishra
2025-08-14 12:39:21 +05:30
committed by GitHub
parent 14471cc20c
commit c6be04cdc1
21 changed files with 1437 additions and 16 deletions

View File

@@ -9,6 +9,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
describe '#perform' do
let(:conversation) { create(:conversation, inbox: inbox, account: account) }
let(:mock_llm_chat_service) { instance_double(Captain::Llm::AssistantChatService) }
let(:mock_agent_runner_service) { instance_double(Captain::Assistant::AgentRunnerService) }
before do
create(:message, conversation: conversation, content: 'Hello', message_type: :incoming)
@@ -16,19 +17,79 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
allow(inbox).to receive(:captain_active?).and_return(true)
allow(Captain::Llm::AssistantChatService).to receive(:new).and_return(mock_llm_chat_service)
allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain Specs' })
allow(Captain::Assistant::AgentRunnerService).to receive(:new).and_return(mock_agent_runner_service)
allow(mock_agent_runner_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain V2' })
end
it 'generates and processes response' do
described_class.perform_now(conversation, assistant)
expect(conversation.messages.count).to eq(2)
expect(conversation.messages.outgoing.count).to eq(1)
expect(conversation.messages.last.content).to eq('Hey, welcome to Captain Specs')
context 'when captain_v2 is disabled' do
before do
allow(account).to receive(:feature_enabled?).and_return(false)
allow(account).to receive(:feature_enabled?).with('captain_integration_v2').and_return(false)
end
it 'uses Captain::Llm::AssistantChatService' do
expect(Captain::Llm::AssistantChatService).to receive(:new).with(assistant: assistant)
expect(Captain::Assistant::AgentRunnerService).not_to receive(:new)
described_class.perform_now(conversation, assistant)
expect(conversation.messages.last.content).to eq('Hey, welcome to Captain Specs')
end
it 'generates and processes response' do
described_class.perform_now(conversation, assistant)
expect(conversation.messages.count).to eq(2)
expect(conversation.messages.outgoing.count).to eq(1)
expect(conversation.messages.last.content).to eq('Hey, welcome to Captain Specs')
end
it 'increments usage response' do
described_class.perform_now(conversation, assistant)
account.reload
expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1)
end
end
it 'increments usage response' do
described_class.perform_now(conversation, assistant)
account.reload
expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1)
context 'when captain_v2 is enabled' do
before do
allow(account).to receive(:feature_enabled?).and_return(false)
allow(account).to receive(:feature_enabled?).with('captain_integration_v2').and_return(true)
end
it 'uses Captain::Assistant::AgentRunnerService' do
expect(Captain::Assistant::AgentRunnerService).to receive(:new).with(
assistant: assistant,
conversation: conversation
)
expect(Captain::Llm::AssistantChatService).not_to receive(:new)
described_class.perform_now(conversation, assistant)
expect(conversation.messages.last.content).to eq('Hey, welcome to Captain V2')
end
it 'passes message history to agent runner service' do
expected_messages = [
{ content: 'Hello', role: 'user' }
]
expect(mock_agent_runner_service).to receive(:generate_response).with(
message_history: expected_messages
)
described_class.perform_now(conversation, assistant)
end
it 'generates and processes response' do
described_class.perform_now(conversation, assistant)
expect(conversation.messages.count).to eq(2)
expect(conversation.messages.outgoing.count).to eq(1)
expect(conversation.messages.last.content).to eq('Hey, welcome to Captain V2')
end
it 'increments usage response' do
described_class.perform_now(conversation, assistant)
account.reload
expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1)
end
end
context 'when message contains an image' do

View File

@@ -0,0 +1,123 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Captain::PromptRenderer do
let(:template_name) { 'test_template' }
let(:template_content) { 'Hello {{name}}, your balance is {{balance}}' }
let(:template_path) { Rails.root.join('enterprise', 'lib', 'captain', 'prompts', "#{template_name}.liquid") }
let(:context) { { name: 'John', balance: 100 } }
before do
allow(File).to receive(:exist?).and_return(false)
allow(File).to receive(:exist?).with(template_path).and_return(true)
allow(File).to receive(:read).with(template_path).and_return(template_content)
end
describe '.render' do
it 'renders template with context' do
result = described_class.render(template_name, context)
expect(result).to eq('Hello John, your balance is 100')
end
it 'handles string keys in context' do
string_context = { 'name' => 'Jane', 'balance' => 200 }
result = described_class.render(template_name, string_context)
expect(result).to eq('Hello Jane, your balance is 200')
end
it 'handles mixed symbol and string keys' do
mixed_context = { :name => 'Bob', 'balance' => 300 }
result = described_class.render(template_name, mixed_context)
expect(result).to eq('Hello Bob, your balance is 300')
end
it 'handles nested hash context' do
nested_template = 'User: {{user.name}}, Account: {{user.account.type}}'
nested_context = { user: { name: 'Alice', account: { type: 'premium' } } }
allow(File).to receive(:read).with(template_path).and_return(nested_template)
result = described_class.render(template_name, nested_context)
expect(result).to eq('User: Alice, Account: premium')
end
it 'handles empty context' do
simple_template = 'Hello World'
allow(File).to receive(:read).with(template_path).and_return(simple_template)
result = described_class.render(template_name, {})
expect(result).to eq('Hello World')
end
it 'loads and parses liquid template' do
liquid_template_double = instance_double(Liquid::Template)
allow(Liquid::Template).to receive(:parse).with(template_content).and_return(liquid_template_double)
allow(liquid_template_double).to receive(:render).with(hash_including('name', 'balance')).and_return('rendered')
result = described_class.render(template_name, context)
expect(result).to eq('rendered')
expect(Liquid::Template).to have_received(:parse).with(template_content)
end
end
describe '.load_template' do
it 'reads template file from correct path' do
described_class.send(:load_template, template_name)
expect(File).to have_received(:read).with(template_path)
end
it 'raises error when template does not exist' do
allow(File).to receive(:exist?).with(template_path).and_return(false)
expect { described_class.send(:load_template, template_name) }
.to raise_error("Template not found: #{template_name}")
end
it 'constructs correct template path' do
expected_path = Rails.root.join('enterprise/lib/captain/prompts/my_template.liquid')
allow(File).to receive(:exist?).with(expected_path).and_return(true)
allow(File).to receive(:read).with(expected_path).and_return('test content')
described_class.send(:load_template, 'my_template')
expect(File).to have_received(:exist?).with(expected_path)
end
end
describe '.stringify_keys' do
it 'converts symbol keys to strings' do
hash = { name: 'John', age: 30 }
result = described_class.send(:stringify_keys, hash)
expect(result).to eq({ 'name' => 'John', 'age' => 30 })
end
it 'handles nested hashes' do
hash = { user: { name: 'John', profile: { age: 30 } } }
result = described_class.send(:stringify_keys, hash)
expect(result).to eq({ 'user' => { 'name' => 'John', 'profile' => { 'age' => 30 } } })
end
it 'handles arrays with hashes' do
hash = { users: [{ name: 'John' }, { name: 'Jane' }] }
result = described_class.send(:stringify_keys, hash)
expect(result).to eq({ 'users' => [{ 'name' => 'John' }, { 'name' => 'Jane' }] })
end
it 'handles empty hash' do
result = described_class.send(:stringify_keys, {})
expect(result).to eq({})
end
end
end

View File

@@ -0,0 +1,186 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Concerns::Agentable do
let(:dummy_class) do
Class.new do
include Concerns::Agentable
attr_accessor :temperature
def initialize(name: 'Test Agent', temperature: 0.8)
@name = name
@temperature = temperature
end
def self.name
'DummyClass'
end
private
def agent_name
@name
end
def prompt_context
{ base_key: 'base_value' }
end
end
end
let(:dummy_instance) { dummy_class.new }
let(:mock_agents_agent) { instance_double(Agents::Agent) }
let(:mock_installation_config) { instance_double(InstallationConfig, value: 'gpt-4-turbo') }
before do
allow(Agents::Agent).to receive(:new).and_return(mock_agents_agent)
allow(InstallationConfig).to receive(:find_by).with(name: 'CAPTAIN_OPEN_AI_MODEL').and_return(mock_installation_config)
allow(Captain::PromptRenderer).to receive(:render).and_return('rendered_template')
end
describe '#agent' do
it 'creates an Agents::Agent with correct parameters' do
expect(Agents::Agent).to receive(:new).with(
name: 'Test Agent',
instructions: instance_of(Proc),
tools: [],
model: 'gpt-4-turbo',
temperature: 0.8,
response_schema: Captain::ResponseSchema
)
dummy_instance.agent
end
it 'converts nil temperature to 0.0' do
dummy_instance.temperature = nil
expect(Agents::Agent).to receive(:new).with(
hash_including(temperature: 0.0)
)
dummy_instance.agent
end
it 'converts temperature to float' do
dummy_instance.temperature = '0.5'
expect(Agents::Agent).to receive(:new).with(
hash_including(temperature: 0.5)
)
dummy_instance.agent
end
end
describe '#agent_instructions' do
it 'calls Captain::PromptRenderer with base context' do
expect(Captain::PromptRenderer).to receive(:render).with(
'dummy_class',
hash_including(base_key: 'base_value')
)
dummy_instance.agent_instructions
end
it 'merges context state when provided' do
context_double = instance_double(Agents::RunContext,
context: {
state: {
conversation: { id: 123 },
contact: { name: 'John' }
}
})
expected_context = {
base_key: 'base_value',
conversation: { id: 123 },
contact: { name: 'John' }
}
expect(Captain::PromptRenderer).to receive(:render).with(
'dummy_class',
hash_including(expected_context)
)
dummy_instance.agent_instructions(context_double)
end
it 'handles context without state' do
context_double = instance_double(Agents::RunContext, context: {})
expect(Captain::PromptRenderer).to receive(:render).with(
'dummy_class',
hash_including(
base_key: 'base_value',
conversation: {},
contact: {}
)
)
dummy_instance.agent_instructions(context_double)
end
end
describe '#template_name' do
it 'returns underscored class name' do
expect(dummy_instance.send(:template_name)).to eq('dummy_class')
end
end
describe '#agent_tools' do
it 'returns empty array by default' do
expect(dummy_instance.send(:agent_tools)).to eq([])
end
end
describe '#agent_model' do
it 'returns value from InstallationConfig when present' do
expect(dummy_instance.send(:agent_model)).to eq('gpt-4-turbo')
end
it 'returns default model when config not found' do
allow(InstallationConfig).to receive(:find_by).and_return(nil)
expect(dummy_instance.send(:agent_model)).to eq('gpt-4.1-mini')
end
it 'returns default model when config value is nil' do
allow(mock_installation_config).to receive(:value).and_return(nil)
expect(dummy_instance.send(:agent_model)).to eq('gpt-4.1-mini')
end
end
describe '#agent_response_schema' do
it 'returns Captain::ResponseSchema' do
expect(dummy_instance.send(:agent_response_schema)).to eq(Captain::ResponseSchema)
end
end
describe 'required methods' do
let(:incomplete_class) do
Class.new do
include Concerns::Agentable
end
end
let(:incomplete_instance) { incomplete_class.new }
describe '#agent_name' do
it 'raises NotImplementedError when not implemented' do
expect { incomplete_instance.send(:agent_name) }
.to raise_error(NotImplementedError, /must implement agent_name/)
end
end
describe '#prompt_context' do
it 'raises NotImplementedError when not implemented' do
expect { incomplete_instance.send(:prompt_context) }
.to raise_error(NotImplementedError, /must implement prompt_context/)
end
end
end
end

View File

@@ -0,0 +1,320 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Captain::Assistant::AgentRunnerService do
let(:account) { create(:account) }
let(:inbox) { create(:inbox, account: account) }
let(:contact) { create(:contact, account: account) }
let(:conversation) { create(:conversation, account: account, inbox: inbox, contact: contact) }
let(:assistant) { create(:captain_assistant, account: account) }
let(:scenario) { create(:captain_scenario, assistant: assistant, enabled: true) }
let(:mock_runner) { instance_double(Agents::Runner) }
let(:mock_agent) { instance_double(Agents::Agent) }
let(:mock_scenario_agent) { instance_double(Agents::Agent) }
let(:mock_result) { instance_double(Agents::RunResult, output: { 'response' => 'Test response' }) }
let(:message_history) do
[
{ role: 'user', content: 'Hello there' },
{ role: 'assistant', content: 'Hi! How can I help you?', agent_name: 'Assistant' },
{ role: 'user', content: 'I need help with my account' }
]
end
before do
allow(assistant).to receive(:agent).and_return(mock_agent)
scenarios_relation = instance_double(Captain::Scenario)
allow(scenarios_relation).to receive(:enabled).and_return([scenario])
allow(assistant).to receive(:scenarios).and_return(scenarios_relation)
allow(scenario).to receive(:agent).and_return(mock_scenario_agent)
allow(Agents::Runner).to receive(:with_agents).and_return(mock_runner)
allow(mock_runner).to receive(:run).and_return(mock_result)
allow(mock_agent).to receive(:register_handoffs)
allow(mock_scenario_agent).to receive(:register_handoffs)
end
describe '#initialize' do
it 'sets instance variables correctly' do
service = described_class.new(assistant: assistant, conversation: conversation)
expect(service.instance_variable_get(:@assistant)).to eq(assistant)
expect(service.instance_variable_get(:@conversation)).to eq(conversation)
expect(service.instance_variable_get(:@callbacks)).to eq({})
end
it 'accepts callbacks parameter' do
callbacks = { on_agent_thinking: proc { |x| x } }
service = described_class.new(assistant: assistant, callbacks: callbacks)
expect(service.instance_variable_get(:@callbacks)).to eq(callbacks)
end
end
describe '#generate_response' do
subject(:service) { described_class.new(assistant: assistant, conversation: conversation) }
it 'builds agents and wires them together' do
expect(assistant).to receive(:agent).and_return(mock_agent)
scenarios_relation = instance_double(Captain::Scenario)
allow(scenarios_relation).to receive(:enabled).and_return([scenario])
expect(assistant).to receive(:scenarios).and_return(scenarios_relation)
expect(scenario).to receive(:agent).and_return(mock_scenario_agent)
expect(mock_agent).to receive(:register_handoffs).with(mock_scenario_agent)
expect(mock_scenario_agent).to receive(:register_handoffs).with(mock_agent)
service.generate_response(message_history: message_history)
end
it 'creates runner with agents' do
expect(Agents::Runner).to receive(:with_agents).with(mock_agent, mock_scenario_agent)
service.generate_response(message_history: message_history)
end
it 'runs agent with extracted user message and context' do
expected_context = {
conversation_history: [
{ role: :user, content: 'Hello there', agent_name: nil },
{ role: :assistant, content: 'Hi! How can I help you?', agent_name: 'Assistant' },
{ role: :user, content: 'I need help with my account', agent_name: nil }
],
state: hash_including(
account_id: account.id,
assistant_id: assistant.id,
conversation: hash_including(id: conversation.id),
contact: hash_including(id: contact.id)
)
}
expect(mock_runner).to receive(:run).with(
'I need help with my account',
context: expected_context
)
service.generate_response(message_history: message_history)
end
it 'processes and formats agent result' do
result = service.generate_response(message_history: message_history)
expect(result).to eq({ 'response' => 'Test response' })
end
context 'when no scenarios are enabled' do
before do
scenarios_relation = instance_double(Captain::Scenario)
allow(scenarios_relation).to receive(:enabled).and_return([])
allow(assistant).to receive(:scenarios).and_return(scenarios_relation)
end
it 'only uses assistant agent' do
expect(Agents::Runner).to receive(:with_agents).with(mock_agent)
expect(mock_agent).not_to receive(:register_handoffs)
service.generate_response(message_history: message_history)
end
end
context 'when agent result is a string' do
let(:mock_result) { instance_double(Agents::RunResult, output: 'Simple string response') }
it 'formats string response correctly' do
result = service.generate_response(message_history: message_history)
expect(result).to eq({
'response' => 'Simple string response',
'reasoning' => 'Processed by agent'
})
end
end
context 'when an error occurs' do
let(:error) { StandardError.new('Test error') }
before do
allow(mock_runner).to receive(:run).and_raise(error)
allow(ChatwootExceptionTracker).to receive(:new).and_return(
instance_double(ChatwootExceptionTracker, capture_exception: true)
)
end
it 'captures exception and returns error response' do
expect(ChatwootExceptionTracker).to receive(:new).with(error, account: conversation.account)
result = service.generate_response(message_history: message_history)
expect(result).to eq({
'response' => 'conversation_handoff',
'reasoning' => 'Error occurred: Test error'
})
end
it 'logs error details' do
expect(Rails.logger).to receive(:error).with('[Captain V2] AgentRunnerService error: Test error')
expect(Rails.logger).to receive(:error).with(kind_of(String))
service.generate_response(message_history: message_history)
end
context 'when conversation is nil' do
subject(:service) { described_class.new(assistant: assistant, conversation: nil) }
it 'handles missing conversation gracefully' do
expect(ChatwootExceptionTracker).to receive(:new).with(error, account: nil)
result = service.generate_response(message_history: message_history)
expect(result).to eq({
'response' => 'conversation_handoff',
'reasoning' => 'Error occurred: Test error'
})
end
end
end
end
describe '#build_context' do
subject(:service) { described_class.new(assistant: assistant, conversation: conversation) }
it 'builds context with conversation history and state' do
context = service.send(:build_context, message_history)
expect(context).to include(
conversation_history: array_including(
{ role: :user, content: 'Hello there', agent_name: nil },
{ role: :assistant, content: 'Hi! How can I help you?', agent_name: 'Assistant' }
),
state: hash_including(
account_id: account.id,
assistant_id: assistant.id
)
)
end
context 'with multimodal content' do
let(:multimodal_message_history) do
[
{
role: 'user',
content: [
{ type: 'text', text: 'Can you help with this image?' },
{ type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } }
]
}
]
end
it 'extracts text content from multimodal messages' do
context = service.send(:build_context, multimodal_message_history)
expect(context[:conversation_history].first[:content]).to eq('Can you help with this image?')
end
end
end
describe '#extract_last_user_message' do
subject(:service) { described_class.new(assistant: assistant, conversation: conversation) }
it 'extracts the last user message' do
result = service.send(:extract_last_user_message, message_history)
expect(result).to eq('I need help with my account')
end
end
describe '#extract_text_from_content' do
subject(:service) { described_class.new(assistant: assistant, conversation: conversation) }
it 'extracts text from string content' do
result = service.send(:extract_text_from_content, 'Simple text')
expect(result).to eq('Simple text')
end
it 'extracts response from hash content' do
content = { 'response' => 'Hash response' }
result = service.send(:extract_text_from_content, content)
expect(result).to eq('Hash response')
end
it 'extracts text from multimodal array content' do
content = [
{ type: 'text', text: 'First part' },
{ type: 'image_url', image_url: { url: 'image.jpg' } },
{ type: 'text', text: 'Second part' }
]
result = service.send(:extract_text_from_content, content)
expect(result).to eq('First part Second part')
end
end
describe '#build_state' do
subject(:service) { described_class.new(assistant: assistant, conversation: conversation) }
it 'builds state with assistant and account information' do
state = service.send(:build_state)
expect(state).to include(
account_id: account.id,
assistant_id: assistant.id,
assistant_config: assistant.config
)
end
it 'includes conversation attributes when conversation is present' do
state = service.send(:build_state)
expect(state[:conversation]).to include(
id: conversation.id,
inbox_id: inbox.id,
contact_id: contact.id,
status: conversation.status
)
end
it 'includes contact attributes when contact is present' do
state = service.send(:build_state)
expect(state[:contact]).to include(
id: contact.id,
name: contact.name,
email: contact.email
)
end
context 'when conversation is nil' do
subject(:service) { described_class.new(assistant: assistant, conversation: nil) }
it 'builds state without conversation and contact' do
state = service.send(:build_state)
expect(state).to include(
account_id: account.id,
assistant_id: assistant.id,
assistant_config: assistant.config
)
expect(state).not_to have_key(:conversation)
expect(state).not_to have_key(:contact)
end
end
end
describe 'constants' do
it 'defines conversation state attributes' do
expect(described_class::CONVERSATION_STATE_ATTRIBUTES).to include(
:id, :display_id, :inbox_id, :contact_id, :status, :priority
)
end
it 'defines contact state attributes' do
expect(described_class::CONTACT_STATE_ATTRIBUTES).to include(
:id, :name, :email, :phone_number, :identifier, :contact_type
)
end
end
end