LeadMagnet/leadchat
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat: captain decides if conversation should be resolved or kept open (#13336)

Browse Source 
# Pull Request Template

## Description

captain decides if conversation should be resolved or open

Fixes
https://linear.app/chatwoot/issue/AI-91/make-captain-resolution-time-configurable

Update: Added 2 entries in reporting events:
`conversation_captain_handoff` and `conversation_captain_resolved`

## Type of change

Please delete options that are not relevant.

- [x] New feature (non-breaking change which adds functionality)
- [x] This change requires a documentation update

## How Has This Been Tested?

Please describe the tests that you ran to verify your changes. Provide
instructions so we can reproduce. Please also list any relevant details
for your test configuration.

LLM call decides that conversation is resolved, drops a private note
<img width="1228" height="438" alt="image"
src="https://github.com/user-attachments/assets/fb2cf1e9-4b2b-458b-a1e2-45c53d6a0158"
/>

LLM call decides conversation is still open as query was not resolved
<img width="1215" height="573" alt="image"
src="https://github.com/user-attachments/assets/2d1d5322-f567-487e-954e-11ab0798d11c"
/>


## Checklist:

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my code
- [x] I have commented on my code, particularly in hard-to-understand
areas
- [] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [x] Any dependent changes have been merged and published in downstream
modules

---------

Co-authored-by: Sojan Jose <sojan@pepalo.com>
This commit is contained in:
Aakash Bakhle
2026-03-13 10:03:58 +05:30
committed by GitHub
parent 199dcd382e
commit d6d38cdd7d
22 changed files with 949 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

336
spec/enterprise/jobs/captain/inbox_pending_conversations_resolution_job_spec.rb
View File

@@ -2,16 +2,14 @@ require 'rails_helper'
RSpec.describe Captain::InboxPendingConversationsResolutionJob, type: :job do
let!(:inbox) { create(:inbox) }
let!(:resolvable_pending_conversation) { create(:conversation, inbox: inbox, last_activity_at: 2.hours.ago, status: :pending) }
let!(:recent_pending_conversation) { create(:conversation, inbox: inbox, last_activity_at: 10.minutes.ago, status: :pending) }
let!(:recent_pending_conversation) { create(:conversation, inbox: inbox, last_activity_at: 1.minute.ago, status: :pending) }
let!(:open_conversation) { create(:conversation, inbox: inbox, last_activity_at: 1.hour.ago, status: :open) }
let!(:captain_assistant) { create(:captain_assistant, account: inbox.account) }
before do
create(:captain_inbox, inbox: inbox, captain_assistant: captain_assistant)
stub_const('Limits::BULK_ACTIONS_LIMIT', 2)
stub_const('Limits::BULK_ACTIONS_LIMIT', 3)
inbox.reload
end
@@ -20,49 +18,307 @@ RSpec.describe Captain::InboxPendingConversationsResolutionJob, type: :job do
.to have_enqueued_job.on_queue('low')
end
it 'resolves only the eligible pending conversations' do
described_class.perform_now(inbox)
expect(resolvable_pending_conversation.reload.status).to eq('resolved')
expect(recent_pending_conversation.reload.status).to eq('pending')
expect(open_conversation.reload.status).to eq('open')
end
it 'creates exactly one outgoing message with configured content' do
custom_message = 'This is a custom resolution message.'
captain_assistant.update!(config: { 'resolution_message' => custom_message })
expect do
context 'when captain_tasks is disabled' do
it 'resolves pending conversations inactive for over 1 hour' do
described_class.perform_now(inbox)
end.to change { resolvable_pending_conversation.messages.outgoing.reload.count }.by(1)
outgoing_message = resolvable_pending_conversation.messages.outgoing.last
expect(outgoing_message.content).to eq(custom_message)
expect(resolvable_pending_conversation.reload.status).to eq('resolved')
end
it 'does not resolve recent pending conversations' do
described_class.perform_now(inbox)
expect(recent_pending_conversation.reload.status).to eq('pending')
end
it 'does not affect open conversations' do
described_class.perform_now(inbox)
expect(open_conversation.reload.status).to eq('open')
end
it 'does not call ConversationCompletionService' do
allow(Captain::ConversationCompletionService).to receive(:new)
described_class.perform_now(inbox)
expect(Captain::ConversationCompletionService).not_to have_received(:new)
end
end
it 'creates an outgoing message with default auto resolution message if not configured' do
captain_assistant.update!(config: {})
context 'when captain_tasks is enabled' do
before do
allow(inbox.account).to receive(:feature_enabled?).and_call_original
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
end
described_class.perform_now(inbox)
outgoing_message = resolvable_pending_conversation.messages.outgoing.last
expect(outgoing_message.content).to eq(
I18n.t('conversations.activity.auto_resolution_message')
)
end
it 'only evaluates eligible pending conversations (inactive > 1 hour)' do
allow(Captain::ConversationCompletionService).to receive(:new).and_call_original
it 'adds the correct activity message after resolution by Captain' do
described_class.perform_now(inbox)
expected_content = I18n.t('conversations.activity.captain.resolved', user_name: captain_assistant.name)
expect(Conversations::ActivityMessageJob)
.to have_been_enqueued.with(
resolvable_pending_conversation,
{
account_id: resolvable_pending_conversation.account_id,
inbox_id: resolvable_pending_conversation.inbox_id,
message_type: :activity,
content: expected_content
}
# Mock the service to return complete for all conversations
mock_service = instance_double(Captain::ConversationCompletionService)
allow(mock_service).to receive(:perform).and_return({ complete: true, reason: 'Test' })
allow(Captain::ConversationCompletionService).to receive(:new).and_return(mock_service)
described_class.perform_now(inbox)
# Only resolvable conversation should be evaluated (not recent or open)
expect(Captain::ConversationCompletionService).to have_received(:new).with(
account: inbox.account,
conversation_display_id: resolvable_pending_conversation.display_id
)
expect(recent_pending_conversation.reload.status).to eq('pending')
expect(open_conversation.reload.status).to eq('open')
end
it 'skips auto-action if conversation receives new activity after evaluation' do
mock_service = instance_double(Captain::ConversationCompletionService)
allow(mock_service).to receive(:perform) do
resolvable_pending_conversation.update!(last_activity_at: Time.current)
{ complete: true, reason: 'Customer question was answered' }
end
allow(Captain::ConversationCompletionService).to receive(:new).and_return(mock_service)
described_class.perform_now(inbox)
expect(resolvable_pending_conversation.reload.status).to eq('pending')
expect(resolvable_pending_conversation.messages.outgoing).to be_empty
end
end
context 'when LLM evaluation returns complete' do
before do
allow(inbox.account).to receive(:feature_enabled?).and_call_original
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
mock_service = instance_double(Captain::ConversationCompletionService)
allow(mock_service).to receive(:perform).and_return({ complete: true, reason: 'Customer question was answered' })
allow(Captain::ConversationCompletionService).to receive(:new).and_return(mock_service)
end
it 'resolves the conversation' do
described_class.perform_now(inbox)
expect(resolvable_pending_conversation.reload.status).to eq('resolved')
end
it 'creates a private note with the reason' do
described_class.perform_now(inbox)
private_note = resolvable_pending_conversation.messages.where(private: true).last
expect(private_note.content).to eq('Auto-resolved: Customer question was answered')
end
it 'creates resolution message with configured content' do
custom_message = 'This is a custom resolution message.'
captain_assistant.update!(config: { 'resolution_message' => custom_message })
inbox.reload
described_class.perform_now(inbox)
public_message = resolvable_pending_conversation.messages.where(private: false).outgoing.last
expect(public_message.content).to eq(custom_message)
end
it 'creates resolution message with default if not configured' do
captain_assistant.update!(config: {})
inbox.reload
described_class.perform_now(inbox)
public_message = resolvable_pending_conversation.messages.where(private: false).outgoing.last
expect(public_message.content).to eq(I18n.t('conversations.activity.auto_resolution_message'))
end
it 'adds the correct activity message after resolution' do
described_class.perform_now(inbox)
expected_content = I18n.t(
'conversations.activity.captain.resolved_with_reason',
user_name: captain_assistant.name,
reason: 'no outstanding questions'
)
expect(Conversations::ActivityMessageJob)
.to have_been_enqueued.with(
resolvable_pending_conversation,
{
account_id: resolvable_pending_conversation.account_id,
inbox_id: resolvable_pending_conversation.inbox_id,
message_type: :activity,
content: expected_content
}
)
end
it 'creates a captain inference resolved reporting event' do
perform_enqueued_jobs do
described_class.perform_now(inbox)
end
inference_event = ReportingEvent.find_by(
conversation_id: resolvable_pending_conversation.id,
name: 'conversation_captain_inference_resolved'
)
expect(inference_event).to be_present
end
end
context 'when LLM evaluation returns incomplete' do
let(:handoff_reason) { 'Assistant asked for order number but customer did not respond' }
before do
allow(inbox.account).to receive(:feature_enabled?).and_call_original
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
mock_service = instance_double(Captain::ConversationCompletionService)
allow(mock_service).to receive(:perform).and_return({ complete: false, reason: handoff_reason })
allow(Captain::ConversationCompletionService).to receive(:new).and_return(mock_service)
end
it 'hands off the conversation to agents (status becomes open)' do
described_class.perform_now(inbox)
expect(resolvable_pending_conversation.reload.status).to eq('open')
end
it 'creates a private note with the reason' do
described_class.perform_now(inbox)
private_note = resolvable_pending_conversation.messages.where(private: true).last
expect(private_note.content).to eq("Auto-handoff: #{handoff_reason}")
end
it 'creates handoff message with configured content' do
handoff_message = 'Connecting you to a human agent...'
captain_assistant.update!(config: { 'handoff_message' => handoff_message })
inbox.reload
allow(inbox.account).to receive(:feature_enabled?).and_call_original
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
described_class.perform_now(inbox)
public_message = resolvable_pending_conversation.messages.where(private: false).outgoing.last
expect(public_message.content).to eq(handoff_message)
expect(public_message.additional_attributes['preserve_waiting_since']).to be_nil
end
it 'preserves existing waiting_since when handoff message is configured' do
handoff_message = 'Connecting you to a human agent...'
original_waiting_since = 3.hours.ago
captain_assistant.update!(config: { 'handoff_message' => handoff_message })
resolvable_pending_conversation.update!(waiting_since: original_waiting_since)
allow(MessageTemplates::Template::OutOfOffice).to receive(:perform_if_applicable)
inbox.reload
allow(inbox.account).to receive(:feature_enabled?).and_call_original
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
described_class.perform_now(inbox)
expect(resolvable_pending_conversation.reload.waiting_since).to be_within(1.second).of(original_waiting_since)
end
it 'does not create handoff message if not configured' do
captain_assistant.update!(config: {})
inbox.reload
allow(inbox.account).to receive(:feature_enabled?).and_call_original
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
expect do
described_class.perform_now(inbox)
end.not_to(change { resolvable_pending_conversation.messages.where(private: false).count })
end
it 'adds the correct activity message after handoff' do
described_class.perform_now(inbox)
expected_content = I18n.t(
'conversations.activity.captain.open_with_reason',
user_name: captain_assistant.name,
reason: 'pending clarification from customer'
)
expect(Conversations::ActivityMessageJob)
.to have_been_enqueued.with(
resolvable_pending_conversation,
{
account_id: resolvable_pending_conversation.account_id,
inbox_id: resolvable_pending_conversation.inbox_id,
message_type: :activity,
content: expected_content
}
)
end
it 'creates a captain inference handoff reporting event' do
perform_enqueued_jobs do
described_class.perform_now(inbox)
end
inference_event = ReportingEvent.find_by(
conversation_id: resolvable_pending_conversation.id,
name: 'conversation_captain_inference_handoff'
)
expect(inference_event).to be_present
end
end
context 'when handoff occurs outside business hours' do
let(:handoff_reason) { 'Customer has not responded to clarifying question' }
before do
allow(inbox.account).to receive(:feature_enabled?).and_call_original
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
mock_service = instance_double(Captain::ConversationCompletionService)
allow(mock_service).to receive(:perform).and_return({ complete: false, reason: handoff_reason })
allow(Captain::ConversationCompletionService).to receive(:new).and_return(mock_service)
inbox.update!(working_hours_enabled: true, out_of_office_message: 'We are currently unavailable.')
end
it 'sends OOO message for non-campaign conversations' do
travel_to '01.11.2020 13:00'.to_datetime do
resolvable_pending_conversation.update!(last_activity_at: 2.hours.ago)
described_class.perform_now(inbox)
ooo_message = resolvable_pending_conversation.messages.template.last
expect(ooo_message).to be_present
expect(ooo_message.content).to eq('We are currently unavailable.')
end
end
it 'does not send OOO message for campaign conversations' do
campaign = create(:campaign, account: inbox.account, inbox: inbox)
resolvable_pending_conversation.update!(campaign: campaign)
travel_to '01.11.2020 13:00'.to_datetime do
resolvable_pending_conversation.update!(last_activity_at: 2.hours.ago)
described_class.perform_now(inbox)
expect(resolvable_pending_conversation.messages.template).to be_empty
end
end
it 'does not send OOO message during business hours' do
travel_to '26.10.2020 10:00'.to_datetime do
resolvable_pending_conversation.update!(last_activity_at: 2.hours.ago)
described_class.perform_now(inbox)
expect(resolvable_pending_conversation.messages.template).to be_empty
end
end
end
context 'when LLM evaluation fails' do
before do
allow(inbox.account).to receive(:feature_enabled?).and_call_original
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
mock_service = instance_double(Captain::ConversationCompletionService)
allow(mock_service).to receive(:perform).and_return({ complete: false, reason: 'API Error' })
allow(Captain::ConversationCompletionService).to receive(:new).and_return(mock_service)
end
it 'hands off as safe default' do
described_class.perform_now(inbox)
expect(resolvable_pending_conversation.reload.status).to eq('open')
end
end
it 'does not resolve conversations when auto-resolve is disabled at execution time' do

163
spec/enterprise/lib/captain/conversation_completion_service_spec.rb Normal file
View File

@@ -0,0 +1,163 @@
require 'rails_helper'
RSpec.describe Captain::ConversationCompletionService do
let(:account) { create(:account) }
let(:inbox) { create(:inbox, account: account) }
let(:conversation) { create(:conversation, account: account, inbox: inbox) }
let(:service) { described_class.new(account: account, conversation_display_id: conversation.display_id) }
let(:mock_chat) { instance_double(RubyLLM::Chat) }
let(:mock_context) { instance_double(RubyLLM::Context, chat: mock_chat) }
before do
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
allow(Llm::Config).to receive(:with_api_key).and_yield(mock_context)
allow(mock_chat).to receive(:with_instructions)
allow(mock_chat).to receive(:with_schema).and_return(mock_chat)
allow(account).to receive(:feature_enabled?).and_call_original
allow(account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
end
describe '#perform' do
context 'when conversation is complete' do
let(:mock_response) do
instance_double(
RubyLLM::Message,
content: { 'complete' => true, 'reason' => 'Customer question was fully answered' },
input_tokens: 100,
output_tokens: 20
)
end
before do
create(:message, conversation: conversation, message_type: :incoming, content: 'What are your hours?')
create(:message, conversation: conversation, message_type: :outgoing, content: 'We are open 9-5 Monday to Friday.')
create(:message, conversation: conversation, message_type: :incoming, content: 'Thanks!')
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
it 'returns complete: true with reason' do
result = service.perform
expect(result[:complete]).to be true
expect(result[:reason]).to eq('Customer question was fully answered')
end
end
context 'when conversation is incomplete' do
let(:mock_response) do
instance_double(
RubyLLM::Message,
content: { 'complete' => false, 'reason' => 'Assistant asked for order number but customer did not respond' },
input_tokens: 100,
output_tokens: 20
)
end
before do
create(:message, conversation: conversation, message_type: :incoming, content: 'Where is my order?')
create(:message, conversation: conversation, message_type: :outgoing, content: 'Can you please share your order number?')
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
it 'returns complete: false with reason' do
result = service.perform
expect(result[:complete]).to be false
expect(result[:reason]).to eq('Assistant asked for order number but customer did not respond')
end
end
context 'when conversation has no messages' do
it 'returns incomplete with appropriate reason' do
result = service.perform
expect(result[:complete]).to be false
expect(result[:reason]).to eq('No messages found')
end
end
context 'when LLM returns non-hash response' do
let(:mock_response) do
instance_double(
RubyLLM::Message,
content: 'unexpected string response',
input_tokens: 100,
output_tokens: 20
)
end
before do
create(:message, conversation: conversation, message_type: :incoming, content: 'Hello')
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
it 'returns incomplete as safe default' do
result = service.perform
expect(result[:complete]).to be false
expect(result[:reason]).to eq('Invalid response format')
end
end
context 'when API call fails' do
before do
create(:message, conversation: conversation, message_type: :incoming, content: 'Hello')
allow(mock_chat).to receive(:ask).and_raise(StandardError.new('API Error'))
end
it 'returns incomplete with error message' do
result = service.perform
expect(result[:complete]).to be false
expect(result[:reason]).to eq('API Error')
end
end
context 'when captain_tasks feature is disabled' do
before do
allow(account).to receive(:feature_enabled?).with('captain_tasks').and_return(false)
create(:message, conversation: conversation, message_type: :incoming, content: 'Hello')
end
it 'does not evaluate the conversation as complete' do
result = service.perform
expect(result[:complete]).not_to be true
end
end
context 'when customer quota is exhausted' do
let(:mock_response) do
instance_double(
RubyLLM::Message,
content: { 'complete' => true, 'reason' => 'Customer question was fully answered' },
input_tokens: 100,
output_tokens: 20
)
end
before do
allow(ChatwootApp).to receive(:chatwoot_cloud?).and_return(true)
allow(account).to receive(:usage_limits).and_return({
captain: { responses: { current_available: 0 } }
})
create(:message, conversation: conversation, message_type: :incoming, content: 'What are your hours?')
create(:message, conversation: conversation, message_type: :outgoing, content: 'We are open 9-5 Monday to Friday.')
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
it 'still runs the evaluation bypassing quota check' do
result = service.perform
expect(result[:error]).to be_nil
expect(result[:complete]).to be true
expect(result[:reason]).to eq('Customer question was fully answered')
end
it 'does not increment usage' do
expect(account).not_to receive(:increment_response_usage)
service.perform
end
end
end
end

14
spec/enterprise/lib/captain/tools/handoff_tool_spec.rb
View File

@@ -63,6 +63,20 @@ RSpec.describe Captain::Tools::HandoffTool, type: :model do
tool.perform(tool_context, reason: 'Test reason')
end
it 'creates a conversation_bot_handoff reporting event' do
create(:captain_inbox, captain_assistant: assistant, inbox: inbox)
Current.executed_by = assistant
perform_enqueued_jobs do
tool.perform(tool_context, reason: 'Customer needs specialized support')
end
reporting_event = ReportingEvent.find_by(conversation_id: conversation.id, name: 'conversation_bot_handoff')
expect(reporting_event).to be_present
ensure
Current.reset
end
it 'logs tool usage with reason' do
reason = 'Customer needs help'
expect(tool).to receive(:log_tool_usage).with(

10
spec/enterprise/lib/captain/tools/resolve_conversation_tool_spec.rb
View File

@@ -29,10 +29,14 @@ RSpec.describe Captain::Tools::ResolveConversationTool do
)
end
it 'clears captain_resolve_reason after execution' do
tool.perform(tool_context, reason: 'Possible spam')
it 'creates a conversation_resolved reporting event' do
create(:captain_inbox, captain_assistant: assistant, inbox: inbox)
expect(Current.captain_resolve_reason).to be_nil
expect do
perform_enqueued_jobs do
tool.perform(tool_context, reason: 'Possible spam')
end
end.to change { ReportingEvent.where(conversation_id: conversation.id, name: 'conversation_resolved').count }.by(1)
end
end

3
spec/enterprise/listeners/captain_listener_spec.rb
View File

@@ -8,8 +8,7 @@ describe CaptainListener do
let(:assistant) { create(:captain_assistant, account: account, config: { feature_memory: true, feature_faq: true }) }
describe '#conversation_resolved' do
let(:agent) { create(:user, account: account) }
let(:conversation) { create(:conversation, account: account, inbox: inbox, assignee: agent) }
let(:conversation) { create(:conversation, account: account, inbox: inbox, assignee: user) }
let(:event_name) { :conversation_resolved }
let(:event) { Events::Base.new(event_name, Time.zone.now, conversation: conversation) }