LeadMagnet/leadchat
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat: captain decides if conversation should be resolved or kept open (#13336)

Browse Source 
# Pull Request Template

## Description

captain decides if conversation should be resolved or open

Fixes
https://linear.app/chatwoot/issue/AI-91/make-captain-resolution-time-configurable

Update: Added 2 entries in reporting events:
`conversation_captain_handoff` and `conversation_captain_resolved`

## Type of change

Please delete options that are not relevant.

- [x] New feature (non-breaking change which adds functionality)
- [x] This change requires a documentation update

## How Has This Been Tested?

Please describe the tests that you ran to verify your changes. Provide
instructions so we can reproduce. Please also list any relevant details
for your test configuration.

LLM call decides that conversation is resolved, drops a private note
<img width="1228" height="438" alt="image"
src="https://github.com/user-attachments/assets/fb2cf1e9-4b2b-458b-a1e2-45c53d6a0158"
/>

LLM call decides conversation is still open as query was not resolved
<img width="1215" height="573" alt="image"
src="https://github.com/user-attachments/assets/2d1d5322-f567-487e-954e-11ab0798d11c"
/>


## Checklist:

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my code
- [x] I have commented on my code, particularly in hard-to-understand
areas
- [] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [x] Any dependent changes have been merged and published in downstream
modules

---------

Co-authored-by: Sojan Jose <sojan@pepalo.com>
This commit is contained in:
Aakash Bakhle
2026-03-13 10:03:58 +05:30
committed by GitHub
parent 199dcd382e
commit d6d38cdd7d
22 changed files with 949 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
spec/enterprise/lib/captain/conversation_completion_service_spec.rb Normal file
View File

@@ -0,0 +1,163 @@
require 'rails_helper'
RSpec.describe Captain::ConversationCompletionService do
let(:account) { create(:account) }
let(:inbox) { create(:inbox, account: account) }
let(:conversation) { create(:conversation, account: account, inbox: inbox) }
let(:service) { described_class.new(account: account, conversation_display_id: conversation.display_id) }
let(:mock_chat) { instance_double(RubyLLM::Chat) }
let(:mock_context) { instance_double(RubyLLM::Context, chat: mock_chat) }
before do
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
allow(Llm::Config).to receive(:with_api_key).and_yield(mock_context)
allow(mock_chat).to receive(:with_instructions)
allow(mock_chat).to receive(:with_schema).and_return(mock_chat)
allow(account).to receive(:feature_enabled?).and_call_original
allow(account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
end
describe '#perform' do
context 'when conversation is complete' do
let(:mock_response) do
instance_double(
RubyLLM::Message,
content: { 'complete' => true, 'reason' => 'Customer question was fully answered' },
input_tokens: 100,
output_tokens: 20
)
end
before do
create(:message, conversation: conversation, message_type: :incoming, content: 'What are your hours?')
create(:message, conversation: conversation, message_type: :outgoing, content: 'We are open 9-5 Monday to Friday.')
create(:message, conversation: conversation, message_type: :incoming, content: 'Thanks!')
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
it 'returns complete: true with reason' do
result = service.perform
expect(result[:complete]).to be true
expect(result[:reason]).to eq('Customer question was fully answered')
end
end
context 'when conversation is incomplete' do
let(:mock_response) do
instance_double(
RubyLLM::Message,
content: { 'complete' => false, 'reason' => 'Assistant asked for order number but customer did not respond' },
input_tokens: 100,
output_tokens: 20
)
end
before do
create(:message, conversation: conversation, message_type: :incoming, content: 'Where is my order?')
create(:message, conversation: conversation, message_type: :outgoing, content: 'Can you please share your order number?')
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
it 'returns complete: false with reason' do
result = service.perform
expect(result[:complete]).to be false
expect(result[:reason]).to eq('Assistant asked for order number but customer did not respond')
end
end
context 'when conversation has no messages' do
it 'returns incomplete with appropriate reason' do
result = service.perform
expect(result[:complete]).to be false
expect(result[:reason]).to eq('No messages found')
end
end
context 'when LLM returns non-hash response' do
let(:mock_response) do
instance_double(
RubyLLM::Message,
content: 'unexpected string response',
input_tokens: 100,
output_tokens: 20
)
end
before do
create(:message, conversation: conversation, message_type: :incoming, content: 'Hello')
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
it 'returns incomplete as safe default' do
result = service.perform
expect(result[:complete]).to be false
expect(result[:reason]).to eq('Invalid response format')
end
end
context 'when API call fails' do
before do
create(:message, conversation: conversation, message_type: :incoming, content: 'Hello')
allow(mock_chat).to receive(:ask).and_raise(StandardError.new('API Error'))
end
it 'returns incomplete with error message' do
result = service.perform
expect(result[:complete]).to be false
expect(result[:reason]).to eq('API Error')
end
end
context 'when captain_tasks feature is disabled' do
before do
allow(account).to receive(:feature_enabled?).with('captain_tasks').and_return(false)
create(:message, conversation: conversation, message_type: :incoming, content: 'Hello')
end
it 'does not evaluate the conversation as complete' do
result = service.perform
expect(result[:complete]).not_to be true
end
end
context 'when customer quota is exhausted' do
let(:mock_response) do
instance_double(
RubyLLM::Message,
content: { 'complete' => true, 'reason' => 'Customer question was fully answered' },
input_tokens: 100,
output_tokens: 20
)
end
before do
allow(ChatwootApp).to receive(:chatwoot_cloud?).and_return(true)
allow(account).to receive(:usage_limits).and_return({
captain: { responses: { current_available: 0 } }
})
create(:message, conversation: conversation, message_type: :incoming, content: 'What are your hours?')
create(:message, conversation: conversation, message_type: :outgoing, content: 'We are open 9-5 Monday to Friday.')
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
it 'still runs the evaluation bypassing quota check' do
result = service.perform
expect(result[:error]).to be_nil
expect(result[:complete]).to be true
expect(result[:reason]).to eq('Customer question was fully answered')
end
it 'does not increment usage' do
expect(account).not_to receive(:increment_response_usage)
service.perform
end
end
end
end

14
spec/enterprise/lib/captain/tools/handoff_tool_spec.rb
View File

@@ -63,6 +63,20 @@ RSpec.describe Captain::Tools::HandoffTool, type: :model do
tool.perform(tool_context, reason: 'Test reason')
end
it 'creates a conversation_bot_handoff reporting event' do
create(:captain_inbox, captain_assistant: assistant, inbox: inbox)
Current.executed_by = assistant
perform_enqueued_jobs do
tool.perform(tool_context, reason: 'Customer needs specialized support')
end
reporting_event = ReportingEvent.find_by(conversation_id: conversation.id, name: 'conversation_bot_handoff')
expect(reporting_event).to be_present
ensure
Current.reset
end
it 'logs tool usage with reason' do
reason = 'Customer needs help'
expect(tool).to receive(:log_tool_usage).with(

10
spec/enterprise/lib/captain/tools/resolve_conversation_tool_spec.rb
View File

@@ -29,10 +29,14 @@ RSpec.describe Captain::Tools::ResolveConversationTool do
)
end
it 'clears captain_resolve_reason after execution' do
tool.perform(tool_context, reason: 'Possible spam')
it 'creates a conversation_resolved reporting event' do
create(:captain_inbox, captain_assistant: assistant, inbox: inbox)
expect(Current.captain_resolve_reason).to be_nil
expect do
perform_enqueued_jobs do
tool.perform(tool_context, reason: 'Possible spam')
end
end.to change { ReportingEvent.where(conversation_id: conversation.id, name: 'conversation_resolved').count }.by(1)
end
end