feat: captain decides if conversation should be resolved or kept open (#13336)
# Pull Request Template ## Description captain decides if conversation should be resolved or open Fixes https://linear.app/chatwoot/issue/AI-91/make-captain-resolution-time-configurable Update: Added 2 entries in reporting events: `conversation_captain_handoff` and `conversation_captain_resolved` ## Type of change Please delete options that are not relevant. - [x] New feature (non-breaking change which adds functionality) - [x] This change requires a documentation update ## How Has This Been Tested? Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration. LLM call decides that conversation is resolved, drops a private note <img width="1228" height="438" alt="image" src="https://github.com/user-attachments/assets/fb2cf1e9-4b2b-458b-a1e2-45c53d6a0158" /> LLM call decides conversation is still open as query was not resolved <img width="1215" height="573" alt="image" src="https://github.com/user-attachments/assets/2d1d5322-f567-487e-954e-11ab0798d11c" /> ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] Any dependent changes have been merged and published in downstream modules --------- Co-authored-by: Sojan Jose <sojan@pepalo.com>
This commit is contained in:
Aakash Bakhle
@@ -2,16 +2,14 @@ require 'rails_helper'
|
||||
|
||||
RSpec.describe Captain::InboxPendingConversationsResolutionJob, type: :job do
|
||||
let!(:inbox) { create(:inbox) }
|
||||
|
||||
let!(:resolvable_pending_conversation) { create(:conversation, inbox: inbox, last_activity_at: 2.hours.ago, status: :pending) }
|
||||
let!(:recent_pending_conversation) { create(:conversation, inbox: inbox, last_activity_at: 10.minutes.ago, status: :pending) }
|
||||
let!(:recent_pending_conversation) { create(:conversation, inbox: inbox, last_activity_at: 1.minute.ago, status: :pending) }
|
||||
let!(:open_conversation) { create(:conversation, inbox: inbox, last_activity_at: 1.hour.ago, status: :open) }
|
||||
|
||||
let!(:captain_assistant) { create(:captain_assistant, account: inbox.account) }
|
||||
|
||||
before do
|
||||
create(:captain_inbox, inbox: inbox, captain_assistant: captain_assistant)
|
||||
stub_const('Limits::BULK_ACTIONS_LIMIT', 2)
|
||||
stub_const('Limits::BULK_ACTIONS_LIMIT', 3)
|
||||
inbox.reload
|
||||
end
|
||||
|
||||
@@ -20,49 +18,307 @@ RSpec.describe Captain::InboxPendingConversationsResolutionJob, type: :job do
|
||||
.to have_enqueued_job.on_queue('low')
|
||||
end
|
||||
|
||||
it 'resolves only the eligible pending conversations' do
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expect(resolvable_pending_conversation.reload.status).to eq('resolved')
|
||||
expect(recent_pending_conversation.reload.status).to eq('pending')
|
||||
expect(open_conversation.reload.status).to eq('open')
|
||||
end
|
||||
|
||||
it 'creates exactly one outgoing message with configured content' do
|
||||
custom_message = 'This is a custom resolution message.'
|
||||
captain_assistant.update!(config: { 'resolution_message' => custom_message })
|
||||
|
||||
expect do
|
||||
context 'when captain_tasks is disabled' do
|
||||
it 'resolves pending conversations inactive for over 1 hour' do
|
||||
described_class.perform_now(inbox)
|
||||
end.to change { resolvable_pending_conversation.messages.outgoing.reload.count }.by(1)
|
||||
|
||||
outgoing_message = resolvable_pending_conversation.messages.outgoing.last
|
||||
expect(outgoing_message.content).to eq(custom_message)
|
||||
expect(resolvable_pending_conversation.reload.status).to eq('resolved')
|
||||
end
|
||||
|
||||
it 'does not resolve recent pending conversations' do
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expect(recent_pending_conversation.reload.status).to eq('pending')
|
||||
end
|
||||
|
||||
it 'does not affect open conversations' do
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expect(open_conversation.reload.status).to eq('open')
|
||||
end
|
||||
|
||||
it 'does not call ConversationCompletionService' do
|
||||
allow(Captain::ConversationCompletionService).to receive(:new)
|
||||
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expect(Captain::ConversationCompletionService).not_to have_received(:new)
|
||||
end
|
||||
end
|
||||
|
||||
it 'creates an outgoing message with default auto resolution message if not configured' do
|
||||
captain_assistant.update!(config: {})
|
||||
context 'when captain_tasks is enabled' do
|
||||
before do
|
||||
allow(inbox.account).to receive(:feature_enabled?).and_call_original
|
||||
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
|
||||
end
|
||||
|
||||
described_class.perform_now(inbox)
|
||||
outgoing_message = resolvable_pending_conversation.messages.outgoing.last
|
||||
expect(outgoing_message.content).to eq(
|
||||
I18n.t('conversations.activity.auto_resolution_message')
|
||||
)
|
||||
end
|
||||
it 'only evaluates eligible pending conversations (inactive > 1 hour)' do
|
||||
allow(Captain::ConversationCompletionService).to receive(:new).and_call_original
|
||||
|
||||
it 'adds the correct activity message after resolution by Captain' do
|
||||
described_class.perform_now(inbox)
|
||||
expected_content = I18n.t('conversations.activity.captain.resolved', user_name: captain_assistant.name)
|
||||
expect(Conversations::ActivityMessageJob)
|
||||
.to have_been_enqueued.with(
|
||||
resolvable_pending_conversation,
|
||||
{
|
||||
account_id: resolvable_pending_conversation.account_id,
|
||||
inbox_id: resolvable_pending_conversation.inbox_id,
|
||||
message_type: :activity,
|
||||
content: expected_content
|
||||
}
|
||||
# Mock the service to return complete for all conversations
|
||||
mock_service = instance_double(Captain::ConversationCompletionService)
|
||||
allow(mock_service).to receive(:perform).and_return({ complete: true, reason: 'Test' })
|
||||
allow(Captain::ConversationCompletionService).to receive(:new).and_return(mock_service)
|
||||
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
# Only resolvable conversation should be evaluated (not recent or open)
|
||||
expect(Captain::ConversationCompletionService).to have_received(:new).with(
|
||||
account: inbox.account,
|
||||
conversation_display_id: resolvable_pending_conversation.display_id
|
||||
)
|
||||
expect(recent_pending_conversation.reload.status).to eq('pending')
|
||||
expect(open_conversation.reload.status).to eq('open')
|
||||
end
|
||||
|
||||
it 'skips auto-action if conversation receives new activity after evaluation' do
|
||||
mock_service = instance_double(Captain::ConversationCompletionService)
|
||||
allow(mock_service).to receive(:perform) do
|
||||
resolvable_pending_conversation.update!(last_activity_at: Time.current)
|
||||
{ complete: true, reason: 'Customer question was answered' }
|
||||
end
|
||||
allow(Captain::ConversationCompletionService).to receive(:new).and_return(mock_service)
|
||||
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expect(resolvable_pending_conversation.reload.status).to eq('pending')
|
||||
expect(resolvable_pending_conversation.messages.outgoing).to be_empty
|
||||
end
|
||||
end
|
||||
|
||||
context 'when LLM evaluation returns complete' do
|
||||
before do
|
||||
allow(inbox.account).to receive(:feature_enabled?).and_call_original
|
||||
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
|
||||
mock_service = instance_double(Captain::ConversationCompletionService)
|
||||
allow(mock_service).to receive(:perform).and_return({ complete: true, reason: 'Customer question was answered' })
|
||||
allow(Captain::ConversationCompletionService).to receive(:new).and_return(mock_service)
|
||||
end
|
||||
|
||||
it 'resolves the conversation' do
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expect(resolvable_pending_conversation.reload.status).to eq('resolved')
|
||||
end
|
||||
|
||||
it 'creates a private note with the reason' do
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
private_note = resolvable_pending_conversation.messages.where(private: true).last
|
||||
expect(private_note.content).to eq('Auto-resolved: Customer question was answered')
|
||||
end
|
||||
|
||||
it 'creates resolution message with configured content' do
|
||||
custom_message = 'This is a custom resolution message.'
|
||||
captain_assistant.update!(config: { 'resolution_message' => custom_message })
|
||||
inbox.reload
|
||||
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
public_message = resolvable_pending_conversation.messages.where(private: false).outgoing.last
|
||||
expect(public_message.content).to eq(custom_message)
|
||||
end
|
||||
|
||||
it 'creates resolution message with default if not configured' do
|
||||
captain_assistant.update!(config: {})
|
||||
inbox.reload
|
||||
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
public_message = resolvable_pending_conversation.messages.where(private: false).outgoing.last
|
||||
expect(public_message.content).to eq(I18n.t('conversations.activity.auto_resolution_message'))
|
||||
end
|
||||
|
||||
it 'adds the correct activity message after resolution' do
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expected_content = I18n.t(
|
||||
'conversations.activity.captain.resolved_with_reason',
|
||||
user_name: captain_assistant.name,
|
||||
reason: 'no outstanding questions'
|
||||
)
|
||||
expect(Conversations::ActivityMessageJob)
|
||||
.to have_been_enqueued.with(
|
||||
resolvable_pending_conversation,
|
||||
{
|
||||
account_id: resolvable_pending_conversation.account_id,
|
||||
inbox_id: resolvable_pending_conversation.inbox_id,
|
||||
message_type: :activity,
|
||||
content: expected_content
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
it 'creates a captain inference resolved reporting event' do
|
||||
perform_enqueued_jobs do
|
||||
described_class.perform_now(inbox)
|
||||
end
|
||||
|
||||
inference_event = ReportingEvent.find_by(
|
||||
conversation_id: resolvable_pending_conversation.id,
|
||||
name: 'conversation_captain_inference_resolved'
|
||||
)
|
||||
expect(inference_event).to be_present
|
||||
end
|
||||
end
|
||||
|
||||
context 'when LLM evaluation returns incomplete' do
|
||||
let(:handoff_reason) { 'Assistant asked for order number but customer did not respond' }
|
||||
|
||||
before do
|
||||
allow(inbox.account).to receive(:feature_enabled?).and_call_original
|
||||
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
|
||||
mock_service = instance_double(Captain::ConversationCompletionService)
|
||||
allow(mock_service).to receive(:perform).and_return({ complete: false, reason: handoff_reason })
|
||||
allow(Captain::ConversationCompletionService).to receive(:new).and_return(mock_service)
|
||||
end
|
||||
|
||||
it 'hands off the conversation to agents (status becomes open)' do
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expect(resolvable_pending_conversation.reload.status).to eq('open')
|
||||
end
|
||||
|
||||
it 'creates a private note with the reason' do
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
private_note = resolvable_pending_conversation.messages.where(private: true).last
|
||||
expect(private_note.content).to eq("Auto-handoff: #{handoff_reason}")
|
||||
end
|
||||
|
||||
it 'creates handoff message with configured content' do
|
||||
handoff_message = 'Connecting you to a human agent...'
|
||||
captain_assistant.update!(config: { 'handoff_message' => handoff_message })
|
||||
inbox.reload
|
||||
allow(inbox.account).to receive(:feature_enabled?).and_call_original
|
||||
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
|
||||
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
public_message = resolvable_pending_conversation.messages.where(private: false).outgoing.last
|
||||
expect(public_message.content).to eq(handoff_message)
|
||||
expect(public_message.additional_attributes['preserve_waiting_since']).to be_nil
|
||||
end
|
||||
|
||||
it 'preserves existing waiting_since when handoff message is configured' do
|
||||
handoff_message = 'Connecting you to a human agent...'
|
||||
original_waiting_since = 3.hours.ago
|
||||
|
||||
captain_assistant.update!(config: { 'handoff_message' => handoff_message })
|
||||
resolvable_pending_conversation.update!(waiting_since: original_waiting_since)
|
||||
allow(MessageTemplates::Template::OutOfOffice).to receive(:perform_if_applicable)
|
||||
inbox.reload
|
||||
allow(inbox.account).to receive(:feature_enabled?).and_call_original
|
||||
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
|
||||
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expect(resolvable_pending_conversation.reload.waiting_since).to be_within(1.second).of(original_waiting_since)
|
||||
end
|
||||
|
||||
it 'does not create handoff message if not configured' do
|
||||
captain_assistant.update!(config: {})
|
||||
inbox.reload
|
||||
allow(inbox.account).to receive(:feature_enabled?).and_call_original
|
||||
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
|
||||
|
||||
expect do
|
||||
described_class.perform_now(inbox)
|
||||
end.not_to(change { resolvable_pending_conversation.messages.where(private: false).count })
|
||||
end
|
||||
|
||||
it 'adds the correct activity message after handoff' do
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expected_content = I18n.t(
|
||||
'conversations.activity.captain.open_with_reason',
|
||||
user_name: captain_assistant.name,
|
||||
reason: 'pending clarification from customer'
|
||||
)
|
||||
expect(Conversations::ActivityMessageJob)
|
||||
.to have_been_enqueued.with(
|
||||
resolvable_pending_conversation,
|
||||
{
|
||||
account_id: resolvable_pending_conversation.account_id,
|
||||
inbox_id: resolvable_pending_conversation.inbox_id,
|
||||
message_type: :activity,
|
||||
content: expected_content
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
it 'creates a captain inference handoff reporting event' do
|
||||
perform_enqueued_jobs do
|
||||
described_class.perform_now(inbox)
|
||||
end
|
||||
|
||||
inference_event = ReportingEvent.find_by(
|
||||
conversation_id: resolvable_pending_conversation.id,
|
||||
name: 'conversation_captain_inference_handoff'
|
||||
)
|
||||
expect(inference_event).to be_present
|
||||
end
|
||||
end
|
||||
|
||||
context 'when handoff occurs outside business hours' do
|
||||
let(:handoff_reason) { 'Customer has not responded to clarifying question' }
|
||||
|
||||
before do
|
||||
allow(inbox.account).to receive(:feature_enabled?).and_call_original
|
||||
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
|
||||
mock_service = instance_double(Captain::ConversationCompletionService)
|
||||
allow(mock_service).to receive(:perform).and_return({ complete: false, reason: handoff_reason })
|
||||
allow(Captain::ConversationCompletionService).to receive(:new).and_return(mock_service)
|
||||
inbox.update!(working_hours_enabled: true, out_of_office_message: 'We are currently unavailable.')
|
||||
end
|
||||
|
||||
it 'sends OOO message for non-campaign conversations' do
|
||||
travel_to '01.11.2020 13:00'.to_datetime do
|
||||
resolvable_pending_conversation.update!(last_activity_at: 2.hours.ago)
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
ooo_message = resolvable_pending_conversation.messages.template.last
|
||||
expect(ooo_message).to be_present
|
||||
expect(ooo_message.content).to eq('We are currently unavailable.')
|
||||
end
|
||||
end
|
||||
|
||||
it 'does not send OOO message for campaign conversations' do
|
||||
campaign = create(:campaign, account: inbox.account, inbox: inbox)
|
||||
resolvable_pending_conversation.update!(campaign: campaign)
|
||||
|
||||
travel_to '01.11.2020 13:00'.to_datetime do
|
||||
resolvable_pending_conversation.update!(last_activity_at: 2.hours.ago)
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expect(resolvable_pending_conversation.messages.template).to be_empty
|
||||
end
|
||||
end
|
||||
|
||||
it 'does not send OOO message during business hours' do
|
||||
travel_to '26.10.2020 10:00'.to_datetime do
|
||||
resolvable_pending_conversation.update!(last_activity_at: 2.hours.ago)
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expect(resolvable_pending_conversation.messages.template).to be_empty
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'when LLM evaluation fails' do
|
||||
before do
|
||||
allow(inbox.account).to receive(:feature_enabled?).and_call_original
|
||||
allow(inbox.account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
|
||||
mock_service = instance_double(Captain::ConversationCompletionService)
|
||||
allow(mock_service).to receive(:perform).and_return({ complete: false, reason: 'API Error' })
|
||||
allow(Captain::ConversationCompletionService).to receive(:new).and_return(mock_service)
|
||||
end
|
||||
|
||||
it 'hands off as safe default' do
|
||||
described_class.perform_now(inbox)
|
||||
|
||||
expect(resolvable_pending_conversation.reload.status).to eq('open')
|
||||
end
|
||||
end
|
||||
|
||||
it 'does not resolve conversations when auto-resolve is disabled at execution time' do
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Captain::ConversationCompletionService do
|
||||
let(:account) { create(:account) }
|
||||
let(:inbox) { create(:inbox, account: account) }
|
||||
let(:conversation) { create(:conversation, account: account, inbox: inbox) }
|
||||
let(:service) { described_class.new(account: account, conversation_display_id: conversation.display_id) }
|
||||
let(:mock_chat) { instance_double(RubyLLM::Chat) }
|
||||
let(:mock_context) { instance_double(RubyLLM::Context, chat: mock_chat) }
|
||||
|
||||
before do
|
||||
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
|
||||
allow(Llm::Config).to receive(:with_api_key).and_yield(mock_context)
|
||||
allow(mock_chat).to receive(:with_instructions)
|
||||
allow(mock_chat).to receive(:with_schema).and_return(mock_chat)
|
||||
allow(account).to receive(:feature_enabled?).and_call_original
|
||||
allow(account).to receive(:feature_enabled?).with('captain_tasks').and_return(true)
|
||||
end
|
||||
|
||||
describe '#perform' do
|
||||
context 'when conversation is complete' do
|
||||
let(:mock_response) do
|
||||
instance_double(
|
||||
RubyLLM::Message,
|
||||
content: { 'complete' => true, 'reason' => 'Customer question was fully answered' },
|
||||
input_tokens: 100,
|
||||
output_tokens: 20
|
||||
)
|
||||
end
|
||||
|
||||
before do
|
||||
create(:message, conversation: conversation, message_type: :incoming, content: 'What are your hours?')
|
||||
create(:message, conversation: conversation, message_type: :outgoing, content: 'We are open 9-5 Monday to Friday.')
|
||||
create(:message, conversation: conversation, message_type: :incoming, content: 'Thanks!')
|
||||
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
||||
end
|
||||
|
||||
it 'returns complete: true with reason' do
|
||||
result = service.perform
|
||||
|
||||
expect(result[:complete]).to be true
|
||||
expect(result[:reason]).to eq('Customer question was fully answered')
|
||||
end
|
||||
end
|
||||
|
||||
context 'when conversation is incomplete' do
|
||||
let(:mock_response) do
|
||||
instance_double(
|
||||
RubyLLM::Message,
|
||||
content: { 'complete' => false, 'reason' => 'Assistant asked for order number but customer did not respond' },
|
||||
input_tokens: 100,
|
||||
output_tokens: 20
|
||||
)
|
||||
end
|
||||
|
||||
before do
|
||||
create(:message, conversation: conversation, message_type: :incoming, content: 'Where is my order?')
|
||||
create(:message, conversation: conversation, message_type: :outgoing, content: 'Can you please share your order number?')
|
||||
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
||||
end
|
||||
|
||||
it 'returns complete: false with reason' do
|
||||
result = service.perform
|
||||
|
||||
expect(result[:complete]).to be false
|
||||
expect(result[:reason]).to eq('Assistant asked for order number but customer did not respond')
|
||||
end
|
||||
end
|
||||
|
||||
context 'when conversation has no messages' do
|
||||
it 'returns incomplete with appropriate reason' do
|
||||
result = service.perform
|
||||
|
||||
expect(result[:complete]).to be false
|
||||
expect(result[:reason]).to eq('No messages found')
|
||||
end
|
||||
end
|
||||
|
||||
context 'when LLM returns non-hash response' do
|
||||
let(:mock_response) do
|
||||
instance_double(
|
||||
RubyLLM::Message,
|
||||
content: 'unexpected string response',
|
||||
input_tokens: 100,
|
||||
output_tokens: 20
|
||||
)
|
||||
end
|
||||
|
||||
before do
|
||||
create(:message, conversation: conversation, message_type: :incoming, content: 'Hello')
|
||||
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
||||
end
|
||||
|
||||
it 'returns incomplete as safe default' do
|
||||
result = service.perform
|
||||
|
||||
expect(result[:complete]).to be false
|
||||
expect(result[:reason]).to eq('Invalid response format')
|
||||
end
|
||||
end
|
||||
|
||||
context 'when API call fails' do
|
||||
before do
|
||||
create(:message, conversation: conversation, message_type: :incoming, content: 'Hello')
|
||||
allow(mock_chat).to receive(:ask).and_raise(StandardError.new('API Error'))
|
||||
end
|
||||
|
||||
it 'returns incomplete with error message' do
|
||||
result = service.perform
|
||||
|
||||
expect(result[:complete]).to be false
|
||||
expect(result[:reason]).to eq('API Error')
|
||||
end
|
||||
end
|
||||
|
||||
context 'when captain_tasks feature is disabled' do
|
||||
before do
|
||||
allow(account).to receive(:feature_enabled?).with('captain_tasks').and_return(false)
|
||||
create(:message, conversation: conversation, message_type: :incoming, content: 'Hello')
|
||||
end
|
||||
|
||||
it 'does not evaluate the conversation as complete' do
|
||||
result = service.perform
|
||||
|
||||
expect(result[:complete]).not_to be true
|
||||
end
|
||||
end
|
||||
|
||||
context 'when customer quota is exhausted' do
|
||||
let(:mock_response) do
|
||||
instance_double(
|
||||
RubyLLM::Message,
|
||||
content: { 'complete' => true, 'reason' => 'Customer question was fully answered' },
|
||||
input_tokens: 100,
|
||||
output_tokens: 20
|
||||
)
|
||||
end
|
||||
|
||||
before do
|
||||
allow(ChatwootApp).to receive(:chatwoot_cloud?).and_return(true)
|
||||
allow(account).to receive(:usage_limits).and_return({
|
||||
captain: { responses: { current_available: 0 } }
|
||||
})
|
||||
create(:message, conversation: conversation, message_type: :incoming, content: 'What are your hours?')
|
||||
create(:message, conversation: conversation, message_type: :outgoing, content: 'We are open 9-5 Monday to Friday.')
|
||||
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
||||
end
|
||||
|
||||
it 'still runs the evaluation bypassing quota check' do
|
||||
result = service.perform
|
||||
|
||||
expect(result[:error]).to be_nil
|
||||
expect(result[:complete]).to be true
|
||||
expect(result[:reason]).to eq('Customer question was fully answered')
|
||||
end
|
||||
|
||||
it 'does not increment usage' do
|
||||
expect(account).not_to receive(:increment_response_usage)
|
||||
service.perform
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -63,6 +63,20 @@ RSpec.describe Captain::Tools::HandoffTool, type: :model do
|
||||
tool.perform(tool_context, reason: 'Test reason')
|
||||
end
|
||||
|
||||
it 'creates a conversation_bot_handoff reporting event' do
|
||||
create(:captain_inbox, captain_assistant: assistant, inbox: inbox)
|
||||
Current.executed_by = assistant
|
||||
|
||||
perform_enqueued_jobs do
|
||||
tool.perform(tool_context, reason: 'Customer needs specialized support')
|
||||
end
|
||||
|
||||
reporting_event = ReportingEvent.find_by(conversation_id: conversation.id, name: 'conversation_bot_handoff')
|
||||
expect(reporting_event).to be_present
|
||||
ensure
|
||||
Current.reset
|
||||
end
|
||||
|
||||
it 'logs tool usage with reason' do
|
||||
reason = 'Customer needs help'
|
||||
expect(tool).to receive(:log_tool_usage).with(
|
||||
|
||||
@@ -29,10 +29,14 @@ RSpec.describe Captain::Tools::ResolveConversationTool do
|
||||
)
|
||||
end
|
||||
|
||||
it 'clears captain_resolve_reason after execution' do
|
||||
tool.perform(tool_context, reason: 'Possible spam')
|
||||
it 'creates a conversation_resolved reporting event' do
|
||||
create(:captain_inbox, captain_assistant: assistant, inbox: inbox)
|
||||
|
||||
expect(Current.captain_resolve_reason).to be_nil
|
||||
expect do
|
||||
perform_enqueued_jobs do
|
||||
tool.perform(tool_context, reason: 'Possible spam')
|
||||
end
|
||||
end.to change { ReportingEvent.where(conversation_id: conversation.id, name: 'conversation_resolved').count }.by(1)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@@ -8,8 +8,7 @@ describe CaptainListener do
|
||||
let(:assistant) { create(:captain_assistant, account: account, config: { feature_memory: true, feature_faq: true }) }
|
||||
|
||||
describe '#conversation_resolved' do
|
||||
let(:agent) { create(:user, account: account) }
|
||||
let(:conversation) { create(:conversation, account: account, inbox: inbox, assignee: agent) }
|
||||
let(:conversation) { create(:conversation, account: account, inbox: inbox, assignee: user) }
|
||||
|
||||
let(:event_name) { :conversation_resolved }
|
||||
let(:event) { Events::Base.new(event_name, Time.zone.now, conversation: conversation) }
|
||||
|
||||
@@ -27,12 +27,24 @@ describe ReportingEventListener do
|
||||
end
|
||||
|
||||
it 'creates conversation_resolved event with business hour value' do
|
||||
event = Events::Base.new('conversation.resolved', Time.zone.now, conversation: new_conversation)
|
||||
event = Events::Base.new('conversation.resolved', updated_at, conversation: new_conversation)
|
||||
listener.conversation_resolved(event)
|
||||
expect(account.reporting_events.where(name: 'conversation_resolved')[0]['value_in_business_hours']).to be 144_000.0
|
||||
end
|
||||
end
|
||||
|
||||
it 'uses event timestamp even when conversation updated_at changes later' do
|
||||
resolved_at = conversation.created_at + 20.minutes
|
||||
allow(conversation).to receive(:updated_at).and_return(resolved_at + 10.minutes)
|
||||
event = Events::Base.new('conversation.resolved', resolved_at, conversation: conversation)
|
||||
|
||||
listener.conversation_resolved(event)
|
||||
|
||||
reporting_event = account.reporting_events.where(name: 'conversation_resolved').first
|
||||
expect(reporting_event.value).to eq 1200
|
||||
expect(reporting_event.event_end_time).to be_within(1.second).of(resolved_at)
|
||||
end
|
||||
|
||||
describe 'conversation_bot_resolved' do
|
||||
# create an agent bot
|
||||
let!(:agent_bot_inbox) { create(:inbox, account: account) }
|
||||
@@ -261,11 +273,55 @@ describe ReportingEventListener do
|
||||
end
|
||||
|
||||
it 'creates conversation_bot_handoff event with business hour value' do
|
||||
event = Events::Base.new('conversation.bot_handoff', Time.zone.now, conversation: new_conversation)
|
||||
event = Events::Base.new('conversation.bot_handoff', updated_at, conversation: new_conversation)
|
||||
listener.conversation_bot_handoff(event)
|
||||
expect(account.reporting_events.where(name: 'conversation_bot_handoff')[0]['value_in_business_hours']).to be 144_000.0
|
||||
end
|
||||
end
|
||||
|
||||
it 'uses event timestamp even when conversation updated_at changes later' do
|
||||
handoff_at = conversation.created_at + 10.minutes
|
||||
allow(conversation).to receive(:updated_at).and_return(handoff_at + 15.minutes)
|
||||
event = Events::Base.new('conversation.bot_handoff', handoff_at, conversation: conversation)
|
||||
|
||||
listener.conversation_bot_handoff(event)
|
||||
|
||||
reporting_event = account.reporting_events.where(name: 'conversation_bot_handoff').first
|
||||
expect(reporting_event.value).to eq 600
|
||||
expect(reporting_event.event_end_time).to be_within(1.second).of(handoff_at)
|
||||
end
|
||||
end
|
||||
|
||||
describe '#conversation_captain_inference_resolved' do
|
||||
it 'creates conversation_captain_inference_resolved event' do
|
||||
expect(account.reporting_events.where(name: 'conversation_captain_inference_resolved').count).to be 0
|
||||
decision_time = conversation.created_at + 60.seconds
|
||||
event = Events::Base.new('conversation.captain_inference_resolved', decision_time, conversation: conversation)
|
||||
allow(conversation).to receive(:updated_at).and_return(decision_time + 5.minutes)
|
||||
|
||||
listener.conversation_captain_inference_resolved(event)
|
||||
|
||||
reporting_event = account.reporting_events.where(name: 'conversation_captain_inference_resolved').first
|
||||
expect(reporting_event).to be_present
|
||||
expect(reporting_event.value).to eq 60
|
||||
expect(reporting_event.event_end_time).to be_within(1.second).of(decision_time)
|
||||
end
|
||||
end
|
||||
|
||||
describe '#conversation_captain_inference_handoff' do
|
||||
it 'creates conversation_captain_inference_handoff event' do
|
||||
expect(account.reporting_events.where(name: 'conversation_captain_inference_handoff').count).to be 0
|
||||
decision_time = conversation.created_at + 90.seconds
|
||||
event = Events::Base.new('conversation.captain_inference_handoff', decision_time, conversation: conversation)
|
||||
allow(conversation).to receive(:updated_at).and_return(decision_time + 5.minutes)
|
||||
|
||||
listener.conversation_captain_inference_handoff(event)
|
||||
|
||||
reporting_event = account.reporting_events.where(name: 'conversation_captain_inference_handoff').first
|
||||
expect(reporting_event).to be_present
|
||||
expect(reporting_event.value).to eq 90
|
||||
expect(reporting_event.event_end_time).to be_within(1.second).of(decision_time)
|
||||
end
|
||||
end
|
||||
|
||||
describe '#conversation_opened' do
|
||||
@@ -274,7 +330,8 @@ describe ReportingEventListener do
|
||||
|
||||
it 'creates conversation_opened event with value 0' do
|
||||
expect(account.reporting_events.where(name: 'conversation_opened').count).to be 0
|
||||
event = Events::Base.new('conversation.opened', Time.zone.now, conversation: new_conversation)
|
||||
opened_at = Time.zone.now
|
||||
event = Events::Base.new('conversation.opened', opened_at, conversation: new_conversation)
|
||||
listener.conversation_opened(event)
|
||||
expect(account.reporting_events.where(name: 'conversation_opened').count).to be 1
|
||||
|
||||
@@ -282,7 +339,7 @@ describe ReportingEventListener do
|
||||
expect(opened_event.value).to eq 0
|
||||
expect(opened_event.value_in_business_hours).to eq 0
|
||||
expect(opened_event.event_start_time).to be_within(1.second).of(new_conversation.created_at)
|
||||
expect(opened_event.event_end_time).to be_within(1.second).of(new_conversation.updated_at)
|
||||
expect(opened_event.event_end_time).to be_within(1.second).of(opened_at)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -334,6 +391,17 @@ describe ReportingEventListener do
|
||||
expect(reopened_event.user_id).to eq(user.id)
|
||||
end
|
||||
|
||||
it 'uses event timestamp even when conversation updated_at changes later' do
|
||||
allow(reopened_conversation).to receive(:updated_at).and_return(reopened_time + 20.minutes)
|
||||
event = Events::Base.new('conversation.opened', reopened_time, conversation: reopened_conversation)
|
||||
|
||||
listener.conversation_opened(event)
|
||||
|
||||
reopened_event = account.reporting_events.where(name: 'conversation_opened').first
|
||||
expect(reopened_event.value).to be_within(1).of(3600)
|
||||
expect(reopened_event.event_end_time).to be_within(1.second).of(reopened_time)
|
||||
end
|
||||
|
||||
context 'when business hours enabled for inbox' do
|
||||
let(:resolved_time) { Time.zone.parse('March 20, 2022 12:00') }
|
||||
let(:reopened_time) { Time.zone.parse('March 21, 2022 14:00') }
|
||||
@@ -406,6 +474,45 @@ describe ReportingEventListener do
|
||||
end
|
||||
end
|
||||
|
||||
context 'when latest resolved event is after the conversation opened event timestamp' do
|
||||
let(:previous_resolved_time) { Time.zone.parse('March 22, 2022 09:00') }
|
||||
let(:future_resolved_time) { Time.zone.parse('March 22, 2022 11:00') }
|
||||
let(:reopened_time) { Time.zone.parse('March 22, 2022 10:00') }
|
||||
let(:reopened_conversation) do
|
||||
create(:conversation, account: account, inbox: inbox, assignee: user, updated_at: reopened_time)
|
||||
end
|
||||
|
||||
before do
|
||||
create(:reporting_event,
|
||||
name: 'conversation_resolved',
|
||||
account_id: account.id,
|
||||
inbox_id: inbox.id,
|
||||
conversation_id: reopened_conversation.id,
|
||||
user_id: user.id,
|
||||
event_start_time: reopened_conversation.created_at,
|
||||
event_end_time: previous_resolved_time)
|
||||
|
||||
create(:reporting_event,
|
||||
name: 'conversation_resolved',
|
||||
account_id: account.id,
|
||||
inbox_id: inbox.id,
|
||||
conversation_id: reopened_conversation.id,
|
||||
user_id: user.id,
|
||||
event_start_time: previous_resolved_time,
|
||||
event_end_time: future_resolved_time)
|
||||
end
|
||||
|
||||
it 'ignores future resolved events when computing reopen duration' do
|
||||
event = Events::Base.new('conversation.opened', reopened_time, conversation: reopened_conversation)
|
||||
listener.conversation_opened(event)
|
||||
|
||||
reopened_event = account.reporting_events.where(name: 'conversation_opened').first
|
||||
expect(reopened_event.value).to be_within(1).of(3600)
|
||||
expect(reopened_event.event_start_time).to be_within(1.second).of(previous_resolved_time)
|
||||
expect(reopened_event.event_end_time).to be_within(1.second).of(reopened_time)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when agent bot resolves and conversation is reopened' do
|
||||
# This implicitly tests that the first_response time is correctly calculated
|
||||
# By checking that a conversation reopened event is created with the correct values
|
||||
|
||||
@@ -366,6 +366,25 @@ RSpec.describe Message do
|
||||
expect(conversation.waiting_since).to be_nil
|
||||
end
|
||||
end
|
||||
|
||||
context 'when bot response should preserve waiting_since' do
|
||||
let(:agent_bot) { create(:agent_bot, account: conversation.account) }
|
||||
|
||||
it 'does not clear waiting_since when preserve_waiting_since is set' do
|
||||
original_waiting_since = 45.minutes.ago
|
||||
conversation.update!(waiting_since: original_waiting_since)
|
||||
|
||||
create(
|
||||
:message,
|
||||
conversation: conversation,
|
||||
message_type: :outgoing,
|
||||
sender: agent_bot,
|
||||
preserve_waiting_since: true
|
||||
)
|
||||
|
||||
expect(conversation.reload.waiting_since).to be_within(1.second).of(original_waiting_since)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'with webhook_data' do
|
||||
|
||||
Reference in New Issue
Block a user