From c7d259d5fd383e6f7c7ab66dde46247cb5de598a Mon Sep 17 00:00:00 2001 From: Pranav Date: Mon, 3 Feb 2025 02:55:08 -0800 Subject: [PATCH] chore: Update the behavior of Captain resolutions (#10794) This PR ensures that only conversations from quick conversation channels are resolved, avoiding resolutions on the email channel (we still need to improve the UX here). It also updates the FAQ generation logic, limiting it to conversations that had at least one human interaction. --- .../conversations_resolution_scheduler_job.rb | 8 ++- .../captain/llm/conversation_faq_service.rb | 8 +++ ...ersations_resolution_scheduler_job_spec.rb | 49 ++++++++++++------- .../llm/conversation_faq_service_spec.rb | 10 +++- spec/factories/inboxes.rb | 5 ++ 5 files changed, 61 insertions(+), 19 deletions(-) diff --git a/enterprise/app/jobs/enterprise/account/conversations_resolution_scheduler_job.rb b/enterprise/app/jobs/enterprise/account/conversations_resolution_scheduler_job.rb index 2dc6fd67b..599dee96a 100644 --- a/enterprise/app/jobs/enterprise/account/conversations_resolution_scheduler_job.rb +++ b/enterprise/app/jobs/enterprise/account/conversations_resolution_scheduler_job.rb @@ -9,7 +9,13 @@ module Enterprise::Account::ConversationsResolutionSchedulerJob def resolve_captain_conversations CaptainInbox.all.find_each(batch_size: 100) do |captain_inbox| - Captain::InboxPendingConversationsResolutionJob.perform_later(captain_inbox.inbox) + inbox = captain_inbox.inbox + + next if inbox.email? + + Captain::InboxPendingConversationsResolutionJob.perform_later( + inbox + ) end end end diff --git a/enterprise/app/services/captain/llm/conversation_faq_service.rb b/enterprise/app/services/captain/llm/conversation_faq_service.rb index 47cdd6cf4..ea33493f2 100644 --- a/enterprise/app/services/captain/llm/conversation_faq_service.rb +++ b/enterprise/app/services/captain/llm/conversation_faq_service.rb @@ -8,7 +8,11 @@ class Captain::Llm::ConversationFaqService < Captain::Llm::BaseOpenAiService @content = conversation.to_llm_text end + # Generates and deduplicates FAQs from conversation content + # Skips processing if there was no human interaction def generate_and_deduplicate + return [] if no_human_interaction? + new_faqs = generate return [] if new_faqs.empty? @@ -21,6 +25,10 @@ class Captain::Llm::ConversationFaqService < Captain::Llm::BaseOpenAiService attr_reader :content, :conversation, :assistant + def no_human_interaction? + conversation.first_reply_created_at.nil? + end + def find_and_separate_duplicates(faqs) duplicate_faqs = [] unique_faqs = [] diff --git a/spec/enterprise/jobs/enterprise/account/conversations_resolution_scheduler_job_spec.rb b/spec/enterprise/jobs/enterprise/account/conversations_resolution_scheduler_job_spec.rb index 03c799962..b67877412 100644 --- a/spec/enterprise/jobs/enterprise/account/conversations_resolution_scheduler_job_spec.rb +++ b/spec/enterprise/jobs/enterprise/account/conversations_resolution_scheduler_job_spec.rb @@ -1,29 +1,44 @@ require 'rails_helper' RSpec.describe Account::ConversationsResolutionSchedulerJob, type: :job do - let!(:account_with_bot) { create(:account) } let(:account) { create(:account) } - let(:assistant) { create(:captain_assistant, account: account_with_bot) } - - let!(:account_without_bot) { create(:account) } - let!(:inbox_with_bot) { create(:inbox, account: account_with_bot) } - let!(:inbox_without_bot) { create(:inbox, account: account_without_bot) } + let(:assistant) { create(:captain_assistant, account: account) } describe '#perform - captain resolutions' do - before do - create(:captain_inbox, captain_assistant: assistant, inbox: inbox_with_bot) + context 'when handling different inbox types' do + let!(:regular_inbox) { create(:inbox, account: account) } + let!(:email_inbox) { create(:inbox, :with_email, account: account) } + + before do + create(:captain_inbox, captain_assistant: assistant, inbox: regular_inbox) + create(:captain_inbox, captain_assistant: assistant, inbox: email_inbox) + end + + it 'enqueues resolution jobs only for non-email inboxes with captain enabled' do + expect do + described_class.perform_now + end.to have_enqueued_job(Captain::InboxPendingConversationsResolutionJob) + .with(regular_inbox) + .exactly(:once) + end + + it 'does not enqueue resolution jobs for email inboxes even with captain enabled' do + expect do + described_class.perform_now + end.not_to have_enqueued_job(Captain::InboxPendingConversationsResolutionJob) + .with(email_inbox) + end end - it 'enqueues resolution jobs only for inboxes with captain enabled' do - expect do - described_class.perform_now - end.to have_enqueued_job(Captain::InboxPendingConversationsResolutionJob).with(inbox_with_bot).and have_enqueued_job.exactly(:once) - end + context 'when inbox has no captain enabled' do + let!(:inbox_without_captain) { create(:inbox, account: create(:account)) } - it 'does not enqueue resolution jobs for inboxes without captain enabled' do - expect do - described_class.perform_now - end.not_to have_enqueued_job(Captain::InboxPendingConversationsResolutionJob).with(inbox_without_bot) + it 'does not enqueue resolution jobs' do + expect do + described_class.perform_now + end.not_to have_enqueued_job(Captain::InboxPendingConversationsResolutionJob) + .with(inbox_without_captain) + end end end end diff --git a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb index 6c729e428..fe0276b49 100644 --- a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb +++ b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb @@ -2,7 +2,7 @@ require 'rails_helper' RSpec.describe Captain::Llm::ConversationFaqService do let(:captain_assistant) { create(:captain_assistant) } - let(:conversation) { create(:conversation) } + let(:conversation) { create(:conversation, first_reply_created_at: Time.zone.now) } let(:service) { described_class.new(captain_assistant, conversation) } let(:client) { instance_double(OpenAI::Client) } let(:embedding_service) { instance_double(Captain::Llm::EmbeddingService) } @@ -57,6 +57,14 @@ RSpec.describe Captain::Llm::ConversationFaqService do end end + context 'without human interaction' do + let(:conversation) { create(:conversation) } + + it 'returns an empty array without generating FAQs' do + expect(service.generate_and_deduplicate).to eq([]) + end + end + context 'when finding duplicates' do let(:existing_response) do create(:captain_assistant_response, assistant: captain_assistant, question: 'Similar question', answer: 'Similar answer') diff --git a/spec/factories/inboxes.rb b/spec/factories/inboxes.rb index cdd23f3c3..e8ca50200 100644 --- a/spec/factories/inboxes.rb +++ b/spec/factories/inboxes.rb @@ -9,5 +9,10 @@ FactoryBot.define do after(:create) do |inbox| inbox.channel.save! end + + trait :with_email do + channel { FactoryBot.build(:channel_email, account: account) } + name { 'Email Inbox' } + end end end