chore: Update the prompts to include language of the account for FAQs (#12280)

There were customer reported issues with FAQs which were generated in a
different langauge than what they were expecting. The reason behind this
was that the language of the account was not considered in the prompt
provided. If the language of the content was say Spanish, and the
account locale was english. The output was not predicable. The output
depends on the model and the execution time.

This PR would update the prompt to behave consistently with the account
locale. Even though the content provided is in a different language, it
would generate FAQs in the account locale.

Changes:
- Updated the prompt to include a detailed expectation of the FAQs
quality along with the language
- Added specs for the services where the prompt generator is called.

Tested the prompt using Phoenix playground across GPT 5, GPT 4.1, GPT
4.0. The reasoning setting for GPT 5 needs to be low so that it doesn't
generate random questions like "What was this updated?"
This commit is contained in:
Pranav
2025-08-22 10:03:52 -07:00
committed by GitHub
parent c07529c2b0
commit 7f56cd92f8
5 changed files with 150 additions and 12 deletions

View File

@@ -13,7 +13,7 @@ RSpec.describe Captain::Documents::ResponseBuilderJob, type: :job do
before do
allow(Captain::Llm::FaqGeneratorService).to receive(:new)
.with(document.content)
.with(document.content, document.account.locale_english_name)
.and_return(faq_generator)
allow(faq_generator).to receive(:generate).and_return(faqs)
end
@@ -43,5 +43,26 @@ RSpec.describe Captain::Documents::ResponseBuilderJob, type: :job do
expect(first_response.documentable).to eq(document)
end
end
context 'with different locales' do
let(:spanish_account) { create(:account, locale: 'pt') }
let(:spanish_assistant) { create(:captain_assistant, account: spanish_account) }
let(:spanish_document) { create(:captain_document, assistant: spanish_assistant, account: spanish_account) }
let(:spanish_faq_generator) { instance_double(Captain::Llm::FaqGeneratorService) }
before do
allow(Captain::Llm::FaqGeneratorService).to receive(:new)
.with(spanish_document.content, 'portuguese')
.and_return(spanish_faq_generator)
allow(spanish_faq_generator).to receive(:generate).and_return(faqs)
end
it 'passes the correct locale to FAQ generator' do
described_class.new.perform(spanish_document)
expect(Captain::Llm::FaqGeneratorService).to have_received(:new)
.with(spanish_document.content, 'portuguese')
end
end
end
end

View File

@@ -0,0 +1,87 @@
require 'rails_helper'
RSpec.describe Captain::Llm::FaqGeneratorService do
let(:content) { 'Sample content for FAQ generation' }
let(:language) { 'english' }
let(:service) { described_class.new(content, language) }
let(:client) { instance_double(OpenAI::Client) }
before do
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
allow(OpenAI::Client).to receive(:new).and_return(client)
end
describe '#generate' do
let(:sample_faqs) do
[
{ 'question' => 'What is this service?', 'answer' => 'It generates FAQs.' },
{ 'question' => 'How does it work?', 'answer' => 'Using AI technology.' }
]
end
let(:openai_response) do
{
'choices' => [
{
'message' => {
'content' => { faqs: sample_faqs }.to_json
}
}
]
}
end
context 'when successful' do
before do
allow(client).to receive(:chat).and_return(openai_response)
allow(Captain::Llm::SystemPromptsService).to receive(:faq_generator).and_return('system prompt')
end
it 'returns parsed FAQs' do
result = service.generate
expect(result).to eq(sample_faqs)
end
it 'calls OpenAI client with chat parameters' do
expect(client).to receive(:chat).with(parameters: hash_including(
model: 'gpt-4o-mini',
response_format: { type: 'json_object' },
messages: array_including(
hash_including(role: 'system'),
hash_including(role: 'user', content: content)
)
))
service.generate
end
it 'calls SystemPromptsService with correct language' do
expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with(language)
service.generate
end
end
context 'with different language' do
let(:language) { 'spanish' }
before do
allow(client).to receive(:chat).and_return(openai_response)
end
it 'passes the correct language to SystemPromptsService' do
expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with('spanish')
service.generate
end
end
context 'when OpenAI API fails' do
before do
allow(client).to receive(:chat).and_raise(OpenAI::Error.new('API Error'))
end
it 'handles the error and returns empty array' do
expect(Rails.logger).to receive(:error).with('OpenAI API Error: API Error')
expect(service.generate).to eq([])
end
end
end
end