LeadMagnet/leadchat
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat: legacy features to ruby llm (#12994)

Browse Source
This commit is contained in:
Aakash Bakhle
2025-12-11 14:17:28 +05:30
committed by GitHub
parent f2054e703a
commit 1de8d3e56d
39 changed files with 860 additions and 755 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb
View File

@@ -4,49 +4,42 @@ RSpec.describe Captain::Llm::ConversationFaqService do
let(:captain_assistant) { create(:captain_assistant) }
let(:conversation) { create(:conversation, first_reply_created_at: Time.zone.now) }
let(:service) { described_class.new(captain_assistant, conversation) }
let(:client) { instance_double(OpenAI::Client) }
let(:embedding_service) { instance_double(Captain::Llm::EmbeddingService) }
let(:mock_chat) { instance_double(RubyLLM::Chat) }
let(:sample_faqs) do
[
{ 'question' => 'What is the purpose?', 'answer' => 'To help users.' },
{ 'question' => 'How does it work?', 'answer' => 'Through AI.' }
]
end
let(:mock_response) do
instance_double(RubyLLM::Message, content: { faqs: sample_faqs }.to_json)
end
before do
create(:installation_config) { create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key') }
allow(OpenAI::Client).to receive(:new).and_return(client)
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
allow(Captain::Llm::EmbeddingService).to receive(:new).and_return(embedding_service)
allow(RubyLLM).to receive(:chat).and_return(mock_chat)
allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
allow(mock_chat).to receive(:with_params).and_return(mock_chat)
allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
describe '#generate_and_deduplicate' do
let(:sample_faqs) do
[
{ 'question' => 'What is the purpose?', 'answer' => 'To help users.' },
{ 'question' => 'How does it work?', 'answer' => 'Through AI.' }
]
end
let(:openai_response) do
{
'choices' => [
{
'message' => {
'content' => { faqs: sample_faqs }.to_json
}
}
]
}
end
context 'when successful' do
before do
allow(client).to receive(:chat).and_return(openai_response)
allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3])
allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([])
end
it 'creates new FAQs' do
it 'creates new FAQs for valid conversation content' do
expect do
service.generate_and_deduplicate
end.to change(captain_assistant.responses, :count).by(2)
end
it 'saves the correct FAQ content' do
it 'saves FAQs with pending status linked to conversation' do
service.generate_and_deduplicate
expect(
captain_assistant.responses.pluck(:question, :answer, :status, :documentable_id)
@@ -63,6 +56,11 @@ RSpec.describe Captain::Llm::ConversationFaqService do
it 'returns an empty array without generating FAQs' do
expect(service.generate_and_deduplicate).to eq([])
end
it 'does not call the LLM API' do
expect(RubyLLM).not_to receive(:chat)
service.generate_and_deduplicate
end
end
context 'when finding duplicates' do
@@ -70,9 +68,6 @@ RSpec.describe Captain::Llm::ConversationFaqService do
create(:captain_assistant_response, assistant: captain_assistant, question: 'Similar question', answer: 'Similar answer')
end
let(:similar_neighbor) do
# Using OpenStruct here to mock as the Captain:AssistantResponse does not implement
# neighbor_distance as a method or attribute rather it is returned directly
# from SQL query in neighbor gem
OpenStruct.new(
id: 1,
question: existing_response.question,
@@ -82,87 +77,78 @@ RSpec.describe Captain::Llm::ConversationFaqService do
end
before do
allow(client).to receive(:chat).and_return(openai_response)
allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3])
allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([similar_neighbor])
end
it 'filters out duplicate FAQs' do
it 'filters out duplicate FAQs based on embedding similarity' do
expect do
service.generate_and_deduplicate
end.not_to change(captain_assistant.responses, :count)
end
end
context 'when OpenAI API fails' do
context 'when LLM API fails' do
before do
allow(client).to receive(:chat).and_raise(OpenAI::Error.new('API Error'))
allow(mock_chat).to receive(:ask).and_raise(RubyLLM::Error.new(nil, 'API Error'))
allow(Rails.logger).to receive(:error)
end
it 'handles the error and returns empty array' do
expect(Rails.logger).to receive(:error).with('OpenAI API Error: API Error')
it 'returns empty array and logs the error' do
expect(Rails.logger).to receive(:error).with('LLM API Error: API Error')
expect(service.generate_and_deduplicate).to eq([])
end
end
context 'when JSON parsing fails' do
let(:invalid_response) do
{
'choices' => [
{
'message' => {
'content' => 'invalid json'
}
}
]
}
instance_double(RubyLLM::Message, content: 'invalid json')
end
before do
allow(client).to receive(:chat).and_return(invalid_response)
allow(mock_chat).to receive(:ask).and_return(invalid_response)
end
it 'handles JSON parsing errors' do
it 'handles JSON parsing errors gracefully' do
expect(Rails.logger).to receive(:error).with(/Error in parsing GPT processed response:/)
expect(service.generate_and_deduplicate).to eq([])
end
end
context 'when response content is nil' do
let(:nil_response) do
instance_double(RubyLLM::Message, content: nil)
end
before do
allow(mock_chat).to receive(:ask).and_return(nil_response)
end
it 'returns empty array' do
expect(service.generate_and_deduplicate).to eq([])
end
end
end
describe '#chat_parameters' do
it 'includes correct model and response format' do
params = service.send(:chat_parameters)
expect(params[:model]).to eq('gpt-4o-mini')
expect(params[:response_format]).to eq({ type: 'json_object' })
end
it 'includes system prompt and conversation content' do
allow(Captain::Llm::SystemPromptsService).to receive(:conversation_faq_generator).and_return('system prompt')
params = service.send(:chat_parameters)
expect(params[:messages]).to include(
{ role: 'system', content: 'system prompt' },
{ role: 'user', content: conversation.to_llm_text }
)
end
describe 'language handling' do
context 'when conversation has different language' do
let(:account) { create(:account, locale: 'fr') }
let(:conversation) do
create(:conversation, account: account,
first_reply_created_at: Time.zone.now)
create(:conversation, account: account, first_reply_created_at: Time.zone.now)
end
it 'includes system prompt with correct language' do
allow(Captain::Llm::SystemPromptsService).to receive(:conversation_faq_generator)
before do
allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3])
allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([])
end
it 'uses account language for system prompt' do
expect(Captain::Llm::SystemPromptsService).to receive(:conversation_faq_generator)
.with('french')
.and_return('system prompt in french')
.at_least(:once)
.and_call_original
params = service.send(:chat_parameters)
expect(params[:messages]).to include(
{ role: 'system', content: 'system prompt in french' }
)
service.generate_and_deduplicate
end
end
end

110
spec/enterprise/services/captain/llm/faq_generator_service_spec.rb
View File

@@ -4,58 +4,40 @@ RSpec.describe Captain::Llm::FaqGeneratorService do
let(:content) { 'Sample content for FAQ generation' }
let(:language) { 'english' }
let(:service) { described_class.new(content, language) }
let(:client) { instance_double(OpenAI::Client) }
let(:mock_chat) { instance_double(RubyLLM::Chat) }
let(:sample_faqs) do
[
{ 'question' => 'What is this service?', 'answer' => 'It generates FAQs.' },
{ 'question' => 'How does it work?', 'answer' => 'Using AI technology.' }
]
end
let(:mock_response) do
instance_double(RubyLLM::Message, content: { faqs: sample_faqs }.to_json)
end
before do
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
allow(OpenAI::Client).to receive(:new).and_return(client)
allow(RubyLLM).to receive(:chat).and_return(mock_chat)
allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
allow(mock_chat).to receive(:with_params).and_return(mock_chat)
allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
describe '#generate' do
let(:sample_faqs) do
[
{ 'question' => 'What is this service?', 'answer' => 'It generates FAQs.' },
{ 'question' => 'How does it work?', 'answer' => 'Using AI technology.' }
]
end
let(:openai_response) do
{
'choices' => [
{
'message' => {
'content' => { faqs: sample_faqs }.to_json
}
}
]
}
end
context 'when successful' do
before do
allow(client).to receive(:chat).and_return(openai_response)
allow(Captain::Llm::SystemPromptsService).to receive(:faq_generator).and_return('system prompt')
end
it 'returns parsed FAQs' do
it 'returns parsed FAQs from the LLM response' do
result = service.generate
expect(result).to eq(sample_faqs)
end
it 'calls OpenAI client with chat parameters' do
expect(client).to receive(:chat).with(parameters: hash_including(
model: 'gpt-4o-mini',
response_format: { type: 'json_object' },
messages: array_including(
hash_including(role: 'system'),
hash_including(role: 'user', content: content)
)
))
it 'sends content to LLM with JSON response format' do
expect(mock_chat).to receive(:with_params).with(response_format: { type: 'json_object' }).and_return(mock_chat)
service.generate
end
it 'calls SystemPromptsService with correct language' do
expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with(language)
it 'uses SystemPromptsService with the specified language' do
expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with(language).at_least(:once).and_call_original
service.generate
end
end
@@ -63,23 +45,57 @@ RSpec.describe Captain::Llm::FaqGeneratorService do
context 'with different language' do
let(:language) { 'spanish' }
before do
allow(client).to receive(:chat).and_return(openai_response)
end
it 'passes the correct language to SystemPromptsService' do
expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with('spanish')
expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with('spanish').at_least(:once).and_call_original
service.generate
end
end
context 'when OpenAI API fails' do
context 'when LLM API fails' do
before do
allow(client).to receive(:chat).and_raise(OpenAI::Error.new('API Error'))
allow(mock_chat).to receive(:ask).and_raise(RubyLLM::Error.new(nil, 'API Error'))
allow(Rails.logger).to receive(:error)
end
it 'handles the error and returns empty array' do
expect(Rails.logger).to receive(:error).with('OpenAI API Error: API Error')
it 'returns empty array and logs the error' do
expect(Rails.logger).to receive(:error).with('LLM API Error: API Error')
expect(service.generate).to eq([])
end
end
context 'when response content is nil' do
let(:nil_response) { instance_double(RubyLLM::Message, content: nil) }
before do
allow(mock_chat).to receive(:ask).and_return(nil_response)
end
it 'returns empty array' do
expect(service.generate).to eq([])
end
end
context 'when JSON parsing fails' do
let(:invalid_response) { instance_double(RubyLLM::Message, content: 'invalid json') }
before do
allow(mock_chat).to receive(:ask).and_return(invalid_response)
end
it 'logs error and returns empty array' do
expect(Rails.logger).to receive(:error).with(/Error in parsing GPT processed response:/)
expect(service.generate).to eq([])
end
end
context 'when response is missing faqs key' do
let(:missing_key_response) { instance_double(RubyLLM::Message, content: '{"data": []}') }
before do
allow(mock_chat).to receive(:ask).and_return(missing_key_response)
end
it 'returns empty array via KeyError rescue' do
expect(service.generate).to eq([])
end
end