This PR adds LLM instrumentation on langfuse for ai-editor feature ## Type of change New feature (non-breaking change which adds functionality) Needs langfuse account and env vars to be set ## How Has This Been Tested? I configured personal langfuse credentials and instrumented the app, traces can be seen in langfuse. each conversation is one session. <img width="1683" height="714" alt="image" src="https://github.com/user-attachments/assets/3fcba1c9-63cf-44b9-a355-fd6608691559" /> <img width="1446" height="172" alt="image" src="https://github.com/user-attachments/assets/dfa6e98f-4741-4e04-9a9e-078d1f01e97b" /> ## Checklist: - [x ] My code follows the style guidelines of this project - [ x] I have performed a self-review of my code - [ x] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ x] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules --------- Co-authored-by: aakashb95 <aakash@chatwoot.com> Co-authored-by: Vishnu Narayanan <iamwishnu@gmail.com> Co-authored-by: Pranav <pranav@chatwoot.com>
232 lines
9.2 KiB
Ruby
232 lines
9.2 KiB
Ruby
require 'rails_helper'
|
|
|
|
RSpec.describe Integrations::Openai::ProcessorService do
|
|
subject(:service) { described_class.new(hook: hook, event: event) }
|
|
|
|
let(:account) { create(:account) }
|
|
let(:hook) { create(:integrations_hook, :openai, account: account) }
|
|
let(:expected_headers) { { 'Authorization' => "Bearer #{hook.settings['api_key']}" } }
|
|
let(:openai_response) do
|
|
{
|
|
'choices' => [{ 'message' => { 'content' => 'This is a reply from openai.' } }]
|
|
}.to_json
|
|
end
|
|
let(:openai_response_with_usage) do
|
|
{
|
|
'choices' => [{ 'message' => { 'content' => 'This is a reply from openai.' } }],
|
|
'usage' => {
|
|
'prompt_tokens' => 50,
|
|
'completion_tokens' => 20,
|
|
'total_tokens' => 70
|
|
}
|
|
}.to_json
|
|
end
|
|
|
|
describe '#perform' do
|
|
shared_examples 'text transformation operation' do |event_name, system_prompt|
|
|
let(:event) { { 'name' => event_name, 'data' => { 'content' => 'This is a test' } } }
|
|
let(:expected_request_body) do
|
|
{
|
|
'model' => 'gpt-4o-mini',
|
|
'messages' => [
|
|
{ 'role' => 'system', 'content' => system_prompt },
|
|
{ 'role' => 'user', 'content' => 'This is a test' }
|
|
]
|
|
}.to_json
|
|
end
|
|
|
|
it "returns the #{event_name.tr('_', ' ')} text" do
|
|
stub_request(:post, 'https://api.openai.com/v1/chat/completions')
|
|
.with(body: expected_request_body, headers: expected_headers)
|
|
.to_return(status: 200, body: openai_response)
|
|
|
|
result = service.perform
|
|
expect(result[:message]).to eq('This is a reply from openai.')
|
|
end
|
|
end
|
|
|
|
shared_examples 'successful openai response' do
|
|
it 'returns the expected message' do
|
|
result = service.perform
|
|
expect(result[:message]).to eq('This is a reply from openai.')
|
|
end
|
|
end
|
|
|
|
describe 'text transformation operations' do
|
|
base_prompt = 'You are a helpful support agent. '
|
|
language_suffix = 'Ensure that the reply should be in user language.'
|
|
|
|
it_behaves_like 'text transformation operation', 'rephrase',
|
|
"#{base_prompt}Please rephrase the following response. #{language_suffix}"
|
|
it_behaves_like 'text transformation operation', 'fix_spelling_grammar',
|
|
"#{base_prompt}Please fix the spelling and grammar of the following response. #{language_suffix}"
|
|
it_behaves_like 'text transformation operation', 'shorten',
|
|
"#{base_prompt}Please shorten the following response. #{language_suffix}"
|
|
it_behaves_like 'text transformation operation', 'expand',
|
|
"#{base_prompt}Please expand the following response. #{language_suffix}"
|
|
it_behaves_like 'text transformation operation', 'make_friendly',
|
|
"#{base_prompt}Please make the following response more friendly. #{language_suffix}"
|
|
it_behaves_like 'text transformation operation', 'make_formal',
|
|
"#{base_prompt}Please make the following response more formal. #{language_suffix}"
|
|
it_behaves_like 'text transformation operation', 'simplify',
|
|
"#{base_prompt}Please simplify the following response. #{language_suffix}"
|
|
end
|
|
|
|
describe 'conversation-based operations' do
|
|
let!(:conversation) { create(:conversation, account: account) }
|
|
let!(:customer_message) do
|
|
create(:message, account: account, conversation: conversation, message_type: :incoming, content: 'hello agent')
|
|
end
|
|
let!(:agent_message) do
|
|
create(:message, account: account, conversation: conversation, message_type: :outgoing, content: 'hello customer')
|
|
end
|
|
|
|
context 'with reply_suggestion event' do
|
|
let(:event) { { 'name' => 'reply_suggestion', 'data' => { 'conversation_display_id' => conversation.display_id } } }
|
|
let(:expected_request_body) do
|
|
{
|
|
'model' => 'gpt-4o-mini',
|
|
'messages' => [
|
|
{ role: 'system', content: Rails.root.join('lib/integrations/openai/openai_prompts/reply.txt').read },
|
|
{ role: 'user', content: customer_message.content },
|
|
{ role: 'assistant', content: agent_message.content }
|
|
]
|
|
}.to_json
|
|
end
|
|
|
|
before do
|
|
stub_request(:post, 'https://api.openai.com/v1/chat/completions')
|
|
.with(body: expected_request_body, headers: expected_headers)
|
|
.to_return(status: 200, body: openai_response)
|
|
end
|
|
|
|
it_behaves_like 'successful openai response'
|
|
end
|
|
|
|
context 'with summarize event' do
|
|
let(:event) { { 'name' => 'summarize', 'data' => { 'conversation_display_id' => conversation.display_id } } }
|
|
let(:conversation_messages) do
|
|
"Customer #{customer_message.sender.name} : #{customer_message.content}\n" \
|
|
"Agent #{agent_message.sender.name} : #{agent_message.content}\n"
|
|
end
|
|
let(:summary_prompt) do
|
|
if ChatwootApp.enterprise?
|
|
Rails.root.join('enterprise/lib/enterprise/integrations/openai_prompts/summary.txt').read
|
|
else
|
|
'Please summarize the key points from the following conversation between support agents and customer as bullet points ' \
|
|
"for the next support agent looking into the conversation. Reply in the user's language."
|
|
end
|
|
end
|
|
let(:expected_request_body) do
|
|
{
|
|
'model' => 'gpt-4o-mini',
|
|
'messages' => [
|
|
{ 'role' => 'system', 'content' => summary_prompt },
|
|
{ 'role' => 'user', 'content' => conversation_messages }
|
|
]
|
|
}.to_json
|
|
end
|
|
|
|
before do
|
|
stub_request(:post, 'https://api.openai.com/v1/chat/completions')
|
|
.with(body: expected_request_body, headers: expected_headers)
|
|
.to_return(status: 200, body: openai_response)
|
|
end
|
|
|
|
it_behaves_like 'successful openai response'
|
|
end
|
|
|
|
context 'with label_suggestion event and no labels' do
|
|
let(:event) { { 'name' => 'label_suggestion', 'data' => { 'conversation_display_id' => conversation.display_id } } }
|
|
|
|
it 'returns nil' do
|
|
expect(service.perform).to be_nil
|
|
end
|
|
end
|
|
end
|
|
|
|
describe 'edge cases' do
|
|
context 'with unknown event name' do
|
|
let(:event) { { 'name' => 'unknown', 'data' => {} } }
|
|
|
|
it 'returns nil' do
|
|
expect(service.perform).to be_nil
|
|
end
|
|
end
|
|
end
|
|
|
|
describe 'response structure' do
|
|
let(:event) { { 'name' => 'rephrase', 'data' => { 'content' => 'test message' } } }
|
|
|
|
context 'when response includes usage data' do
|
|
before do
|
|
stub_request(:post, 'https://api.openai.com/v1/chat/completions')
|
|
.with(body: anything, headers: expected_headers)
|
|
.to_return(status: 200, body: openai_response_with_usage)
|
|
end
|
|
|
|
it 'returns message, usage, and request_messages' do
|
|
result = service.perform
|
|
|
|
expect(result[:message]).to eq('This is a reply from openai.')
|
|
expect(result[:usage]).to eq({
|
|
'prompt_tokens' => 50,
|
|
'completion_tokens' => 20,
|
|
'total_tokens' => 70
|
|
})
|
|
expect(result[:request_messages]).to be_an(Array)
|
|
expect(result[:request_messages].length).to eq(2)
|
|
end
|
|
end
|
|
|
|
context 'when response does not include usage data' do
|
|
before do
|
|
stub_request(:post, 'https://api.openai.com/v1/chat/completions')
|
|
.with(body: anything, headers: expected_headers)
|
|
.to_return(status: 200, body: openai_response)
|
|
end
|
|
|
|
it 'returns message and request_messages with nil usage' do
|
|
result = service.perform
|
|
|
|
expect(result[:message]).to eq('This is a reply from openai.')
|
|
expect(result[:usage]).to be_nil
|
|
expect(result[:request_messages]).to be_an(Array)
|
|
end
|
|
end
|
|
end
|
|
|
|
describe 'endpoint configuration' do
|
|
let(:event) { { 'name' => 'rephrase', 'data' => { 'content' => 'test message' } } }
|
|
|
|
shared_examples 'endpoint request' do |endpoint_url|
|
|
it "makes request to #{endpoint_url}" do
|
|
stub_request(:post, "#{endpoint_url}/v1/chat/completions")
|
|
.with(body: anything, headers: expected_headers)
|
|
.to_return(status: 200, body: openai_response)
|
|
|
|
result = service.perform
|
|
expect(result[:message]).to eq('This is a reply from openai.')
|
|
expect(result[:request_messages]).to be_an(Array)
|
|
expect(result[:usage]).to be_nil
|
|
end
|
|
end
|
|
|
|
context 'without CAPTAIN_OPEN_AI_ENDPOINT configured' do
|
|
before { InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_ENDPOINT')&.destroy }
|
|
|
|
it_behaves_like 'endpoint request', 'https://api.openai.com'
|
|
end
|
|
|
|
context 'with CAPTAIN_OPEN_AI_ENDPOINT configured' do
|
|
before do
|
|
InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_ENDPOINT')&.destroy
|
|
create(:installation_config, name: 'CAPTAIN_OPEN_AI_ENDPOINT', value: 'https://custom.azure.com/')
|
|
end
|
|
|
|
it_behaves_like 'endpoint request', 'https://custom.azure.com'
|
|
end
|
|
end
|
|
end
|
|
end
|