feat: Add support for image files in Captain (#11730)
# Pull Request Template ## Linear links: - https://linear.app/chatwoot/issue/CW-4479/if-image-is-sent-by-the-customer-send-it-to-openai ## Description This pull request adds “Captain image support” to Chatwoot. It introduces multimodal message handling so that when a customer sends an image, Captain can forward the file to OpenAI’s vision endpoint, generate a caption/analysis ## Type of change Please delete options that are not relevant. - [x] New feature (non-breaking change which adds functionality) ## How Has This Been Tested? <img width="891" alt="image" src="https://github.com/user-attachments/assets/c7cc98ed-cc44-4865-a53a-83d129e2fe2c" /> ## Checklist: - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my code - [ ] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules --------- Co-authored-by: Pranav <pranav@chatwoot.com>
This commit is contained in:
Tanmay Deep Sharma
committed by
GitHub
GitHub
parent
257cd07ee6
commit
811eb66615
@@ -0,0 +1,309 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Captain::OpenAiMessageBuilderService do
|
||||
subject(:service) { described_class.new(message: message) }
|
||||
|
||||
let(:message) { create(:message, content: 'Hello world') }
|
||||
|
||||
describe '#generate_content' do
|
||||
context 'when message has only text content' do
|
||||
it 'returns the text content directly' do
|
||||
expect(service.generate_content).to eq('Hello world')
|
||||
end
|
||||
end
|
||||
|
||||
context 'when message has no content and no attachments' do
|
||||
let(:message) { create(:message, content: nil) }
|
||||
|
||||
it 'returns default message' do
|
||||
expect(service.generate_content).to eq('Message without content')
|
||||
end
|
||||
end
|
||||
|
||||
context 'when message has text content and attachments' do
|
||||
before do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :image, external_url: 'https://example.com/image.jpg')
|
||||
attachment.save!
|
||||
end
|
||||
|
||||
it 'returns an array of content parts' do
|
||||
result = service.generate_content
|
||||
expect(result).to be_an(Array)
|
||||
expect(result).to include({ type: 'text', text: 'Hello world' })
|
||||
expect(result).to include({ type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } })
|
||||
end
|
||||
end
|
||||
|
||||
context 'when message has only non-text attachments' do
|
||||
let(:message) { create(:message, content: nil) }
|
||||
|
||||
before do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :image, external_url: 'https://example.com/image.jpg')
|
||||
attachment.save!
|
||||
end
|
||||
|
||||
it 'returns an array of content parts without text' do
|
||||
result = service.generate_content
|
||||
expect(result).to be_an(Array)
|
||||
expect(result).to include({ type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } })
|
||||
expect(result).not_to include(hash_including(type: 'text', text: 'Hello world'))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#attachment_parts' do
|
||||
let(:message) { create(:message, content: nil) }
|
||||
let(:attachments) { message.attachments }
|
||||
|
||||
context 'with image attachments' do
|
||||
before do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :image, external_url: 'https://example.com/image.jpg')
|
||||
attachment.save!
|
||||
end
|
||||
|
||||
it 'includes image parts' do
|
||||
result = service.send(:attachment_parts, attachments)
|
||||
expect(result).to include({ type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } })
|
||||
end
|
||||
end
|
||||
|
||||
context 'with audio attachments' do
|
||||
let(:audio_attachment) do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :audio)
|
||||
attachment.save!
|
||||
attachment
|
||||
end
|
||||
|
||||
before do
|
||||
allow(Messages::AudioTranscriptionService).to receive(:new).with(audio_attachment).and_return(
|
||||
instance_double(Messages::AudioTranscriptionService, perform: { success: true, transcriptions: 'Audio transcription text' })
|
||||
)
|
||||
end
|
||||
|
||||
it 'includes transcription text part' do
|
||||
audio_attachment # trigger creation
|
||||
result = service.send(:attachment_parts, attachments)
|
||||
expect(result).to include({ type: 'text', text: 'Audio transcription text' })
|
||||
end
|
||||
end
|
||||
|
||||
context 'with other file types' do
|
||||
before do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :file)
|
||||
attachment.save!
|
||||
end
|
||||
|
||||
it 'includes generic attachment message' do
|
||||
result = service.send(:attachment_parts, attachments)
|
||||
expect(result).to include({ type: 'text', text: 'User has shared an attachment' })
|
||||
end
|
||||
end
|
||||
|
||||
context 'with mixed attachment types' do
|
||||
let(:image_attachment) do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :image, external_url: 'https://example.com/image.jpg')
|
||||
attachment.save!
|
||||
attachment
|
||||
end
|
||||
|
||||
let(:audio_attachment) do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :audio)
|
||||
attachment.save!
|
||||
attachment
|
||||
end
|
||||
|
||||
let(:document_attachment) do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :file)
|
||||
attachment.save!
|
||||
attachment
|
||||
end
|
||||
|
||||
before do
|
||||
allow(Messages::AudioTranscriptionService).to receive(:new).with(audio_attachment).and_return(
|
||||
instance_double(Messages::AudioTranscriptionService, perform: { success: true, transcriptions: 'Audio text' })
|
||||
)
|
||||
end
|
||||
|
||||
it 'includes all relevant parts' do
|
||||
image_attachment # trigger creation
|
||||
audio_attachment # trigger creation
|
||||
document_attachment # trigger creation
|
||||
|
||||
result = service.send(:attachment_parts, attachments)
|
||||
expect(result).to include({ type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } })
|
||||
expect(result).to include({ type: 'text', text: 'Audio text' })
|
||||
expect(result).to include({ type: 'text', text: 'User has shared an attachment' })
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#image_parts' do
|
||||
let(:message) { create(:message, content: nil) }
|
||||
|
||||
context 'with valid image attachments' do
|
||||
let(:image1) do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :image, external_url: 'https://example.com/image1.jpg')
|
||||
attachment.save!
|
||||
attachment
|
||||
end
|
||||
|
||||
let(:image2) do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :image, external_url: 'https://example.com/image2.jpg')
|
||||
attachment.save!
|
||||
attachment
|
||||
end
|
||||
|
||||
it 'returns image parts for all valid images' do
|
||||
image1 # trigger creation
|
||||
image2 # trigger creation
|
||||
|
||||
image_attachments = message.attachments.where(file_type: :image)
|
||||
result = service.send(:image_parts, image_attachments)
|
||||
|
||||
expect(result).to include({ type: 'image_url', image_url: { url: 'https://example.com/image1.jpg' } })
|
||||
expect(result).to include({ type: 'image_url', image_url: { url: 'https://example.com/image2.jpg' } })
|
||||
end
|
||||
end
|
||||
|
||||
context 'with image attachments without URLs' do
|
||||
let(:image_attachment) do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :image, external_url: nil)
|
||||
attachment.save!
|
||||
attachment
|
||||
end
|
||||
|
||||
before do
|
||||
allow(image_attachment).to receive(:file).and_return(instance_double(ActiveStorage::Attached::One, attached?: false))
|
||||
end
|
||||
|
||||
it 'skips images without valid URLs' do
|
||||
image_attachment # trigger creation
|
||||
|
||||
image_attachments = message.attachments.where(file_type: :image)
|
||||
result = service.send(:image_parts, image_attachments)
|
||||
|
||||
expect(result).to be_empty
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#get_attachment_url' do
|
||||
let(:attachment) do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :image)
|
||||
attachment.save!
|
||||
attachment
|
||||
end
|
||||
|
||||
context 'when attachment has external_url' do
|
||||
before { attachment.update(external_url: 'https://example.com/image.jpg') }
|
||||
|
||||
it 'returns external_url' do
|
||||
expect(service.send(:get_attachment_url, attachment)).to eq('https://example.com/image.jpg')
|
||||
end
|
||||
end
|
||||
|
||||
context 'when attachment has attached file' do
|
||||
before do
|
||||
attachment.update(external_url: nil)
|
||||
allow(attachment).to receive(:file).and_return(instance_double(ActiveStorage::Attached::One, attached?: true))
|
||||
allow(attachment).to receive(:file_url).and_return('https://local.com/file.jpg')
|
||||
end
|
||||
|
||||
it 'returns file_url' do
|
||||
expect(service.send(:get_attachment_url, attachment)).to eq('https://local.com/file.jpg')
|
||||
end
|
||||
end
|
||||
|
||||
context 'when attachment has no URL or file' do
|
||||
before do
|
||||
attachment.update(external_url: nil)
|
||||
allow(attachment).to receive(:file).and_return(instance_double(ActiveStorage::Attached::One, attached?: false))
|
||||
end
|
||||
|
||||
it 'returns nil' do
|
||||
expect(service.send(:get_attachment_url, attachment)).to be_nil
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#extract_audio_transcriptions' do
|
||||
let(:message) { create(:message, content: nil) }
|
||||
|
||||
context 'with no audio attachments' do
|
||||
it 'returns empty string' do
|
||||
result = service.send(:extract_audio_transcriptions, message.attachments)
|
||||
expect(result).to eq('')
|
||||
end
|
||||
end
|
||||
|
||||
context 'with successful audio transcriptions' do
|
||||
let(:audio1) do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :audio)
|
||||
attachment.save!
|
||||
attachment
|
||||
end
|
||||
|
||||
let(:audio2) do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :audio)
|
||||
attachment.save!
|
||||
attachment
|
||||
end
|
||||
|
||||
before do
|
||||
allow(Messages::AudioTranscriptionService).to receive(:new).with(audio1).and_return(
|
||||
instance_double(Messages::AudioTranscriptionService, perform: { success: true, transcriptions: 'First audio text. ' })
|
||||
)
|
||||
allow(Messages::AudioTranscriptionService).to receive(:new).with(audio2).and_return(
|
||||
instance_double(Messages::AudioTranscriptionService, perform: { success: true, transcriptions: 'Second audio text.' })
|
||||
)
|
||||
end
|
||||
|
||||
it 'concatenates all successful transcriptions' do
|
||||
audio1 # trigger creation
|
||||
audio2 # trigger creation
|
||||
|
||||
attachments = message.attachments
|
||||
result = service.send(:extract_audio_transcriptions, attachments)
|
||||
expect(result).to eq('First audio text. Second audio text.')
|
||||
end
|
||||
end
|
||||
|
||||
context 'with failed audio transcriptions' do
|
||||
let(:audio_attachment) do
|
||||
attachment = message.attachments.build(account_id: message.account_id, file_type: :audio)
|
||||
attachment.save!
|
||||
attachment
|
||||
end
|
||||
|
||||
before do
|
||||
allow(Messages::AudioTranscriptionService).to receive(:new).with(audio_attachment).and_return(
|
||||
instance_double(Messages::AudioTranscriptionService, perform: { success: false, transcriptions: nil })
|
||||
)
|
||||
end
|
||||
|
||||
it 'returns empty string for failed transcriptions' do
|
||||
audio_attachment # trigger creation
|
||||
|
||||
attachments = message.attachments
|
||||
result = service.send(:extract_audio_transcriptions, attachments)
|
||||
expect(result).to eq('')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe 'private helper methods' do
|
||||
describe '#text_part' do
|
||||
it 'returns correct text part format' do
|
||||
result = service.send(:text_part, 'Hello world')
|
||||
expect(result).to eq({ type: 'text', text: 'Hello world' })
|
||||
end
|
||||
end
|
||||
|
||||
describe '#image_part' do
|
||||
it 'returns correct image part format' do
|
||||
result = service.send(:image_part, 'https://example.com/image.jpg')
|
||||
expect(result).to eq({ type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } })
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user