fix: stream attachment handling in workers (#12870)
We’ve been watching Sidekiq workers climb from ~600 MB at boot to 1.4–1.5 GB after an hour whenever attachment-heavy jobs run. This PR is an experiment to curb that growth by streaming attachments instead of loading the whole blob into Ruby: reply-mailer inline attachments, Telegram uploads, and audio transcriptions now read/write in chunks. If this keeps RSS stable in production we’ll keep it; otherwise we’ll roll it back and keep digging
This commit is contained in:
@@ -28,13 +28,14 @@ RSpec.describe Captain::Llm::PdfProcessingService do
|
||||
context 'when uploading PDF to OpenAI' do
|
||||
let(:mock_client) { instance_double(OpenAI::Client) }
|
||||
let(:pdf_content) { 'PDF content' }
|
||||
let(:blob_double) { instance_double(ActiveStorage::Blob) }
|
||||
let(:pdf_file) { instance_double(ActiveStorage::Attachment) }
|
||||
|
||||
before do
|
||||
allow(document).to receive(:openai_file_id).and_return(nil)
|
||||
|
||||
# Use a simple double for ActiveStorage since it's a complex Rails object
|
||||
pdf_file = double('pdf_file', download: pdf_content) # rubocop:disable RSpec/VerifiedDoubles
|
||||
allow(document).to receive(:pdf_file).and_return(pdf_file)
|
||||
allow(pdf_file).to receive(:blob).and_return(blob_double)
|
||||
allow(blob_double).to receive(:open).and_yield(StringIO.new(pdf_content))
|
||||
|
||||
allow(OpenAI::Client).to receive(:new).and_return(mock_client)
|
||||
# Use a simple double for OpenAI::Files as it may not be loaded
|
||||
|
||||
@@ -15,8 +15,11 @@ RSpec.describe DataImportJob do
|
||||
|
||||
describe 'retrying the job' do
|
||||
context 'when ActiveStorage::FileNotFoundError is raised' do
|
||||
let(:import_file_double) { instance_double(ActiveStorage::Blob) }
|
||||
|
||||
before do
|
||||
allow(data_import.import_file).to receive(:download).and_raise(ActiveStorage::FileNotFoundError)
|
||||
allow(data_import).to receive(:import_file).and_return(import_file_double)
|
||||
allow(import_file_double).to receive(:open).and_raise(ActiveStorage::FileNotFoundError)
|
||||
end
|
||||
|
||||
it 'retries the job' do
|
||||
@@ -158,7 +161,9 @@ RSpec.describe DataImportJob do
|
||||
end
|
||||
|
||||
before do
|
||||
allow(data_import.import_file).to receive(:download).and_return(invalid_csv_content)
|
||||
import_file_double = instance_double(ActiveStorage::Blob)
|
||||
allow(data_import).to receive(:import_file).and_return(import_file_double)
|
||||
allow(import_file_double).to receive(:open).and_yield(StringIO.new(invalid_csv_content))
|
||||
end
|
||||
|
||||
it 'does not import any data and handles the MalformedCSVError' do
|
||||
|
||||
@@ -206,6 +206,21 @@ describe Integrations::Slack::SendOnSlackService do
|
||||
expect(message.attachments.count).to eq 2
|
||||
end
|
||||
|
||||
it 'streams attachment blobs and uploads only once' do
|
||||
expect(slack_client).to receive(:chat_postMessage).and_return(slack_message)
|
||||
|
||||
attachment = message.attachments.new(account_id: message.account_id, file_type: :image)
|
||||
attachment.file.attach(io: Rails.root.join('spec/assets/avatar.png').open, filename: 'avatar.png', content_type: 'image/png')
|
||||
blob = attachment.file.blob
|
||||
allow(blob).to receive(:open).and_call_original
|
||||
|
||||
expect(blob).to receive(:open).and_call_original
|
||||
expect(slack_client).to receive(:files_upload_v2).once.and_return(file_attachment)
|
||||
|
||||
message.save!
|
||||
builder.perform
|
||||
end
|
||||
|
||||
it 'handles file upload errors gracefully' do
|
||||
expect(slack_client).to receive(:chat_postMessage).with(
|
||||
channel: hook.reference_id,
|
||||
|
||||
Reference in New Issue
Block a user