fix: stream attachment handling in workers (#12870)
We’ve been watching Sidekiq workers climb from ~600 MB at boot to 1.4–1.5 GB after an hour whenever attachment-heavy jobs run. This PR is an experiment to curb that growth by streaming attachments instead of loading the whole blob into Ruby: reply-mailer inline attachments, Telegram uploads, and audio transcriptions now read/write in chunks. If this keeps RSS stable in production we’ll keep it; otherwise we’ll roll it back and keep digging
This commit is contained in:
51
app/mailers/conversation_reply_mailer_attachment_helper.rb
Normal file
51
app/mailers/conversation_reply_mailer_attachment_helper.rb
Normal file
@@ -0,0 +1,51 @@
|
||||
# Handles attachment processing for ConversationReplyMailer flows.
|
||||
module ConversationReplyMailerAttachmentHelper
|
||||
private
|
||||
|
||||
def process_attachments_as_files_for_email_reply
|
||||
# Attachment processing for direct email replies (when replying to a single message)
|
||||
#
|
||||
# How attachments are handled:
|
||||
# 1. Total file size (<20MB): Added directly to the email as proper attachments
|
||||
# 2. Total file size (>20MB): Added to @large_attachments to be displayed as links in the email
|
||||
|
||||
@options[:attachments] = []
|
||||
@large_attachments = []
|
||||
current_total_size = 0
|
||||
|
||||
@message.attachments.each do |attachment|
|
||||
current_total_size = handle_attachment_inline(current_total_size, attachment)
|
||||
end
|
||||
end
|
||||
|
||||
def read_blob_content(blob)
|
||||
buffer = +''
|
||||
blob.open do |file|
|
||||
while (chunk = file.read(64.kilobytes))
|
||||
buffer << chunk
|
||||
end
|
||||
end
|
||||
buffer
|
||||
end
|
||||
|
||||
def handle_attachment_inline(current_total_size, attachment)
|
||||
blob = attachment.file.blob
|
||||
return current_total_size if blob.blank?
|
||||
|
||||
file_size = blob.byte_size
|
||||
attachment_name = attachment.file.filename.to_s
|
||||
|
||||
if current_total_size + file_size <= 20.megabytes
|
||||
content = read_blob_content(blob)
|
||||
mail.attachments[attachment_name] = {
|
||||
mime_type: attachment.file.content_type || 'application/octet-stream',
|
||||
content: content
|
||||
}
|
||||
@options[:attachments] << { name: attachment_name }
|
||||
current_total_size + file_size
|
||||
else
|
||||
@large_attachments << attachment
|
||||
current_total_size
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,4 +1,6 @@
|
||||
module ConversationReplyMailerHelper
|
||||
include ConversationReplyMailerAttachmentHelper
|
||||
|
||||
def prepare_mail(cc_bcc_enabled)
|
||||
@options = {
|
||||
to: to_emails,
|
||||
@@ -27,34 +29,6 @@ module ConversationReplyMailerHelper
|
||||
mail(@options)
|
||||
end
|
||||
|
||||
def process_attachments_as_files_for_email_reply
|
||||
# Attachment processing for direct email replies (when replying to a single message)
|
||||
#
|
||||
# How attachments are handled:
|
||||
# 1. Total file size (<20MB): Added directly to the email as proper attachments
|
||||
# 2. Total file size (>20MB): Added to @large_attachments to be displayed as links in the email
|
||||
|
||||
@options[:attachments] = []
|
||||
@large_attachments = []
|
||||
current_total_size = 0
|
||||
|
||||
@message.attachments.each do |attachment|
|
||||
raw_data = attachment.file.download
|
||||
attachment_name = attachment.file.filename.to_s
|
||||
file_size = raw_data.bytesize
|
||||
|
||||
# Attach files directly until we hit 20MB total
|
||||
# After reaching 20MB, send remaining files as links
|
||||
if current_total_size + file_size <= 20.megabytes
|
||||
mail.attachments[attachment_name] = raw_data
|
||||
@options[:attachments] << { name: attachment_name }
|
||||
current_total_size += file_size
|
||||
else
|
||||
@large_attachments << attachment
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def oauth_smtp_settings
|
||||
|
||||
Reference in New Issue
Block a user