From 20fa5eeaa544b326d603fa8c87c39c183db442a1 Mon Sep 17 00:00:00 2001 From: Muhsin Keloth Date: Wed, 10 Dec 2025 12:28:47 +0530 Subject: [PATCH] fix: Prevent SLA deletion timeouts by moving to async job (#12944) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR fixes the HTTP 500 timeout errors occurring when deleting SLA policies that have large volumes of historical data. The fix moves the deletion workflow to asynchronous background processing using the existing `DeleteObjectJob`. By offloading heavy cascaded deletions (applied SLAs, SLA events, conversation nullifications) from the request cycle, the API can now return immediately while the cleanup continues in the background avoiding the `Rack::Timeout::RequestTimeoutException`. This ensures that SLA policies can be deleted reliably, regardless of data size. ### Problem Deleting an SLA policy via `DELETE /api/v1/accounts/{account_id}/sla_policies/{id}` fails consistently with `Rack::Timeout::RequestTimeoutException (15s)` for policies with large amounts of related data. Because the current implementation performs all dependent deletions **synchronously**, Rails processes: - `has_many :applied_slas, dependent: :destroy` (thousands) - Each `AppliedSla#destroy` → triggers destruction of many `SlaEvent` records - `has_many :conversations, dependent: :nullify` (thousands) This processing far exceeds the Rack timeout window and consistently triggers HTTP 500 errors for users. ### Solution This PR applies the same pattern used successfully in Inbox deletion. **Move deletion to async background jobs** - Uses `DeleteObjectJob` for centralized, reliable cleanup. - Allows the DELETE API call to respond immediately. **Chunk large datasets** - Records are processed in **batches of 5,000** to reduce DB load and avoid job timeouts. --- app/jobs/delete_object_job.rb | 15 +- .../v1/accounts/sla_policies_controller.rb | 2 +- .../app/jobs/enterprise/delete_object_job.rb | 10 +- enterprise/app/models/sla_policy.rb | 2 +- lib/tasks/apply_sla.rake | 100 ++++++++++ lib/tasks/bulk_conversations.rake | 176 ++++++++++++++++++ .../accounts/sla_policies_controller_spec.rb | 5 +- 7 files changed, 299 insertions(+), 11 deletions(-) create mode 100644 lib/tasks/apply_sla.rake create mode 100644 lib/tasks/bulk_conversations.rake diff --git a/app/jobs/delete_object_job.rb b/app/jobs/delete_object_job.rb index 756d0feb1..4e9030012 100644 --- a/app/jobs/delete_object_job.rb +++ b/app/jobs/delete_object_job.rb @@ -2,10 +2,6 @@ class DeleteObjectJob < ApplicationJob queue_as :low BATCH_SIZE = 5_000 - HEAVY_ASSOCIATIONS = { - Account => %i[conversations contacts inboxes reporting_events], - Inbox => %i[conversations contact_inboxes reporting_events] - }.freeze def perform(object, user = nil, ip = nil) # Pre-purge heavy associations for large objects to avoid @@ -19,11 +15,18 @@ class DeleteObjectJob < ApplicationJob private + def heavy_associations + { + Account => %i[conversations contacts inboxes reporting_events], + Inbox => %i[conversations contact_inboxes reporting_events] + }.freeze + end + def purge_heavy_associations(object) - klass = HEAVY_ASSOCIATIONS.keys.find { |k| object.is_a?(k) } + klass = heavy_associations.keys.find { |k| object.is_a?(k) } return unless klass - HEAVY_ASSOCIATIONS[klass].each do |assoc| + heavy_associations[klass].each do |assoc| next unless object.respond_to?(assoc) batch_destroy(object.public_send(assoc)) diff --git a/enterprise/app/controllers/api/v1/accounts/sla_policies_controller.rb b/enterprise/app/controllers/api/v1/accounts/sla_policies_controller.rb index ec879b042..b3b0f52d4 100644 --- a/enterprise/app/controllers/api/v1/accounts/sla_policies_controller.rb +++ b/enterprise/app/controllers/api/v1/accounts/sla_policies_controller.rb @@ -17,7 +17,7 @@ class Api::V1::Accounts::SlaPoliciesController < Api::V1::Accounts::EnterpriseAc end def destroy - @sla_policy.destroy! + ::DeleteObjectJob.perform_later(@sla_policy, Current.user, request.ip) if @sla_policy.present? head :ok end diff --git a/enterprise/app/jobs/enterprise/delete_object_job.rb b/enterprise/app/jobs/enterprise/delete_object_job.rb index 147fa0d43..375015bc7 100644 --- a/enterprise/app/jobs/enterprise/delete_object_job.rb +++ b/enterprise/app/jobs/enterprise/delete_object_job.rb @@ -1,10 +1,18 @@ module Enterprise::DeleteObjectJob + private + + def heavy_associations + super.merge( + SlaPolicy => %i[applied_slas] + ).freeze + end + def process_post_deletion_tasks(object, user, ip) create_audit_entry(object, user, ip) end def create_audit_entry(object, user, ip) - return unless %w[Inbox Conversation].include?(object.class.to_s) && user.present? + return unless %w[Inbox Conversation SlaPolicy].include?(object.class.to_s) && user.present? Enterprise::AuditLog.create( auditable: object, diff --git a/enterprise/app/models/sla_policy.rb b/enterprise/app/models/sla_policy.rb index f53f00ed5..f61d5a1f3 100644 --- a/enterprise/app/models/sla_policy.rb +++ b/enterprise/app/models/sla_policy.rb @@ -22,7 +22,7 @@ class SlaPolicy < ApplicationRecord validates :name, presence: true has_many :conversations, dependent: :nullify - has_many :applied_slas, dependent: :destroy + has_many :applied_slas, dependent: :destroy_async def push_event_data { diff --git a/lib/tasks/apply_sla.rake b/lib/tasks/apply_sla.rake new file mode 100644 index 000000000..70adf8cf3 --- /dev/null +++ b/lib/tasks/apply_sla.rake @@ -0,0 +1,100 @@ +# Apply SLA Policy to Conversations +# +# This task applies an SLA policy to existing conversations that don't have one assigned. +# It processes conversations in batches and only affects conversations with sla_policy_id = nil. +# +# Usage Examples: +# # Using arguments (may need escaping in some shells) +# bundle exec rake "sla:apply_to_conversations[19,1,500]" +# +# # Using environment variables (recommended) +# SLA_POLICY_ID=19 ACCOUNT_ID=1 BATCH_SIZE=500 bundle exec rake sla:apply_to_conversations +# +# Parameters: +# SLA_POLICY_ID: ID of the SLA policy to apply (required) +# ACCOUNT_ID: ID of the account (required) +# BATCH_SIZE: Number of conversations to process (default: 1000) +# +# Notes: +# - Only runs in development environment +# - Processes conversations in order of newest first (id DESC) +# - Safe to run multiple times - skips conversations that already have SLA policies +# - Creates AppliedSla records automatically via Rails callbacks +# - SlaEvent records are created later by background jobs when violations occur +# +# rubocop:disable Metrics/BlockLength +namespace :sla do + desc 'Apply SLA policy to existing conversations' + task :apply_to_conversations, [:sla_policy_id, :account_id, :batch_size] => :environment do |_t, args| + unless Rails.env.development? + puts 'This task can only be run in the development environment.' + puts "Current environment: #{Rails.env}" + exit(1) + end + + sla_policy_id = args[:sla_policy_id] || ENV.fetch('SLA_POLICY_ID', nil) + account_id = args[:account_id] || ENV.fetch('ACCOUNT_ID', nil) + batch_size = (args[:batch_size] || ENV['BATCH_SIZE'] || 1000).to_i + + if sla_policy_id.blank? + puts 'Error: SLA_POLICY_ID is required' + puts 'Usage: bundle exec rake sla:apply_to_conversations[sla_policy_id,account_id,batch_size]' + puts 'Or: SLA_POLICY_ID=1 ACCOUNT_ID=1 BATCH_SIZE=500 bundle exec rake sla:apply_to_conversations' + exit(1) + end + + if account_id.blank? + puts 'Error: ACCOUNT_ID is required' + puts 'Usage: bundle exec rake sla:apply_to_conversations[sla_policy_id,account_id,batch_size]' + puts 'Or: SLA_POLICY_ID=1 ACCOUNT_ID=1 BATCH_SIZE=500 bundle exec rake sla:apply_to_conversations' + exit(1) + end + + account = Account.find_by(id: account_id) + unless account + puts "Error: Account with ID #{account_id} not found" + exit(1) + end + + sla_policy = account.sla_policies.find_by(id: sla_policy_id) + unless sla_policy + puts "Error: SLA Policy with ID #{sla_policy_id} not found for Account #{account_id}" + exit(1) + end + + conversations = account.conversations.where(sla_policy_id: nil).order(id: :desc).limit(batch_size) + total_count = conversations.count + + if total_count.zero? + puts 'No conversations found without SLA policy' + exit(0) + end + + puts "Applying SLA Policy '#{sla_policy.name}' (ID: #{sla_policy_id}) to #{total_count} conversations in Account #{account_id}" + puts "Processing in batches of #{batch_size}" + puts "Started at: #{Time.current}" + + start_time = Time.current + processed_count = 0 + error_count = 0 + + conversations.find_in_batches(batch_size: batch_size) do |batch| + batch.each do |conversation| + conversation.update!(sla_policy_id: sla_policy_id) + processed_count += 1 + puts "Processed #{processed_count}/#{total_count} conversations" if (processed_count % 100).zero? + rescue StandardError => e + error_count += 1 + puts "Error applying SLA to conversation #{conversation.id}: #{e.message}" + end + end + + elapsed_time = Time.current - start_time + puts "\nCompleted!" + puts "Successfully processed: #{processed_count} conversations" + puts "Errors encountered: #{error_count}" if error_count.positive? + puts "Total time: #{elapsed_time.round(2)}s" + puts "Average time per conversation: #{(elapsed_time / processed_count).round(3)}s" if processed_count.positive? + end +end +# rubocop:enable Metrics/BlockLength diff --git a/lib/tasks/bulk_conversations.rake b/lib/tasks/bulk_conversations.rake new file mode 100644 index 000000000..e17851f31 --- /dev/null +++ b/lib/tasks/bulk_conversations.rake @@ -0,0 +1,176 @@ +# Generate Bulk Conversations +# +# This task creates bulk conversations with fake contacts and movie dialogue messages +# for testing purposes. Each conversation gets random messages between contacts and agents. +# +# Usage Examples: +# # Using arguments (may need escaping in some shells) +# bundle exec rake "conversations:generate_bulk[100,1,1]" +# +# # Using environment variables (recommended) +# COUNT=100 ACCOUNT_ID=1 INBOX_ID=1 bundle exec rake conversations:generate_bulk +# +# # Generate 50 conversations +# COUNT=50 ACCOUNT_ID=1 INBOX_ID=1 bundle exec rake conversations:generate_bulk +# +# Parameters: +# COUNT: Number of conversations to create (default: 10) +# ACCOUNT_ID: ID of the account (required) +# INBOX_ID: ID of the inbox that belongs to the account (required) +# +# What it creates: +# - Unique contacts with fake names, emails, phone numbers +# - Conversations with random status (open/resolved/pending) +# - 3-10 messages per conversation with movie quotes +# - Alternating incoming/outgoing message flow +# +# Notes: +# - Only runs in development environment +# - Creates realistic test data for conversation testing +# - Progress shown every 10 conversations +# - All contacts get unique email addresses to avoid conflicts +# +# rubocop:disable Metrics/BlockLength +namespace :conversations do + desc 'Generate bulk conversations with contacts and movie dialogue messages' + task :generate_bulk, [:count, :account_id, :inbox_id] => :environment do |_t, args| + unless Rails.env.development? + puts 'This task can only be run in the development environment.' + puts "Current environment: #{Rails.env}" + exit(1) + end + + count = (args[:count] || ENV['COUNT'] || 10).to_i + account_id = args[:account_id] || ENV.fetch('ACCOUNT_ID', nil) + inbox_id = args[:inbox_id] || ENV.fetch('INBOX_ID', nil) + + if account_id.blank? + puts 'Error: ACCOUNT_ID is required' + puts 'Usage: bundle exec rake conversations:generate_bulk[count,account_id,inbox_id]' + puts 'Or: COUNT=100 ACCOUNT_ID=1 INBOX_ID=1 bundle exec rake conversations:generate_bulk' + exit(1) + end + + if inbox_id.blank? + puts 'Error: INBOX_ID is required' + puts 'Usage: bundle exec rake conversations:generate_bulk[count,account_id,inbox_id]' + puts 'Or: COUNT=100 ACCOUNT_ID=1 INBOX_ID=1 bundle exec rake conversations:generate_bulk' + exit(1) + end + + account = Account.find_by(id: account_id) + inbox = Inbox.find_by(id: inbox_id) + + unless account + puts "Error: Account with ID #{account_id} not found" + exit(1) + end + + unless inbox + puts "Error: Inbox with ID #{inbox_id} not found" + exit(1) + end + + unless inbox.account_id == account.id + puts "Error: Inbox #{inbox_id} does not belong to Account #{account_id}" + exit(1) + end + + puts "Generating #{count} conversations for Account ##{account.id} in Inbox ##{inbox.id}..." + puts "Started at: #{Time.current}" + + start_time = Time.current + created_count = 0 + + count.times do |i| + contact = create_contact(account) + contact_inbox = create_contact_inbox(contact, inbox) + conversation = create_conversation(contact_inbox) + add_messages(conversation) + + created_count += 1 + puts "Created conversation #{i + 1}/#{count} (ID: #{conversation.id})" if ((i + 1) % 10).zero? + rescue StandardError => e + puts "Error creating conversation #{i + 1}: #{e.message}" + puts e.backtrace.first(5).join("\n") + end + + elapsed_time = Time.current - start_time + puts "\nCompleted!" + puts "Successfully created: #{created_count} conversations" + puts "Total time: #{elapsed_time.round(2)}s" + puts "Average time per conversation: #{(elapsed_time / created_count).round(3)}s" if created_count.positive? + end + + def create_contact(account) + Contact.create!( + account: account, + name: Faker::Name.name, + email: "#{SecureRandom.uuid}@example.com", + phone_number: generate_e164_phone_number, + additional_attributes: { + source: 'bulk_generator', + company: Faker::Company.name, + city: Faker::Address.city + } + ) + end + + def generate_e164_phone_number + country_code = [1, 44, 61, 91, 81].sample + subscriber_number = rand(1_000_000..9_999_999_999).to_s + subscriber_number = subscriber_number[0...(15 - country_code.to_s.length)] + "+#{country_code}#{subscriber_number}" + end + + def create_contact_inbox(contact, inbox) + ContactInboxBuilder.new( + contact: contact, + inbox: inbox + ).perform + end + + def create_conversation(contact_inbox) + ConversationBuilder.new( + params: ActionController::Parameters.new( + status: %w[open resolved pending].sample, + additional_attributes: {}, + custom_attributes: {} + ), + contact_inbox: contact_inbox + ).perform + end + + def add_messages(conversation) + num_messages = rand(3..10) + message_type = %w[incoming outgoing].sample + + num_messages.times do + message_type = message_type == 'incoming' ? 'outgoing' : 'incoming' + create_message(conversation, message_type) + end + end + + def create_message(conversation, message_type) + sender = if message_type == 'incoming' + conversation.contact + else + conversation.account.users.sample || conversation.account.administrators.first + end + + conversation.messages.create!( + account: conversation.account, + inbox: conversation.inbox, + sender: sender, + message_type: message_type, + content: generate_movie_dialogue, + content_type: :text, + private: false + ) + end + + def generate_movie_dialogue + Faker::Movie.quote + end +end +# rubocop:enable Metrics/BlockLength diff --git a/spec/enterprise/controllers/api/v1/accounts/sla_policies_controller_spec.rb b/spec/enterprise/controllers/api/v1/accounts/sla_policies_controller_spec.rb index b1619ef85..e1a4fa538 100644 --- a/spec/enterprise/controllers/api/v1/accounts/sla_policies_controller_spec.rb +++ b/spec/enterprise/controllers/api/v1/accounts/sla_policies_controller_spec.rb @@ -161,12 +161,13 @@ RSpec.describe 'Enterprise SLA API', type: :request do let(:sla_policy) { create(:sla_policy, account: account) } context 'when it is an authenticated user' do - it 'deletes the sla_policy' do + it 'queues the sla_policy for deletion' do + expect(DeleteObjectJob).to receive(:perform_later).with(sla_policy, administrator, kind_of(String)) + delete "/api/v1/accounts/#{account.id}/sla_policies/#{sla_policy.id}", headers: administrator.create_new_auth_token expect(response).to have_http_status(:success) - expect(SlaPolicy.count).to eq(1) end end