chore: Reorganize Sidekiq Queues (#6976)

- Rearrange and reprioritize current sidekiq queues
- Trim the unnecessary queues

ref: https://linear.app/chatwoot/issue/CW-1480/chore-run-all-sidekiq-jobs-async

app/jobs/action_cable_broadcast_job.rb

@@ -1,5 +1,5 @@
 class ActionCableBroadcastJob < ApplicationJob
-  queue_as :default
+  queue_as :critical
 
   def perform(members, event_name, data)
     members.each do |member|

app/jobs/agent_bots/csml_job.rb

@@ -1,5 +1,5 @@
 class AgentBots::CsmlJob < ApplicationJob
-  queue_as :bots
+  queue_as :high
 
   def perform(event, agent_bot, message)
     event_data = { message: message }

app/jobs/agent_bots/webhook_job.rb

@@ -1,3 +1,3 @@
 class AgentBots::WebhookJob < WebhookJob
-  queue_as :bots
+  queue_as :high
 end

app/jobs/agents/destroy_job.rb

@@ -1,5 +1,5 @@
 class Agents::DestroyJob < ApplicationJob
-  queue_as :default
+  queue_as :low
 
   def perform(account, user)
     ActiveRecord::Base.transaction do

app/jobs/avatar/avatar_from_url_job.rb

@@ -1,5 +1,5 @@
 class Avatar::AvatarFromUrlJob < ApplicationJob
-  queue_as :default
+  queue_as :low
 
   def perform(avatarable, avatar_url)
     return unless avatarable.respond_to?(:avatar)

app/jobs/conversations/activity_message_job.rb

@@ -1,5 +1,5 @@
 class Conversations::ActivityMessageJob < ApplicationJob
-  queue_as :default
+  queue_as :high
 
   def perform(conversation, message_params)
     conversation.messages.create!(message_params)

app/jobs/delete_object_job.rb

@@ -1,5 +1,5 @@
 class DeleteObjectJob < ApplicationJob
-  queue_as :default
+  queue_as :low
 
   def perform(object)
     object.destroy!

app/jobs/event_dispatcher_job.rb

@@ -1,5 +1,5 @@
 class EventDispatcherJob < ApplicationJob
-  queue_as :events
+  queue_as :critical
 
   def perform(event_name, timestamp, data)
     Rails.configuration.dispatcher.async_dispatcher.publish_event(event_name, timestamp, data)

app/jobs/hook_job.rb

@@ -1,5 +1,5 @@
 class HookJob < ApplicationJob
-  queue_as :integrations
+  queue_as :medium
 
   def perform(hook, event_name, event_data = {})
     case hook.app_id

app/jobs/inboxes/fetch_imap_email_inboxes_job.rb

@@ -1,5 +1,5 @@
 class Inboxes::FetchImapEmailInboxesJob < ApplicationJob
-  queue_as :low
+  queue_as :scheduled_jobs
 
   def perform
     Inbox.where(channel_type: 'Channel::Email').all.find_each(batch_size: 100) do |inbox|

app/jobs/inboxes/fetch_imap_emails_job.rb

@@ -1,7 +1,7 @@
 require 'net/imap'
 
 class Inboxes::FetchImapEmailsJob < ApplicationJob
-  queue_as :low
+  queue_as :scheduled_jobs
 
   def perform(channel)
     return unless should_fetch_email?(channel)

app/jobs/webhook_job.rb

@@ -1,5 +1,5 @@
 class WebhookJob < ApplicationJob
-  queue_as :webhooks
+  queue_as :medium
 
   def perform(url, payload)
     Webhooks::Trigger.execute(url, payload)

app/models/custom_attribute_definition.rb

@@ -40,10 +40,10 @@ class CustomAttributeDefinition < ApplicationRecord
   private
 
   def sync_widget_pre_chat_custom_fields
-    ::Inboxes::SyncWidgetPreChatCustomFieldsJob.perform_now(account, attribute_key)
+    ::Inboxes::SyncWidgetPreChatCustomFieldsJob.perform_later(account, attribute_key)
   end
 
   def update_widget_pre_chat_custom_fields
-    ::Inboxes::UpdateWidgetPreChatCustomFieldsJob.perform_now(account, self)
+    ::Inboxes::UpdateWidgetPreChatCustomFieldsJob.perform_later(account, self)
   end
 end

config/sidekiq.yml

@@ -11,23 +11,22 @@
 # Sidekiq will run this file through ERB when reading it so you can
 # even put in dynamic logic, like a host-specific queue.
 # http://www.mikeperham.com/2013/11/13/advanced-sidekiq-host-specific-queues/
+# https://github.com/sidekiq/sidekiq/wiki/Advanced-Options
+# Since queues are declared without waits, the jobs in lower ranking queues will only be processed
+# if there are no jobs in higher ranking queues.
 :queues:
-  - [async_database_migration, 1]
-  - [low, 2]
-  - [scheduled_jobs, 2]
-  - [webhooks, 2]
-  - [bots, 2]
-  - [active_storage_analysis, 2]
-  - [action_mailbox_incineration, 2]
-  - [active_storage_purge, 2]
-  - [integrations, 3]
-  - [default, 3]
-  - [mailers, 3]
-  - [medium, 4]
-  - [events, 4]
-  - [action_mailbox_routing, 4]
-  - [high, 5]
-  - [critical, 10]
+  - critical
+  - high
+  - medium
+  - default
+  - mailers
+  - action_mailbox_routing
+  - low
+  - scheduled_jobs
+  - async_database_migration
+  - active_storage_analysis
+  - active_storage_purge
+  - action_mailbox_incineration
 
 # you can override concurrency based on environment
 production:

spec/jobs/agent_bots/webhook_job_spec.rb

@@ -11,7 +11,7 @@ RSpec.describe AgentBots::WebhookJob, type: :job do
   it 'queues the job' do
     expect { job }.to have_enqueued_job(described_class)
       .with(url, payload)
-      .on_queue('bots')
+      .on_queue('high')
   end
 
   it 'executes perform' do

spec/jobs/agents/destroy_job_spec.rb

@@ -17,7 +17,7 @@ RSpec.describe Agents::DestroyJob, type: :job do
   it 'enqueues the job' do
     expect { job }.to have_enqueued_job(described_class)
       .with(account, user)
-      .on_queue('default')
+      .on_queue('low')
   end
 
   describe '#perform' do

spec/jobs/avatar/avatar_from_url_job_spec.rb

@@ -6,7 +6,7 @@ RSpec.describe Avatar::AvatarFromUrlJob, type: :job do
 
   it 'enqueues the job' do
     expect { described_class.perform_later(avatarable, avatar_url) }.to have_enqueued_job(described_class)
-      .on_queue('default')
+      .on_queue('low')
   end
 
   it 'will attach avatar from url' do

spec/jobs/delete_object_job_spec.rb

@@ -8,7 +8,7 @@ RSpec.describe DeleteObjectJob, type: :job do
   it 'enqueues the job' do
     expect { job }.to have_enqueued_job(described_class)
       .with(account)
-      .on_queue('default')
+      .on_queue('low')
   end
 
   context 'when an object is passed to the job' do

spec/jobs/event_dispatcher_job_spec.rb

@@ -11,7 +11,7 @@ RSpec.describe EventDispatcherJob, type: :job do
   it 'queues the job' do
     expect { job }.to have_enqueued_job(described_class)
       .with(event_name, timestamp, event_data)
-      .on_queue('events')
+      .on_queue('critical')
   end
 
   it 'publishes event' do

spec/jobs/hook_job_spec.rb

@@ -12,7 +12,7 @@ RSpec.describe HookJob, type: :job do
   it 'enqueues the job' do
     expect { job }.to have_enqueued_job(described_class)
       .with(hook, event_name, event_data)
-      .on_queue('integrations')
+      .on_queue('medium')
   end
 
   context 'when handleable events like message.created' do

spec/jobs/inboxes/fetch_imap_email_inboxes_job_spec.rb

@@ -10,7 +10,7 @@ RSpec.describe Inboxes::FetchImapEmailInboxesJob, type: :job do
 
   it 'enqueues the job' do
     expect { described_class.perform_later }.to have_enqueued_job(described_class)
-      .on_queue('low')
+      .on_queue('scheduled_jobs')
   end
 
   context 'when called' do

spec/jobs/inboxes/fetch_imap_emails_job_spec.rb

@@ -19,7 +19,7 @@ RSpec.describe Inboxes::FetchImapEmailsJob, type: :job do
 
   it 'enqueues the job' do
     expect { described_class.perform_later }.to have_enqueued_job(described_class)
-      .on_queue('low')
+      .on_queue('scheduled_jobs')
   end
 
   context 'when imap fetch new emails' do

spec/jobs/webhook_job_spec.rb

@@ -11,7 +11,7 @@ RSpec.describe WebhookJob, type: :job do
   it 'queues the job' do
     expect { job }.to have_enqueued_job(described_class)
       .with(url, payload)
-      .on_queue('webhooks')
+      .on_queue('medium')
   end
 
   it 'executes perform' do