fix: mutex timeout and error handling (#8770)

Fixes the follow cases 
- The ensure block released the lock even on LockAcquisitionError
- Custom timeout was not allowed

This also refactored the with_lock method, now the key has to be constructed in the parent function itself

Co-authored-by: Sojan Jose <sojan@pepalo.com>

app/jobs/inboxes/fetch_imap_emails_job.rb

@@ -6,7 +6,8 @@ class Inboxes::FetchImapEmailsJob < MutexApplicationJob
   def perform(channel)
     return unless should_fetch_email?(channel)
 
-    with_lock(::Redis::Alfred::EMAIL_MESSAGE_MUTEX, inbox_id: channel.inbox.id) do
+    key = format(::Redis::Alfred::EMAIL_MESSAGE_MUTEX, inbox_id: channel.inbox.id)
+    with_lock(key, 5.minutes) do
       process_email_for_channel(channel)
     end
   rescue *ExceptionList::IMAP_EXCEPTIONS => e

app/jobs/mutex_application_job.rb

@@ -14,20 +14,36 @@
 class MutexApplicationJob < ApplicationJob
   class LockAcquisitionError < StandardError; end
 
-  def with_lock(key_format, *args)
-    lock_key = format(key_format, *args)
+  def with_lock(lock_key, timeout = Redis::LockManager::LOCK_TIMEOUT)
     lock_manager = Redis::LockManager.new
 
     begin
-      if lock_manager.lock(lock_key)
-        Rails.logger.info "[#{self.class.name}] Acquired lock for: #{lock_key} on attempt #{executions}"
+      if lock_manager.lock(lock_key, timeout)
+        log_attempt(lock_key, executions)
         yield
+        # release the lock after the block has been executed
+        lock_manager.unlock(lock_key)
       else
-        Rails.logger.warn "[#{self.class.name}] Failed to acquire lock on attempt #{executions}: #{lock_key}"
-        raise LockAcquisitionError, "Failed to acquire lock for key: #{lock_key}"
+        handle_failed_lock_acquisition(lock_key)
       end
-    ensure
-      lock_manager.unlock(lock_key)
+    rescue StandardError => e
+      handle_error(e, lock_manager, lock_key)
     end
   end
+
+  private
+
+  def log_attempt(lock_key, executions)
+    Rails.logger.info "[#{self.class.name}] Acquired lock for: #{lock_key} on attempt #{executions}"
+  end
+
+  def handle_error(err, lock_manager, lock_key)
+    lock_manager.unlock(lock_key) unless err.is_a?(LockAcquisitionError)
+    raise err
+  end
+
+  def handle_failed_lock_acquisition(lock_key)
+    Rails.logger.warn "[#{self.class.name}] Failed to acquire lock on attempt #{executions}: #{lock_key}"
+    raise LockAcquisitionError, "Failed to acquire lock for key: #{lock_key}"
+  end
 end

app/jobs/send_on_slack_job.rb

@@ -3,7 +3,8 @@ class SendOnSlackJob < MutexApplicationJob
   retry_on LockAcquisitionError, wait: 1.second, attempts: 8
 
   def perform(message, hook)
-    with_lock(::Redis::Alfred::SLACK_MESSAGE_MUTEX, conversation_id: message.conversation_id, reference_id: hook.reference_id) do
+    key = format(::Redis::Alfred::SLACK_MESSAGE_MUTEX, conversation_id: message.conversation_id, reference_id: hook.reference_id)
+    with_lock(key) do
       Integrations::Slack::SendOnSlackService.new(message: message, hook: hook).perform
     end
   end

app/jobs/webhooks/facebook_events_job.rb

@@ -5,7 +5,8 @@ class Webhooks::FacebookEventsJob < MutexApplicationJob
   def perform(message)
     response = ::Integrations::Facebook::MessageParser.new(message)
 
-    with_lock(::Redis::Alfred::FACEBOOK_MESSAGE_MUTEX, sender_id: response.sender_id, recipient_id: response.recipient_id) do
+    key = format(::Redis::Alfred::FACEBOOK_MESSAGE_MUTEX, sender_id: response.sender_id, recipient_id: response.recipient_id)
+    with_lock(key) do
       process_message(response)
     end
   end

app/jobs/webhooks/instagram_events_job.rb

@@ -12,7 +12,8 @@ class Webhooks::InstagramEventsJob < MutexApplicationJob
   def perform(entries)
     @entries = entries
 
-    with_lock(::Redis::Alfred::IG_MESSAGE_MUTEX, sender_id: sender_id, ig_account_id: ig_account_id) do
+    key = format(::Redis::Alfred::IG_MESSAGE_MUTEX, sender_id: sender_id, ig_account_id: ig_account_id)
+    with_lock(key) do
       process_entries(entries)
     end
   end