chore: Improve Whatsapp Templates Sync (#7210)

- update the templates updated at, even if the API request fails ( to prevent jobs from stacking up in case of API failures upstream )
- sequence the job in batches of 25 requests per minutes schedule ( in case API response time is high, also not to send too many requests in a single batch )
- move the sync job re-rerun to 3 hours ( since we are updating the updated at even in case of failures )(prev 15 minutes )

Fixes: https://linear.app/chatwoot/issue/CW-1590

app/jobs/channels/whatsapp/templates_sync_scheduler_job.rb

@@ -3,10 +3,8 @@ class Channels::Whatsapp::TemplatesSyncSchedulerJob < ApplicationJob
 
   def perform
     Channel::Whatsapp.where('message_templates_last_updated <= ? OR message_templates_last_updated IS NULL',
-                            15.minutes.ago).find_in_batches do |channels_batch|
-      channels_batch.each do |channel|
-        Channels::Whatsapp::TemplatesSyncJob.perform_later(channel)
-      end
+                            3.hours.ago).limit(Limits::BULK_EXTERNAL_HTTP_CALLS_LIMIT).all.each do |channel|
+      Channels::Whatsapp::TemplatesSyncJob.perform_later(channel)
     end
   end
 end

app/services/whatsapp/providers/whatsapp_360_dialog_service.rb

@@ -23,7 +23,8 @@ class Whatsapp::Providers::Whatsapp360DialogService < Whatsapp::Providers::BaseS
 
   def sync_templates
     response = HTTParty.get("#{api_base_path}/configs/templates", headers: api_headers)
-    whatsapp_channel.update(message_templates: response['waba_templates'], message_templates_last_updated: Time.now.utc) if response.success?
+    whatsapp_channel[:message_templates] = response['waba_templates'] if response.success?
+    whatsapp_channel.update(message_templates_last_updated: Time.now.utc)
   end
 
   def validate_provider_config?

app/services/whatsapp/providers/whatsapp_cloud_service.rb

@@ -24,8 +24,8 @@ class Whatsapp::Providers::WhatsappCloudService < Whatsapp::Providers::BaseServi
 
   def sync_templates
     templates = fetch_whatsapp_templates("#{business_account_path}/message_templates?access_token=#{whatsapp_channel.provider_config['api_key']}")
-
-    whatsapp_channel.update(message_templates: templates, message_templates_last_updated: Time.now.utc) if templates.present?
+    whatsapp_channel[:message_templates] = templates if templates.present?
+    whatsapp_channel.update(message_templates_last_updated: Time.now.utc)
   end
 
   def fetch_whatsapp_templates(url)

lib/limits.rb

@@ -1,3 +1,4 @@
 module Limits
   BULK_ACTIONS_LIMIT = 100
+  BULK_EXTERNAL_HTTP_CALLS_LIMIT = 25
 end

spec/jobs/channels/whatsapp/templates_sync_scheduler_job_spec.rb

@@ -11,7 +11,7 @@ RSpec.describe Channels::Whatsapp::TemplatesSyncSchedulerJob do
       stub_request(:post, 'https://waba.360dialog.io/v1/configs/webhook')
       non_synced = create(:channel_whatsapp, sync_templates: false, message_templates_last_updated: nil)
       synced_recently = create(:channel_whatsapp, sync_templates: false, message_templates_last_updated: Time.zone.now)
-      synced_old = create(:channel_whatsapp, sync_templates: false, message_templates_last_updated: 16.minutes.ago)
+      synced_old = create(:channel_whatsapp, sync_templates: false, message_templates_last_updated: 4.hours.ago)
       described_class.perform_now
       expect(Channels::Whatsapp::TemplatesSyncJob).not_to(
         have_been_enqueued.with(synced_recently).on_queue('low')

spec/services/whatsapp/providers/whatsapp360_dialog_service_spec.rb

@@ -0,0 +1,21 @@
+## the specs are covered in send in spec/services/whatsapp/send_on_whatsapp_service_spec.rb
+require 'rails_helper'
+
+describe Whatsapp::Providers::Whatsapp360DialogService do
+  subject(:service) { described_class.new(whatsapp_channel: whatsapp_channel) }
+
+  let!(:whatsapp_channel) { create(:channel_whatsapp, sync_templates: false, validate_provider_config: false) }
+
+  describe '#sync_templates' do
+    context 'when called' do
+      it 'updates message_templates_last_updated even when template request fails' do
+        stub_request(:get, 'https://waba.360dialog.io/v1/configs/templates')
+          .to_return(status: 401)
+
+        timstamp = whatsapp_channel.reload.message_templates_last_updated
+        subject.sync_templates
+        expect(whatsapp_channel.reload.message_templates_last_updated).not_to eq(timstamp)
+      end
+    end
+  end
+end

spec/services/whatsapp/providers/whatsapp_360_dialog_service_spec.rb

@@ -1 +0,0 @@
-## the specs are covered in send in spec/services/whatsapp/send_on_whatsapp_service_spec.rb

spec/services/whatsapp/providers/whatsapp_cloud_service_spec.rb

@@ -127,10 +127,21 @@ describe Whatsapp::Providers::WhatsappCloudService do
               ], paging: { prev: 'https://graph.facebook.com/v14.0/123456789/message_templates?access_token=test_key' } }.to_json }
           )
 
+        timstamp = whatsapp_channel.reload.message_templates_last_updated
         expect(subject.sync_templates).to be(true)
         expect(whatsapp_channel.reload.message_templates.first).to eq({ id: '123456789', name: 'test_template' }.stringify_keys)
         expect(whatsapp_channel.reload.message_templates.second).to eq({ id: '123456789', name: 'next_template' }.stringify_keys)
         expect(whatsapp_channel.reload.message_templates.last).to eq({ id: '123456789', name: 'last_template' }.stringify_keys)
+        expect(whatsapp_channel.reload.message_templates_last_updated).not_to eq(timstamp)
+      end
+
+      it 'updates message_templates_last_updated even when template request fails' do
+        stub_request(:get, 'https://graph.facebook.com/v14.0/123456789/message_templates?access_token=test_key')
+          .to_return(status: 401)
+
+        timstamp = whatsapp_channel.reload.message_templates_last_updated
+        subject.sync_templates
+        expect(whatsapp_channel.reload.message_templates_last_updated).not_to eq(timstamp)
       end
     end
   end