From 6a739530035c1abc0ed2b24fb5de1d11ff5847ac Mon Sep 17 00:00:00 2001
From: Sojan Jose <sojan@pepalo.com>
Date: Tue, 3 Oct 2023 22:18:57 -0700
Subject: [PATCH] chore: Add delay before running dataimport job (#8039)

- We have observed some failures for data import jobs in the cloud due to race conditions with job executions and active storage file uploading. This PR adds delays and retries to accommodate that.
---
 app/jobs/data_import_job.rb       |  1 +
 app/models/data_import.rb         |  3 ++-
 spec/jobs/data_import_job_spec.rb | 14 ++++++++++++++
 spec/models/data_import_spec.rb   | 10 ++++++++++
 4 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/app/jobs/data_import_job.rb b/app/jobs/data_import_job.rb
index b14996fba..0a973c8bd 100644
--- a/app/jobs/data_import_job.rb
+++ b/app/jobs/data_import_job.rb
@@ -3,6 +3,7 @@
 
 class DataImportJob < ApplicationJob
   queue_as :low
+  retry_on ActiveStorage::FileNotFoundError, wait: 1.minute, attempts: 3
 
   def perform(data_import)
     @data_import = data_import
diff --git a/app/models/data_import.rb b/app/models/data_import.rb
index 874eb75f8..a44650a22 100644
--- a/app/models/data_import.rb
+++ b/app/models/data_import.rb
@@ -29,6 +29,7 @@ class DataImport < ApplicationRecord
   private
 
   def process_data_import
-    DataImportJob.perform_later(self)
+    # we wait for the file to be uploaded to the cloud
+    DataImportJob.set(wait: 1.minute).perform_later(self)
   end
 end
diff --git a/spec/jobs/data_import_job_spec.rb b/spec/jobs/data_import_job_spec.rb
index 67ca4dd73..ca20ce43b 100644
--- a/spec/jobs/data_import_job_spec.rb
+++ b/spec/jobs/data_import_job_spec.rb
@@ -13,6 +13,20 @@ RSpec.describe DataImportJob do
     end
   end
 
+  describe 'retrying the job' do
+    context 'when ActiveStorage::FileNotFoundError is raised' do
+      before do
+        allow(data_import.import_file).to receive(:download).and_raise(ActiveStorage::FileNotFoundError)
+      end
+
+      it 'retries the job' do
+        expect do
+          described_class.perform_now(data_import)
+        end.to have_enqueued_job(described_class).at_least(1).times
+      end
+    end
+  end
+
   describe 'importing data' do
     context 'when the data is valid' do
       it 'imports data into the account' do
diff --git a/spec/models/data_import_spec.rb b/spec/models/data_import_spec.rb
index 84805f86d..2b7429cf1 100644
--- a/spec/models/data_import_spec.rb
+++ b/spec/models/data_import_spec.rb
@@ -10,4 +10,14 @@ RSpec.describe DataImport do
       expect(build(:data_import, data_type: 'Xyc').valid?).to be false
     end
   end
+
+  describe 'callbacks' do
+    let(:data_import) { build(:data_import) }
+
+    it 'schedules a job after creation' do
+      expect do
+        data_import.save
+      end.to have_enqueued_job(DataImportJob).with(data_import).on_queue('low')
+    end
+  end
 end