diff --git a/app/jobs/data_import_job.rb b/app/jobs/data_import_job.rb index 6d0de7c89..b14996fba 100644 --- a/app/jobs/data_import_job.rb +++ b/app/jobs/data_import_job.rb @@ -27,7 +27,11 @@ class DataImportJob < ApplicationJob rejected_contacts = [] # Ensuring that importing non utf-8 characters will not throw error data = @data_import.import_file.download - clean_data = data.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') + utf8_data = data.force_encoding('UTF-8') + + # Ensure that the data is valid UTF-8, preserving valid characters + clean_data = utf8_data.valid_encoding? ? utf8_data : utf8_data.encode('UTF-16le', invalid: :replace, replace: '').encode('UTF-8') + csv = CSV.parse(clean_data, headers: true) csv.each do |row| diff --git a/spec/fixtures/data_import/with_emoji.csv b/spec/fixtures/data_import/with_emoji.csv new file mode 100644 index 000000000..f891e348b --- /dev/null +++ b/spec/fixtures/data_import/with_emoji.csv @@ -0,0 +1,2 @@ +phone_number,name ++123456,T 🏠 🔥 Test diff --git a/spec/jobs/data_import_job_spec.rb b/spec/jobs/data_import_job_spec.rb index 032858029..67ca4dd73 100644 --- a/spec/jobs/data_import_job_spec.rb +++ b/spec/jobs/data_import_job_spec.rb @@ -48,6 +48,19 @@ RSpec.describe DataImportJob do expect(invalid_data_import.reload.processed_records).to eq(csv_length) end + it 'will preserve emojis' do + data_import = create(:data_import, + import_file: Rack::Test::UploadedFile.new(Rails.root.join('spec/fixtures/data_import/with_emoji.csv'), + 'text/csv')) + csv_data = CSV.parse(data_import.import_file.download, headers: true) + csv_length = csv_data.length + + described_class.perform_now(data_import) + expect(data_import.account.contacts.count).to eq(csv_length) + + expect(data_import.account.contacts.first.name).to eq('T 🏠 🔥 Test') + end + it 'will not throw error for non utf-8 characters' do invalid_data_import = create(:data_import, import_file: Rack::Test::UploadedFile.new(Rails.root.join('spec/fixtures/data_import/invalid_bytes.csv'),