From 141dfc33217f4c3ad137341fd85d5bceadcca505 Mon Sep 17 00:00:00 2001 From: Vinay Keerthi <11478411+stonecharioteer@users.noreply.github.com> Date: Mon, 8 Dec 2025 22:04:30 +0530 Subject: [PATCH] fix: preserve newlines and formatting in Twilio WhatsApp messages (#13022) ## Description This PR fixes an issue where Twilio WhatsApp messages were losing newlines and markdown formatting. The problem had two root causes: 1. Text-based renderers (WhatsApp, Instagram, SMS) were converting newlines to spaces when processing plain text without markdown list markers 2. Twilio WhatsApp channels were incorrectly using the plain text renderer instead of the WhatsApp renderer, stripping all markdown formatting The fix updates the markdown rendering system to: - Preserve newlines by overriding the `softbreak` method in WhatsApp, Instagram, and PlainText renderers - Detect Twilio WhatsApp channels (via the `medium` field) and route them to use the WhatsApp renderer - Maintain backward compatibility with existing code ## Type of change - [x] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality not to work as expected) - [ ] This change requires a documentation update ## How Has This Been Tested? Added comprehensive test coverage: - 3 new tests for newline preservation in WhatsApp, Instagram, and SMS channels - 4 new tests for Twilio WhatsApp specific behavior (medium detection, formatting preservation, backward compatibility) - All 53 tests passing (up from 50) Manual testing verified: - Twilio WhatsApp messages with plain text preserve newlines - Twilio WhatsApp messages with markdown preserve formatting (bold, italic, links) - Regular WhatsApp, Instagram, and SMS channels continue to work correctly - Backward compatibility maintained when channel parameter is not provided ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] Any dependent changes have been merged and published in downstream modules --- app/presenters/message_content_presenter.rb | 3 +- .../messages/markdown_renderer_service.rb | 14 +++++- .../markdown_renderers/instagram_renderer.rb | 4 ++ .../markdown_renderers/plain_text_renderer.rb | 4 ++ .../markdown_renderers/whats_app_renderer.rb | 4 ++ .../markdown_renderer_service_spec.rb | 45 ++++++++++++++++++- 6 files changed, 70 insertions(+), 4 deletions(-) diff --git a/app/presenters/message_content_presenter.rb b/app/presenters/message_content_presenter.rb index b55c1fe33..fa05d4ae9 100644 --- a/app/presenters/message_content_presenter.rb +++ b/app/presenters/message_content_presenter.rb @@ -10,7 +10,8 @@ class MessageContentPresenter < SimpleDelegator Messages::MarkdownRendererService.new( content_to_send, - conversation.inbox.channel_type + conversation.inbox.channel_type, + conversation.inbox.channel ).render end diff --git a/app/services/messages/markdown_renderer_service.rb b/app/services/messages/markdown_renderer_service.rb index b8a4df067..9348b2bc7 100644 --- a/app/services/messages/markdown_renderer_service.rb +++ b/app/services/messages/markdown_renderer_service.rb @@ -12,20 +12,30 @@ class Messages::MarkdownRendererService 'Channel::TwilioSms' => :render_plain_text }.freeze - def initialize(content, channel_type) + def initialize(content, channel_type, channel = nil) @content = content @channel_type = channel_type + @channel = channel end def render return @content if @content.blank? - renderer_method = CHANNEL_RENDERERS[@channel_type] + renderer_method = CHANNEL_RENDERERS[effective_channel_type] renderer_method ? send(renderer_method) : @content end private + def effective_channel_type + # For Twilio SMS channel, check if it's actually WhatsApp + if @channel_type == 'Channel::TwilioSms' && @channel&.whatsapp? + 'Channel::Whatsapp' + else + @channel_type + end + end + def commonmarker_doc @commonmarker_doc ||= CommonMarker.render_doc(@content, [:DEFAULT, :STRIKETHROUGH_DOUBLE_TILDE]) end diff --git a/app/services/messages/markdown_renderers/instagram_renderer.rb b/app/services/messages/markdown_renderers/instagram_renderer.rb index 57dc558d4..3b47577b2 100644 --- a/app/services/messages/markdown_renderers/instagram_renderer.rb +++ b/app/services/messages/markdown_renderers/instagram_renderer.rb @@ -41,4 +41,8 @@ class Messages::MarkdownRenderers::InstagramRenderer < Messages::MarkdownRendere out(:children) cr end + + def softbreak(_node) + out("\n") + end end diff --git a/app/services/messages/markdown_renderers/plain_text_renderer.rb b/app/services/messages/markdown_renderers/plain_text_renderer.rb index 324f60857..403d3a236 100644 --- a/app/services/messages/markdown_renderers/plain_text_renderer.rb +++ b/app/services/messages/markdown_renderers/plain_text_renderer.rb @@ -55,4 +55,8 @@ class Messages::MarkdownRenderers::PlainTextRenderer < Messages::MarkdownRendere def thematic_break(_node) out("\n") end + + def softbreak(_node) + out("\n") + end end diff --git a/app/services/messages/markdown_renderers/whats_app_renderer.rb b/app/services/messages/markdown_renderers/whats_app_renderer.rb index 0eee627d4..9f7dbab6e 100644 --- a/app/services/messages/markdown_renderers/whats_app_renderer.rb +++ b/app/services/messages/markdown_renderers/whats_app_renderer.rb @@ -29,4 +29,8 @@ class Messages::MarkdownRenderers::WhatsAppRenderer < Messages::MarkdownRenderer out('> ', :children) cr end + + def softbreak(_node) + out("\n") + end end diff --git a/spec/services/messages/markdown_renderer_service_spec.rb b/spec/services/messages/markdown_renderer_service_spec.rb index b77c1b146..26c661495 100644 --- a/spec/services/messages/markdown_renderer_service_spec.rb +++ b/spec/services/messages/markdown_renderer_service_spec.rb @@ -58,6 +58,14 @@ RSpec.describe Messages::MarkdownRendererService, type: :service do result = described_class.new(content, channel_type).render expect(result.strip).to include('- item 1') expect(result.strip).to include('- item 2') + expect(result).to include("- item 1\n- item 2") + end + + it 'preserves newlines in plain text without list markers' do + content = "Line 1\nLine 2\nLine 3" + result = described_class.new(content, channel_type).render + expect(result).to include("Line 1\nLine 2\nLine 3") + expect(result).not_to include('Line 1 Line 2') end end @@ -101,6 +109,13 @@ RSpec.describe Messages::MarkdownRendererService, type: :service do expect(result).to include('1. first step') expect(result).to include('2. second step') end + + it 'preserves newlines in plain text without list markers' do + content = "Line 1\nLine 2\nLine 3" + result = described_class.new(content, channel_type).render + expect(result).to include("Line 1\nLine 2\nLine 3") + expect(result).not_to include('Line 1 Line 2') + end end context 'when channel is Channel::Line' do @@ -179,6 +194,13 @@ RSpec.describe Messages::MarkdownRendererService, type: :service do expect(result).to include('2. second step') expect(result).to include('3. third step') end + + it 'preserves newlines in plain text without list markers' do + content = "Line 1\nLine 2\nLine 3" + result = described_class.new(content, channel_type).render + expect(result).to include("Line 1\nLine 2\nLine 3") + expect(result).not_to include('Line 1 Line 2') + end end context 'when channel is Channel::Telegram' do @@ -293,7 +315,28 @@ RSpec.describe Messages::MarkdownRendererService, type: :service do context 'when channel is Channel::TwilioSms' do let(:channel_type) { 'Channel::TwilioSms' } - it 'strips all markdown like SMS' do + it 'strips all markdown like SMS when medium is sms' do + content = '**bold** _italic_' + channel = instance_double(Channel::TwilioSms, whatsapp?: false) + result = described_class.new(content, channel_type, channel).render + expect(result.strip).to eq('bold italic') + end + + it 'uses WhatsApp renderer when medium is whatsapp' do + content = '**bold** _italic_ [link](https://example.com)' + channel = instance_double(Channel::TwilioSms, whatsapp?: true) + result = described_class.new(content, channel_type, channel).render + expect(result.strip).to eq('*bold* _italic_ https://example.com') + end + + it 'preserves newlines in Twilio WhatsApp' do + content = "Line 1\nLine 2\nLine 3" + channel = instance_double(Channel::TwilioSms, whatsapp?: true) + result = described_class.new(content, channel_type, channel).render + expect(result).to include("Line 1\nLine 2\nLine 3") + end + + it 'backwards compatible when channel is not provided' do content = '**bold** _italic_' result = described_class.new(content, channel_type).render expect(result.strip).to eq('bold italic')