feat(ee): Setup advanced, performant message search (#12193)

We now support searching within the actual message content, email
subject lines, and audio transcriptions. This enables a faster, more
accurate search experience going forward. Unlike the standard message
search, which is limited to the last 3 months, this search has no time
restrictions.

The search engine also accounts for small variations in queries. Minor
spelling mistakes, such as searching for slck instead of Slack, will
still return the correct results. It also ignores differences in accents
and diacritics, so searching for Deja vu will match content containing
Déjà vu.


We can also refine searches in the future by criteria such as:
- Searching within a specific inbox
- Filtering by sender or recipient
- Limiting to messages sent by an agent


Fixes https://github.com/chatwoot/chatwoot/issues/11656
Fixes https://github.com/chatwoot/chatwoot/issues/10669
Fixes https://github.com/chatwoot/chatwoot/issues/5910



---

Rake tasks to reindex all the messages. 

```sh
bundle exec rake search:all
```

Rake task to reindex messages from one account only
```sh
bundle exec rake search:account ACCOUNT_ID=1
```

app/javascript/dashboard/modules/search/components/MessageContent.vue

@@ -1,5 +1,5 @@
 <script setup>
-import { ref, useTemplateRef, onMounted, watch, nextTick } from 'vue';
+import { ref, useTemplateRef, onMounted, watch, nextTick, computed } from 'vue';
 import { useMessageFormatter } from 'shared/composables/useMessageFormatter';
 import ReadMore from './ReadMore.vue';
 
@@ -8,9 +8,9 @@ const props = defineProps({
     type: String,
     default: '',
   },
-  content: {
-    type: String,
-    default: '',
+  message: {
+    type: Object,
+    default: () => ({}),
   },
   searchTerm: {
     type: String,
@@ -18,6 +18,24 @@ const props = defineProps({
   },
 });
 
+const messageContent = computed(() => {
+  // We perform search on either content or email subject or transcribed text
+  if (props.message.content) {
+    return props.message.content;
+  }
+
+  const { content_attributes = {} } = props.message;
+  const { email = {} } = content_attributes || {};
+  if (email.subject) {
+    return email.subject;
+  }
+
+  const audioAttachment = props.message.attachments.find(
+    attachment => attachment.file_type === 'audio'
+  );
+  return audioAttachment?.transcribed_text || '';
+});
+
 const { highlightContent } = useMessageFormatter();
 
 const messageContainer = useTemplateRef('messageContainer');
@@ -38,7 +56,8 @@ const escapeHtml = html => {
   return p.innerText;
 };
 
-const prepareContent = (content = '') => {
+const prepareContent = () => {
+  const content = messageContent.value || '';
   const escapedText = escapeHtml(content);
   return highlightContent(
     escapedText,
@@ -65,7 +84,7 @@ onMounted(() => {
       {{ $t('SEARCH.WROTE') }}
     </p>
     <ReadMore :shrink="isOverflowing" @expand="isOverflowing = false">
-      <div v-dompurify-html="prepareContent(content)" class="message-content" />
+      <div v-dompurify-html="prepareContent()" class="message-content" />
     </ReadMore>
   </blockquote>
 </template>
@@ -74,6 +93,7 @@ onMounted(() => {
 .message {
   @apply py-0 px-2 mt-2;
 }
+
 .message-content::v-deep p,
 .message-content::v-deep li::marker {
   @apply text-n-slate-11 mb-1;

app/javascript/dashboard/modules/search/components/SearchResultMessagesList.vue

@@ -54,7 +54,7 @@ const getName = message => {
         >
           <MessageContent
             :author="getName(message)"
-            :content="message.content"
+            :message="message"
             :search-term="query"
           />
         </SearchResultConversationItem>

app/models/message.rb

@@ -39,6 +39,8 @@
 #
 
 class Message < ApplicationRecord
+  searchkick callbacks: :async if ChatwootApp.advanced_search_allowed?
+
   include MessageFilterHelpers
   include Liquidable
   NUMBER_OF_PERMITTED_ATTACHMENTS = 15
@@ -139,14 +141,23 @@ class Message < ApplicationRecord
     data = attributes.symbolize_keys.merge(
       created_at: created_at.to_i,
       message_type: message_type_before_type_cast,
-      conversation_id: conversation.display_id,
-      conversation: conversation_push_event_data
+      conversation_id: conversation&.display_id,
+      conversation: conversation.present? ? conversation_push_event_data : nil
     )
     data[:echo_id] = echo_id if echo_id.present?
     data[:attachments] = attachments.map(&:push_event_data) if attachments.present?
     merge_sender_attributes(data)
   end
 
+  def search_data
+    data = attributes.symbolize_keys
+    data[:conversation] = conversation.present? ? conversation_push_event_data : nil
+    data[:attachments] = attachments.map(&:push_event_data) if attachments.present?
+    data[:sender] = sender.push_event_data if sender
+    data[:inbox] = inbox
+    data
+  end
+
   def conversation_push_event_data
     {
       assignee_id: conversation.assignee_id,
@@ -228,6 +239,14 @@ class Message < ApplicationRecord
                                                                             previous_changes: previous_changes)
   end
 
+  def should_index?
+    return false unless ChatwootApp.advanced_search_allowed?
+    return false unless account.feature_enabled?('advanced_search')
+    return false unless incoming? || outgoing?
+
+    true
+  end
+
   private
 
   def prevent_message_flooding

app/services/search_service.rb

@@ -43,11 +43,19 @@ class SearchService
   def filter_messages
     @messages = if use_gin_search
                   filter_messages_with_gin
+                elsif should_run_advanced_search?
+                  advanced_search
                 else
                   filter_messages_with_like
                 end
   end
 
+  def should_run_advanced_search?
+    ChatwootApp.advanced_search_allowed? && current_account.feature_enabled?('advanced_search')
+  end
+
+  def advanced_search; end
+
   def filter_messages_with_gin
     base_query = message_base_query
 
@@ -115,3 +123,5 @@ class SearchService
                                .per(15)
   end
 end
+
+SearchService.prepend_mod_with('SearchService')

app/views/api/v1/accounts/search/_message.json.jbuilder

@@ -1,12 +1 @@
-json.id message.id
-json.content message.content
-json.message_type message.message_type_before_type_cast
-json.content_type message.content_type
-json.source_id message.source_id
-json.inbox_id message.inbox_id
-json.conversation_id message.conversation.try(:display_id)
-json.created_at message.created_at.to_i
-json.sender message.sender.push_event_data if message.sender
-json.inbox do
-  json.partial! 'inbox', formats: [:json], inbox: message.inbox if message.inbox.present? && message.try(:inbox).present?
-end
+json.partial! 'api/v1/models/message', message: message