From 6096932f7669a1997395b7bba974532cfae32f0b Mon Sep 17 00:00:00 2001
From: Pranav <pranav@chatwoot.com>
Date: Wed, 15 Jan 2025 20:24:34 -0800
Subject: [PATCH] feat: Add a review step for FAQs generated from conversations
 before using it (#10693)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR introduces a review step for generated FAQs, allowing a human to
validate and approve them before use in customer interactions. While
hallucinations are minimal, this step ensures accurate and reliable FAQs
for Captain to use during LLM calls when responding to customers.

- Added a status field for the FAQ
- Allow the filter on the UI.
<img width="1072" alt="Screenshot 2025-01-15 at 6 39 26 PM"
src="https://github.com/user-attachments/assets/81dfc038-31e9-40e6-8a09-586ebc4e8384"
/>
---
 .../dashboard/api/captain/response.js         |   3 +-
 .../captain/assistant/ResponseCard.story.vue  |   2 +
 .../captain/assistant/ResponseCard.vue        |  29 ++++
 .../document/RelatedResponses.vue             |   1 +
 .../i18n/locale/en/integrations.json          |  22 ++-
 .../dashboard/captain/responses/Index.vue     | 136 +++++++++++++++++-
 ...d_status_to_captain_assistant_responses.rb |   6 +
 db/schema.rb                                  |   4 +-
 .../captain/assistant_responses_controller.rb |   6 +-
 .../app/models/captain/assistant_response.rb  |   9 ++
 .../services/captain/copilot/chat_service.rb  |   1 +
 .../captain/llm/assistant_chat_service.rb     |   1 +
 .../captain/llm/conversation_faq_service.rb   |   2 +-
 .../captain/_assistant_response.json.jbuilder |   1 +
 .../llm/conversation_faq_service_spec.rb      |   9 +-
 15 files changed, 216 insertions(+), 16 deletions(-)
 create mode 100644 db/migrate/20250116000103_add_status_to_captain_assistant_responses.rb

diff --git a/app/javascript/dashboard/api/captain/response.js b/app/javascript/dashboard/api/captain/response.js
index 0cd31fa3c..e3c42757a 100644
--- a/app/javascript/dashboard/api/captain/response.js
+++ b/app/javascript/dashboard/api/captain/response.js
@@ -6,13 +6,14 @@ class CaptainResponses extends ApiClient {
     super('captain/assistant_responses', { accountScoped: true });
   }
 
-  get({ page = 1, searchKey, assistantId, documentId } = {}) {
+  get({ page = 1, searchKey, assistantId, documentId, status } = {}) {
     return axios.get(this.url, {
       params: {
         page,
         searchKey,
         assistant_id: assistantId,
         document_id: documentId,
+        status,
       },
     });
   }
diff --git a/app/javascript/dashboard/components-next/captain/assistant/ResponseCard.story.vue b/app/javascript/dashboard/components-next/captain/assistant/ResponseCard.story.vue
index 2522c113a..27fba8b6b 100644
--- a/app/javascript/dashboard/components-next/captain/assistant/ResponseCard.story.vue
+++ b/app/javascript/dashboard/components-next/captain/assistant/ResponseCard.story.vue
@@ -9,6 +9,7 @@ const responses = [
     created_at: 1736283330,
     id: 87,
     question: 'Why is my Messenger in Chatwoot deactivated?',
+    status: 'pending',
     assistant: {
       account_id: 1,
       config: {
@@ -148,6 +149,7 @@ const responses = [
           :id="response.id"
           :question="response.question"
           :answer="response.answer"
+          :status="response.status"
           :assistant="response.assistant"
           :created-at="response.created_at"
         />
diff --git a/app/javascript/dashboard/components-next/captain/assistant/ResponseCard.vue b/app/javascript/dashboard/components-next/captain/assistant/ResponseCard.vue
index 23ef50a07..b23753441 100644
--- a/app/javascript/dashboard/components-next/captain/assistant/ResponseCard.vue
+++ b/app/javascript/dashboard/components-next/captain/assistant/ResponseCard.vue
@@ -25,6 +25,10 @@ const props = defineProps({
     type: Boolean,
     default: false,
   },
+  status: {
+    type: String,
+    default: 'approved',
+  },
   assistant: {
     type: Object,
     default: () => ({}),
@@ -45,7 +49,22 @@ const { t } = useI18n();
 
 const [showActionsDropdown, toggleDropdown] = useToggle();
 
+const statusAction = computed(() => {
+  if (props.status === 'pending') {
+    return [
+      {
+        label: t('CAPTAIN.RESPONSES.OPTIONS.APPROVE'),
+        value: 'approve',
+        action: 'approve',
+        icon: 'i-lucide-circle-check-big',
+      },
+    ];
+  }
+  return [];
+});
+
 const menuItems = computed(() => [
+  ...statusAction.value,
   {
     label: t('CAPTAIN.RESPONSES.OPTIONS.EDIT_RESPONSE'),
     value: 'edit',
@@ -107,6 +126,16 @@ const handleAssistantAction = ({ action, value }) => {
         <i class="i-woot-captain" />
         {{ assistant?.name || '' }}
       </span>
+      <div
+        v-if="status !== 'approved'"
+        class="shrink-0 text-sm text-n-slate-11 line-clamp-1 inline-flex items-center gap-1 ml-3"
+      >
+        <i
+          class="i-ph-stack text-base"
+          :title="t('CAPTAIN.RESPONSES.STATUS.TITLE')"
+        />
+        {{ t(`CAPTAIN.RESPONSES.STATUS.${status.toUpperCase()}`) }}
+      </div>
       <div
         class="shrink-0 text-sm text-n-slate-11 line-clamp-1 inline-flex items-center gap-1 ml-3"
       >
diff --git a/app/javascript/dashboard/components-next/captain/pageComponents/document/RelatedResponses.vue b/app/javascript/dashboard/components-next/captain/pageComponents/document/RelatedResponses.vue
index 34c138eb4..6e00eda9b 100644
--- a/app/javascript/dashboard/components-next/captain/pageComponents/document/RelatedResponses.vue
+++ b/app/javascript/dashboard/components-next/captain/pageComponents/document/RelatedResponses.vue
@@ -57,6 +57,7 @@ defineExpose({ dialogRef });
         :id="response.id"
         :key="response.id"
         :question="response.question"
+        :status="response.status"
         :answer="response.answer"
         :assistant="response.assistant"
         :created-at="response.created_at"
diff --git a/app/javascript/dashboard/i18n/locale/en/integrations.json b/app/javascript/dashboard/i18n/locale/en/integrations.json
index ba1b73658..0157d5571 100644
--- a/app/javascript/dashboard/i18n/locale/en/integrations.json
+++ b/app/javascript/dashboard/i18n/locale/en/integrations.json
@@ -347,7 +347,7 @@
         },
         "FEATURES": {
           "TITLE": "Features",
-          "ALLOW_CONVERSATION_FAQS": "Generate responses from resolved conversations",
+          "ALLOW_CONVERSATION_FAQS": "Generate FAQs from resolved conversations",
           "ALLOW_MEMORIES": "Capture key details as memories from customer interactions."
         }
       },
@@ -366,8 +366,8 @@
       "HEADER": "Documents",
       "ADD_NEW": "Create a new document",
       "RELATED_RESPONSES": {
-        "TITLE": "Related Responses",
-        "DESCRIPTION": "These responses are generated directly from the document."
+        "TITLE": "Related FAQs",
+        "DESCRIPTION": "These FAQs are generated directly from the document."
       },
       "FORM_DESCRIPTION": "Enter the URL of the document to add it as a knowledge source and choose the assistant to associate it with.",
       "CREATE": {
@@ -410,6 +410,17 @@
         "SUCCESS_MESSAGE": "FAQ deleted successfully",
         "ERROR_MESSAGE": "There was an error deleting the FAQ, please try again."
       },
+      "FILTER" :{
+        "ASSISTANT": "Assistant: {selected}",
+        "STATUS": "Status: {selected}",
+        "ALL_ASSISTANTS": "All"
+      },
+      "STATUS": {
+        "TITLE": "Status",
+        "PENDING": "Pending",
+        "APPROVED": "Approved",
+        "ALL": "All"
+      },
       "FORM_DESCRIPTION": "Add a question and its corresponding answer to the knowledge base and select the assistant it should be associated with.",
       "CREATE": {
         "TITLE": "Add an FAQ",
@@ -437,10 +448,11 @@
       "EDIT": {
         "TITLE": "Update the FAQ",
         "SUCCESS_MESSAGE": "The FAQ has been successfully updated",
-        "ERROR_MESSAGE": "There was an error updating the FAQ, please try again."
+        "ERROR_MESSAGE": "There was an error updating the FAQ, please try again",
+        "APPROVE_SUCCESS_MESSAGE": "The FAQ was marked as approved"
       },
-
       "OPTIONS": {
+        "APPROVE": "Mark as approved",
         "EDIT_RESPONSE": "Edit FAQ",
         "DELETE_RESPONSE": "Delete FAQ"
       }
diff --git a/app/javascript/dashboard/routes/dashboard/captain/responses/Index.vue b/app/javascript/dashboard/routes/dashboard/captain/responses/Index.vue
index 9ce72d367..df43f7395 100644
--- a/app/javascript/dashboard/routes/dashboard/captain/responses/Index.vue
+++ b/app/javascript/dashboard/routes/dashboard/captain/responses/Index.vue
@@ -1,6 +1,11 @@
 <script setup>
 import { computed, onMounted, ref, nextTick } from 'vue';
 import { useMapGetter, useStore } from 'dashboard/composables/store';
+import { useAlert } from 'dashboard/composables';
+import { useI18n } from 'vue-i18n';
+import { OnClickOutside } from '@vueuse/components';
+import Button from 'dashboard/components-next/button/Button.vue';
+import DropdownMenu from 'dashboard/components-next/dropdown-menu/DropdownMenu.vue';
 
 import DeleteDialog from 'dashboard/components-next/captain/pageComponents/DeleteDialog.vue';
 import PageLayout from 'dashboard/components-next/captain/PageLayout.vue';
@@ -16,13 +21,75 @@ const isFetching = computed(() => uiFlags.value.fetchingList);
 
 const selectedResponse = ref(null);
 const deleteDialog = ref(null);
+
+const selectedStatus = ref('all');
+const selectedAssistant = ref('all');
 const dialogType = ref('');
+const { t } = useI18n();
+
+const createDialog = ref(null);
+
+const isStatusFilterOpen = ref(false);
+const isAssistantFilterOpen = ref(false);
+
+const statusOptions = computed(() =>
+  ['all', 'pending', 'approved'].map(key => ({
+    label: t(`CAPTAIN.RESPONSES.STATUS.${key.toUpperCase()}`),
+    value: key,
+    action: 'filter',
+  }))
+);
+
+const selectedStatusLabel = computed(() => {
+  const status = statusOptions.value.find(
+    option => option.value === selectedStatus.value
+  );
+  return t('CAPTAIN.RESPONSES.FILTER.STATUS', {
+    selected: status ? status.label : '',
+  });
+});
+
+const assistants = useMapGetter('captainAssistants/getRecords');
+const assistantOptions = computed(() => [
+  {
+    label: t(`CAPTAIN.RESPONSES.FILTER.ALL_ASSISTANTS`),
+    value: 'all',
+    action: 'filter',
+  },
+  ...assistants.value.map(assistant => ({
+    value: assistant.id,
+    label: assistant.name,
+    action: 'filter',
+  })),
+]);
+
+const selectedAssistantLabel = computed(() => {
+  const assistant = assistantOptions.value.find(
+    option => option.value === selectedAssistant.value
+  );
+  return t('CAPTAIN.RESPONSES.FILTER.ASSISTANT', {
+    selected: assistant ? assistant.label : '',
+  });
+});
 
 const handleDelete = () => {
   deleteDialog.value.dialogRef.open();
 };
-
-const createDialog = ref(null);
+const handleAccept = async () => {
+  try {
+    await store.dispatch('captainResponses/update', {
+      id: selectedResponse.value.id,
+      status: 'approved',
+    });
+    useAlert(t(`CAPTAIN.RESPONSES.EDIT.APPROVE_SUCCESS_MESSAGE`));
+  } catch (error) {
+    const errorMessage =
+      error?.message || t(`CAPTAIN.RESPONSES.EDIT.ERROR_MESSAGE`);
+    useAlert(errorMessage);
+  } finally {
+    selectedResponse.value = null;
+  }
+};
 
 const handleCreate = () => {
   dialogType.value = 'create';
@@ -43,6 +110,9 @@ const handleAction = ({ action, id }) => {
     if (action === 'edit') {
       handleEdit();
     }
+    if (action === 'approve') {
+      handleAccept();
+    }
   });
 };
 
@@ -52,11 +122,30 @@ const handleCreateClose = () => {
 };
 
 const fetchResponses = (page = 1) => {
-  store.dispatch('captainResponses/get', { page });
+  const filterParams = {};
+  if (selectedStatus.value !== 'all') {
+    filterParams.status = selectedStatus.value;
+  }
+  if (selectedAssistant.value !== 'all') {
+    filterParams.assistantId = selectedAssistant.value;
+  }
+  store.dispatch('captainResponses/get', { page, ...filterParams });
 };
 
 const onPageChange = page => fetchResponses(page);
 
+const handleStatusFilterChange = ({ value }) => {
+  selectedStatus.value = value;
+  isStatusFilterOpen.value = false;
+  fetchResponses();
+};
+
+const handleAssistantFilterChange = ({ value }) => {
+  selectedAssistant.value = value;
+  isAssistantFilterOpen.value = false;
+  fetchResponses();
+};
+
 onMounted(() => {
   store.dispatch('captainAssistants/get');
   fetchResponses();
@@ -73,12 +162,52 @@ onMounted(() => {
     @update:current-page="onPageChange"
     @click="handleCreate"
   >
+    <div v-if="!isFetching" class="mb-4 -mt-3 flex gap-3">
+      <OnClickOutside @trigger="isStatusFilterOpen = false">
+        <Button
+          :label="selectedStatusLabel"
+          icon="i-lucide-chevron-down"
+          size="sm"
+          color="slate"
+          trailing-icon
+          class="max-w-48"
+          @click="isStatusFilterOpen = !isStatusFilterOpen"
+        />
+
+        <DropdownMenu
+          v-if="isStatusFilterOpen"
+          :menu-items="statusOptions"
+          class="mt-2"
+          @action="handleStatusFilterChange"
+        />
+      </OnClickOutside>
+
+      <OnClickOutside @trigger="isAssistantFilterOpen = false">
+        <Button
+          :label="selectedAssistantLabel"
+          icon="i-lucide-chevron-down"
+          size="sm"
+          color="slate"
+          trailing-icon
+          class="max-w-48"
+          @click="isAssistantFilterOpen = !isAssistantFilterOpen"
+        />
+
+        <DropdownMenu
+          v-if="isAssistantFilterOpen"
+          :menu-items="assistantOptions"
+          class="mt-2"
+          @action="handleAssistantFilterChange"
+        />
+      </OnClickOutside>
+    </div>
     <div
       v-if="isFetching"
       class="flex items-center justify-center py-10 text-n-slate-11"
     >
       <Spinner />
     </div>
+
     <div v-else-if="responses.length" class="flex flex-col gap-4">
       <ResponseCard
         v-for="response in responses"
@@ -87,6 +216,7 @@ onMounted(() => {
         :question="response.question"
         :answer="response.answer"
         :assistant="response.assistant"
+        :status="response.status"
         :created-at="response.created_at"
         :updated-at="response.updated_at"
         @action="handleAction"
diff --git a/db/migrate/20250116000103_add_status_to_captain_assistant_responses.rb b/db/migrate/20250116000103_add_status_to_captain_assistant_responses.rb
new file mode 100644
index 000000000..c4745b36d
--- /dev/null
+++ b/db/migrate/20250116000103_add_status_to_captain_assistant_responses.rb
@@ -0,0 +1,6 @@
+class AddStatusToCaptainAssistantResponses < ActiveRecord::Migration[7.0]
+  def change
+    add_column :captain_assistant_responses, :status, :integer, default: 1, null: false
+    add_index :captain_assistant_responses, :status
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 89ebdb494..81ca33404 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2025_01_08_211541) do
+ActiveRecord::Schema[7.0].define(version: 2025_01_16_000103) do
   # These extensions should be enabled to support this database
   enable_extension "pg_stat_statements"
   enable_extension "pg_trgm"
@@ -254,10 +254,12 @@ ActiveRecord::Schema[7.0].define(version: 2025_01_08_211541) do
     t.bigint "account_id", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.integer "status", default: 1, null: false
     t.index ["account_id"], name: "index_captain_assistant_responses_on_account_id"
     t.index ["assistant_id"], name: "index_captain_assistant_responses_on_assistant_id"
     t.index ["document_id"], name: "index_captain_assistant_responses_on_document_id"
     t.index ["embedding"], name: "vector_idx_knowledge_entries_embedding", using: :ivfflat
+    t.index ["status"], name: "index_captain_assistant_responses_on_status"
   end
 
   create_table "captain_assistants", force: :cascade do |t|
diff --git a/enterprise/app/controllers/api/v1/accounts/captain/assistant_responses_controller.rb b/enterprise/app/controllers/api/v1/accounts/captain/assistant_responses_controller.rb
index 3bec446e9..eec80b856 100644
--- a/enterprise/app/controllers/api/v1/accounts/captain/assistant_responses_controller.rb
+++ b/enterprise/app/controllers/api/v1/accounts/captain/assistant_responses_controller.rb
@@ -13,6 +13,7 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
     base_query = @responses
     base_query = base_query.where(assistant_id: permitted_params[:assistant_id]) if permitted_params[:assistant_id].present?
     base_query = base_query.where(document_id: permitted_params[:document_id]) if permitted_params[:document_id].present?
+    base_query = base_query.where(status: permitted_params[:status]) if permitted_params[:status].present?
 
     @responses_count = base_query.count
 
@@ -54,7 +55,7 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
   end
 
   def permitted_params
-    params.permit(:id, :assistant_id, :page, :document_id, :account_id)
+    params.permit(:id, :assistant_id, :page, :document_id, :account_id, :status)
   end
 
   def response_params
@@ -62,7 +63,8 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
       :question,
       :answer,
       :document_id,
-      :assistant_id
+      :assistant_id,
+      :status
     )
   end
 end
diff --git a/enterprise/app/models/captain/assistant_response.rb b/enterprise/app/models/captain/assistant_response.rb
index f7dfe4f6c..3caf4f246 100644
--- a/enterprise/app/models/captain/assistant_response.rb
+++ b/enterprise/app/models/captain/assistant_response.rb
@@ -6,6 +6,7 @@
 #  answer       :text             not null
 #  embedding    :vector(1536)
 #  question     :string           not null
+#  status       :integer          default("approved"), not null
 #  created_at   :datetime         not null
 #  updated_at   :datetime         not null
 #  account_id   :bigint           not null
@@ -17,6 +18,7 @@
 #  index_captain_assistant_responses_on_account_id    (account_id)
 #  index_captain_assistant_responses_on_assistant_id  (assistant_id)
 #  index_captain_assistant_responses_on_document_id   (document_id)
+#  index_captain_assistant_responses_on_status        (status)
 #  vector_idx_knowledge_entries_embedding             (embedding) USING ivfflat
 #
 class Captain::AssistantResponse < ApplicationRecord
@@ -31,6 +33,7 @@ class Captain::AssistantResponse < ApplicationRecord
   validates :answer, presence: true
 
   before_validation :ensure_account
+  before_validation :ensure_status
   after_commit :update_response_embedding
 
   scope :ordered, -> { order(created_at: :desc) }
@@ -38,6 +41,8 @@ class Captain::AssistantResponse < ApplicationRecord
   scope :by_assistant, ->(assistant_id) { where(assistant_id: assistant_id) }
   scope :with_document, ->(document_id) { where(document_id: document_id) }
 
+  enum status: { pending: 0, approved: 1 }
+
   def self.search(query)
     embedding = Captain::Llm::EmbeddingService.new.get_embedding(query)
     nearest_neighbors(:embedding, embedding, distance: 'cosine').limit(5)
@@ -45,6 +50,10 @@ class Captain::AssistantResponse < ApplicationRecord
 
   private
 
+  def ensure_status
+    self.status ||= :approved
+  end
+
   def ensure_account
     self.account = assistant&.account
   end
diff --git a/enterprise/app/services/captain/copilot/chat_service.rb b/enterprise/app/services/captain/copilot/chat_service.rb
index 3c117aeb6..e652456ea 100644
--- a/enterprise/app/services/captain/copilot/chat_service.rb
+++ b/enterprise/app/services/captain/copilot/chat_service.rb
@@ -66,6 +66,7 @@ class Captain::Copilot::ChatService
       assistant = Captain::Assistant.find(memory[:assistant_id])
       assistant
         .responses
+        .approved
         .search(inputs['search_query'])
         .map do |response|
         "\n\nQuestion: #{response[:question]}\nAnswer: #{response[:answer]}"
diff --git a/enterprise/app/services/captain/llm/assistant_chat_service.rb b/enterprise/app/services/captain/llm/assistant_chat_service.rb
index 434a7d922..6a2cad6aa 100644
--- a/enterprise/app/services/captain/llm/assistant_chat_service.rb
+++ b/enterprise/app/services/captain/llm/assistant_chat_service.rb
@@ -84,6 +84,7 @@ class Captain::Llm::AssistantChatService < Captain::Llm::BaseOpenAiService
   def fetch_documentation(query)
     @assistant
       .responses
+      .approved
       .search(query)
       .map { |response| format_response(response) }.join
   end
diff --git a/enterprise/app/services/captain/llm/conversation_faq_service.rb b/enterprise/app/services/captain/llm/conversation_faq_service.rb
index 93a78d8ed..2953b8840 100644
--- a/enterprise/app/services/captain/llm/conversation_faq_service.rb
+++ b/enterprise/app/services/captain/llm/conversation_faq_service.rb
@@ -50,7 +50,7 @@ class Captain::Llm::ConversationFaqService < Captain::Llm::BaseOpenAiService
 
   def save_new_faqs(faqs)
     faqs.map do |faq|
-      @assistant.responses.create!(question: faq['question'], answer: faq['answer'])
+      @assistant.responses.create!(question: faq['question'], answer: faq['answer'], status: 'pending')
     end
   end
 
diff --git a/enterprise/app/views/api/v1/models/captain/_assistant_response.json.jbuilder b/enterprise/app/views/api/v1/models/captain/_assistant_response.json.jbuilder
index f8de9e646..216c57fb4 100644
--- a/enterprise/app/views/api/v1/models/captain/_assistant_response.json.jbuilder
+++ b/enterprise/app/views/api/v1/models/captain/_assistant_response.json.jbuilder
@@ -14,3 +14,4 @@ end
 json.id resource.id
 json.question resource.question
 json.updated_at resource.updated_at.to_i
+json.status resource.status
diff --git a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb
index a06c6addf..00446acb0 100644
--- a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb
+++ b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb
@@ -48,9 +48,12 @@ RSpec.describe Captain::Llm::ConversationFaqService do
 
       it 'saves the correct FAQ content' do
         service.generate_and_deduplicate
-        expect(captain_assistant.responses.pluck(:question,
-                                                 :answer)).to contain_exactly(['What is the purpose?', 'To help users.'],
-                                                                              ['How does it work?', 'Through AI.'])
+        expect(
+          captain_assistant.responses.pluck(:question, :answer, :status)
+        ).to contain_exactly(
+          ['What is the purpose?', 'To help users.', 'pending'],
+          ['How does it work?', 'Through AI.', 'pending']
+        )
       end
     end