feat: Add a review step for FAQs generated from conversations before using it (#10693)

This PR introduces a review step for generated FAQs, allowing a human to
validate and approve them before use in customer interactions. While
hallucinations are minimal, this step ensures accurate and reliable FAQs
for Captain to use during LLM calls when responding to customers.

- Added a status field for the FAQ
- Allow the filter on the UI.
<img width="1072" alt="Screenshot 2025-01-15 at 6 39 26 PM"
src="https://github.com/user-attachments/assets/81dfc038-31e9-40e6-8a09-586ebc4e8384"
/>

enterprise/app/controllers/api/v1/accounts/captain/assistant_responses_controller.rb

@@ -13,6 +13,7 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
     base_query = @responses
     base_query = base_query.where(assistant_id: permitted_params[:assistant_id]) if permitted_params[:assistant_id].present?
     base_query = base_query.where(document_id: permitted_params[:document_id]) if permitted_params[:document_id].present?
+    base_query = base_query.where(status: permitted_params[:status]) if permitted_params[:status].present?
 
     @responses_count = base_query.count
 
@@ -54,7 +55,7 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
   end
 
   def permitted_params
-    params.permit(:id, :assistant_id, :page, :document_id, :account_id)
+    params.permit(:id, :assistant_id, :page, :document_id, :account_id, :status)
   end
 
   def response_params
@@ -62,7 +63,8 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
       :question,
       :answer,
       :document_id,
-      :assistant_id
+      :assistant_id,
+      :status
     )
   end
 end

enterprise/app/models/captain/assistant_response.rb

@@ -6,6 +6,7 @@
 #  answer       :text             not null
 #  embedding    :vector(1536)
 #  question     :string           not null
+#  status       :integer          default("approved"), not null
 #  created_at   :datetime         not null
 #  updated_at   :datetime         not null
 #  account_id   :bigint           not null
@@ -17,6 +18,7 @@
 #  index_captain_assistant_responses_on_account_id    (account_id)
 #  index_captain_assistant_responses_on_assistant_id  (assistant_id)
 #  index_captain_assistant_responses_on_document_id   (document_id)
+#  index_captain_assistant_responses_on_status        (status)
 #  vector_idx_knowledge_entries_embedding             (embedding) USING ivfflat
 #
 class Captain::AssistantResponse < ApplicationRecord
@@ -31,6 +33,7 @@ class Captain::AssistantResponse < ApplicationRecord
   validates :answer, presence: true
 
   before_validation :ensure_account
+  before_validation :ensure_status
   after_commit :update_response_embedding
 
   scope :ordered, -> { order(created_at: :desc) }
@@ -38,6 +41,8 @@ class Captain::AssistantResponse < ApplicationRecord
   scope :by_assistant, ->(assistant_id) { where(assistant_id: assistant_id) }
   scope :with_document, ->(document_id) { where(document_id: document_id) }
 
+  enum status: { pending: 0, approved: 1 }
+
   def self.search(query)
     embedding = Captain::Llm::EmbeddingService.new.get_embedding(query)
     nearest_neighbors(:embedding, embedding, distance: 'cosine').limit(5)
@@ -45,6 +50,10 @@ class Captain::AssistantResponse < ApplicationRecord
 
   private
 
+  def ensure_status
+    self.status ||= :approved
+  end
+
   def ensure_account
     self.account = assistant&.account
   end

enterprise/app/services/captain/copilot/chat_service.rb

@@ -66,6 +66,7 @@ class Captain::Copilot::ChatService
       assistant = Captain::Assistant.find(memory[:assistant_id])
       assistant
         .responses
+        .approved
         .search(inputs['search_query'])
         .map do |response|
         "\n\nQuestion: #{response[:question]}\nAnswer: #{response[:answer]}"

enterprise/app/services/captain/llm/assistant_chat_service.rb

@@ -84,6 +84,7 @@ class Captain::Llm::AssistantChatService < Captain::Llm::BaseOpenAiService
   def fetch_documentation(query)
     @assistant
       .responses
+      .approved
       .search(query)
       .map { |response| format_response(response) }.join
   end

enterprise/app/services/captain/llm/conversation_faq_service.rb

@@ -50,7 +50,7 @@ class Captain::Llm::ConversationFaqService < Captain::Llm::BaseOpenAiService
 
   def save_new_faqs(faqs)
     faqs.map do |faq|
-      @assistant.responses.create!(question: faq['question'], answer: faq['answer'])
+      @assistant.responses.create!(question: faq['question'], answer: faq['answer'], status: 'pending')
     end
   end
 

enterprise/app/views/api/v1/models/captain/_assistant_response.json.jbuilder

@@ -14,3 +14,4 @@ end
 json.id resource.id
 json.question resource.question
 json.updated_at resource.updated_at.to_i
+json.status resource.status