feat: Add support for the references in FAQs (#10699)

Currently, it’s unclear whether an FAQ item is generated from a
document, derived from a conversation, or added manually.

This PR resolves the issue by providing visibility into the source of
each FAQ. Users can now see whether an FAQ was generated or manually
added and, if applicable, by whom.

- Move the document_id to a polymorphic relation (documentable).
- Updated the APIs to accommodate the change.
- Update the service to add corresponding references. 
- Updated the specs.

<img width="1007" alt="Screenshot 2025-01-15 at 11 27 56 PM"
src="https://github.com/user-attachments/assets/7d58f798-19c0-4407-b3e2-748a919d14af"
/>

---------

Co-authored-by: Sivin Varghese <64252451+iamsivin@users.noreply.github.com>

enterprise/app/controllers/api/v1/accounts/captain/assistant_responses_controller.rb

@@ -12,7 +12,14 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
   def index
     base_query = @responses
     base_query = base_query.where(assistant_id: permitted_params[:assistant_id]) if permitted_params[:assistant_id].present?
-    base_query = base_query.where(document_id: permitted_params[:document_id]) if permitted_params[:document_id].present?
+
+    if permitted_params[:document_id].present?
+      base_query = base_query.where(
+        documentable_id: permitted_params[:document_id],
+        documentable_type: 'Captain::Document'
+      )
+    end
+
     base_query = base_query.where(status: permitted_params[:status]) if permitted_params[:status].present?
 
     @responses_count = base_query.count
@@ -24,6 +31,7 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
 
   def create
     @response = Current.account.captain_assistant_responses.new(response_params)
+    @response.documentable = Current.user
     @response.save!
   end
 
@@ -43,7 +51,7 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
   end
 
   def set_responses
-    @responses = Current.account.captain_assistant_responses.includes(:assistant, :document).ordered
+    @responses = Current.account.captain_assistant_responses.includes(:assistant, :documentable).ordered
   end
 
   def set_response
@@ -62,7 +70,6 @@ class Api::V1::Accounts::Captain::AssistantResponsesController < Api::V1::Accoun
     params.require(:assistant_response).permit(
       :question,
       :answer,
-      :document_id,
       :assistant_id,
       :status
     )

enterprise/app/jobs/captain/documents/response_builder_job.rb

@@ -21,7 +21,7 @@ class Captain::Documents::ResponseBuilderJob < ApplicationJob
       question: faq['question'],
       answer: faq['answer'],
       assistant: document.assistant,
-      document: document
+      documentable: document
     )
   rescue ActiveRecord::RecordInvalid => e
     Rails.logger.error "Error in creating response document: #{e.message}"

enterprise/app/models/captain/assistant_response.rb

@@ -2,22 +2,23 @@
 #
 # Table name: captain_assistant_responses
 #
-#  id           :bigint           not null, primary key
-#  answer       :text             not null
-#  embedding    :vector(1536)
-#  question     :string           not null
-#  status       :integer          default("approved"), not null
-#  created_at   :datetime         not null
-#  updated_at   :datetime         not null
-#  account_id   :bigint           not null
-#  assistant_id :bigint           not null
-#  document_id  :bigint
+#  id                :bigint           not null, primary key
+#  answer            :text             not null
+#  documentable_type :string
+#  embedding         :vector(1536)
+#  question          :string           not null
+#  status            :integer          default("approved"), not null
+#  created_at        :datetime         not null
+#  updated_at        :datetime         not null
+#  account_id        :bigint           not null
+#  assistant_id      :bigint           not null
+#  documentable_id   :bigint
 #
 # Indexes
 #
+#  idx_cap_asst_resp_on_documentable                  (documentable_id,documentable_type)
 #  index_captain_assistant_responses_on_account_id    (account_id)
 #  index_captain_assistant_responses_on_assistant_id  (assistant_id)
-#  index_captain_assistant_responses_on_document_id   (document_id)
 #  index_captain_assistant_responses_on_status        (status)
 #  vector_idx_knowledge_entries_embedding             (embedding) USING ivfflat
 #
@@ -26,7 +27,7 @@ class Captain::AssistantResponse < ApplicationRecord
 
   belongs_to :assistant, class_name: 'Captain::Assistant'
   belongs_to :account
-  belongs_to :document, optional: true, class_name: 'Captain::Document'
+  belongs_to :documentable, polymorphic: true, optional: true
   has_neighbors :embedding, normalize: true
 
   validates :question, presence: true

enterprise/app/models/captain/document.rb

@@ -23,7 +23,7 @@ class Captain::Document < ApplicationRecord
   self.table_name = 'captain_documents'
 
   belongs_to :assistant, class_name: 'Captain::Assistant'
-  has_many :responses, class_name: 'Captain::AssistantResponse', dependent: :destroy
+  has_many :responses, class_name: 'Captain::AssistantResponse', dependent: :destroy, as: :documentable
   belongs_to :account
 
   validates :external_link, presence: true

enterprise/app/models/enterprise/concerns/conversation.rb

@@ -5,6 +5,7 @@ module Enterprise::Concerns::Conversation
     belongs_to :sla_policy, optional: true
     has_one :applied_sla, dependent: :destroy_async
     has_many :sla_events, dependent: :destroy_async
+    has_many :captain_responses, class_name: 'Captain::AssistantResponse', dependent: :nullify, as: :documentable
     before_validation :validate_sla_policy, if: -> { sla_policy_id_changed? }
     around_save :ensure_applied_sla_is_created, if: -> { sla_policy_id_changed? }
   end

enterprise/app/models/enterprise/concerns/user.rb

@@ -3,6 +3,8 @@ module Enterprise::Concerns::User
 
   included do
     before_validation :ensure_installation_pricing_plan_quantity, on: :create
+
+    has_many :captain_responses, class_name: 'Captain::AssistantResponse', dependent: :nullify, as: :documentable
   end
 
   def ensure_installation_pricing_plan_quantity

enterprise/app/services/captain/llm/conversation_faq_service.rb

@@ -4,6 +4,7 @@ class Captain::Llm::ConversationFaqService < Captain::Llm::BaseOpenAiService
   def initialize(assistant, conversation, model = DEFAULT_MODEL)
     super()
     @assistant = assistant
+    @conversation = conversation
     @content = conversation.to_llm_text
     @model = model
   end
@@ -19,7 +20,7 @@ class Captain::Llm::ConversationFaqService < Captain::Llm::BaseOpenAiService
 
   private
 
-  attr_reader :content
+  attr_reader :content, :conversation, :assistant
 
   def find_and_separate_duplicates(faqs)
     duplicate_faqs = []
@@ -41,7 +42,7 @@ class Captain::Llm::ConversationFaqService < Captain::Llm::BaseOpenAiService
   end
 
   def find_similar_faqs(embedding)
-    similar_faqs = @assistant
+    similar_faqs = assistant
                    .responses
                    .nearest_neighbors(:embedding, embedding, distance: 'cosine')
     Rails.logger.debug(similar_faqs.map { |faq| [faq.question, faq.neighbor_distance] })
@@ -50,7 +51,12 @@ class Captain::Llm::ConversationFaqService < Captain::Llm::BaseOpenAiService
 
   def save_new_faqs(faqs)
     faqs.map do |faq|
-      @assistant.responses.create!(question: faq['question'], answer: faq['answer'], status: 'pending')
+      assistant.responses.create!(
+        question: faq['question'],
+        answer: faq['answer'],
+        status: 'pending',
+        documentable: conversation
+      )
     end
   end
 

enterprise/app/views/api/v1/models/captain/_assistant_response.json.jbuilder

@@ -4,13 +4,27 @@ json.assistant do
   json.partial! 'api/v1/models/captain/assistant', formats: [:json], resource: resource.assistant
 end
 json.created_at resource.created_at.to_i
-if resource.document
-  json.document do
-    json.id resource.document.id
-    json.external_link resource.document.external_link
-    json.name resource.document.name
+
+if resource.documentable
+  json.documentable do
+    json.type resource.documentable_type
+
+    case resource.documentable_type
+    when 'Captain::Document'
+      json.id resource.documentable.id
+      json.external_link resource.documentable.external_link
+      json.name resource.documentable.name
+    when 'Conversation'
+      json.id resource.documentable.display_id
+      json.display_id resource.documentable.display_id
+    when 'User'
+      json.id resource.documentable.id
+      json.email resource.documentable.email
+      json.available_name resource.documentable.available_name
+    end
   end
 end
+
 json.id resource.id
 json.question resource.question
 json.updated_at resource.updated_at.to_i