chore: Refactor Response Bot Data Schema (#8011)

This PR refactors the schema we introduced in #7518 based on the feedback from production tests. Here is the change log

- Decouple Inbox association to a new table inbox_response_sources -> this lets us share the same response source between multiple inboxes
- Add a status field to responses. This ensures that, by default, responses are created in pending status. You can do quality assurance before making them active. In future, this status can be leveraged by the bot to auto-generate response questions from conversations which require a handoff
- Add response_source association to responses and remove hard dependency from response_documents. This lets users write free-form question answers based on conversations, which doesn't necessarily need a response source.

enterprise/app/models/enterprise/audit_log.rb

@@ -1,3 +1,31 @@
+# == Schema Information
+#
+# Table name: audits
+#
+#  id              :bigint           not null, primary key
+#  action          :string
+#  associated_type :string
+#  auditable_type  :string
+#  audited_changes :jsonb
+#  comment         :string
+#  remote_address  :string
+#  request_uuid    :string
+#  user_type       :string
+#  username        :string
+#  version         :integer          default(0)
+#  created_at      :datetime
+#  associated_id   :bigint
+#  auditable_id    :bigint
+#  user_id         :bigint
+#
+# Indexes
+#
+#  associated_index              (associated_type,associated_id)
+#  auditable_index               (auditable_type,auditable_id,version)
+#  index_audits_on_created_at    (created_at)
+#  index_audits_on_request_uuid  (request_uuid)
+#  user_index                    (user_id,user_type)
+#
 class Enterprise::AuditLog < Audited::Audit
   after_save :log_additional_information
 

enterprise/app/models/enterprise/concerns/inbox.rb

@@ -3,9 +3,10 @@ module Enterprise::Concerns::Inbox
 
   included do
     def self.add_response_related_associations
-      has_many :response_sources, dependent: :destroy_async
+      has_many :inbox_response_sources, dependent: :destroy_async
+      has_many :response_sources, through: :inbox_response_sources
       has_many :response_documents, through: :response_sources
-      has_many :responses, through: :response_documents
+      has_many :responses, through: :response_sources
     end
 
     add_response_related_associations if Features::ResponseBotService.new.vector_extension_enabled?

enterprise/app/models/enterprise/inbox.rb

@@ -7,7 +7,7 @@ module Enterprise::Inbox
 
   def get_responses(query)
     embedding = Openai::EmbeddingsService.new.get_embedding(query)
-    responses.nearest_neighbors(:embedding, embedding, distance: 'cosine').first(5)
+    responses.active.nearest_neighbors(:embedding, embedding, distance: 'cosine').first(5)
   end
 
   def active_bot?

enterprise/app/models/inbox_response_source.rb

@@ -0,0 +1,21 @@
+# == Schema Information
+#
+# Table name: inbox_response_sources
+#
+#  id                 :bigint           not null, primary key
+#  created_at         :datetime         not null
+#  updated_at         :datetime         not null
+#  inbox_id           :bigint           not null
+#  response_source_id :bigint           not null
+#
+# Indexes
+#
+#  index_inbox_response_sources_on_inbox_id                         (inbox_id)
+#  index_inbox_response_sources_on_inbox_id_and_response_source_id  (inbox_id,response_source_id) UNIQUE
+#  index_inbox_response_sources_on_response_source_id               (response_source_id)
+#  index_inbox_response_sources_on_response_source_id_and_inbox_id  (response_source_id,inbox_id) UNIQUE
+#
+class InboxResponseSource < ApplicationRecord
+  belongs_to :inbox
+  belongs_to :response_source
+end

enterprise/app/models/response.rb

@@ -6,10 +6,12 @@
 #  answer               :text             not null
 #  embedding            :vector(1536)
 #  question             :string           not null
+#  status               :integer          default(0)
 #  created_at           :datetime         not null
 #  updated_at           :datetime         not null
 #  account_id           :bigint           not null
 #  response_document_id :bigint
+#  response_source_id   :bigint           not null
 #
 # Indexes
 #
@@ -17,11 +19,15 @@
 #  index_responses_on_response_document_id  (response_document_id)
 #
 class Response < ApplicationRecord
-  belongs_to :response_document
+  belongs_to :response_document, optional: true
   belongs_to :account
+  belongs_to :response_source
   has_neighbors :embedding, normalize: true
 
   before_save :update_response_embedding
+  before_validation :ensure_account
+
+  enum status: { pending: 0, active: 1 }
 
   def self.search(query)
     embedding = Openai::EmbeddingsService.new.get_embedding(query)
@@ -30,6 +36,10 @@ class Response < ApplicationRecord
 
   private
 
+  def ensure_account
+    self.account = response_source.account
+  end
+
   def update_response_embedding
     self.embedding = Openai::EmbeddingsService.new.get_embedding("#{question}: #{answer}")
   end

enterprise/app/models/response_document.rb

@@ -18,7 +18,7 @@
 #  index_response_documents_on_response_source_id  (response_source_id)
 #
 class ResponseDocument < ApplicationRecord
-  has_many :responses, dependent: :destroy
+  has_many :responses, dependent: :destroy_async
   belongs_to :account
   belongs_to :response_source
 

enterprise/app/models/response_source.rb

@@ -10,7 +10,6 @@
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
 #  account_id        :bigint           not null
-#  inbox_id          :bigint           not null
 #  source_model_id   :bigint
 #
 # Indexes
@@ -19,10 +18,11 @@
 #
 class ResponseSource < ApplicationRecord
   enum source_type: { external: 0, kbase: 1, inbox: 2 }
+  has_many :inbox_response_sources, dependent: :destroy_async
+  has_many :inboxes, through: :inbox_response_sources
   belongs_to :account
-  belongs_to :inbox
-  has_many :response_documents, dependent: :destroy
-  has_many :responses, through: :response_documents
+  has_many :response_documents, dependent: :destroy_async
+  has_many :responses, dependent: :destroy_async
 
   accepts_nested_attributes_for :response_documents
 end