feat: Response Bot using GPT and Webpage Sources (#7518)

This commit introduces the ability to associate response sources to an inbox, allowing external webpages to be parsed by Chatwoot. The parsed data is converted into embeddings for use with GPT models when managing customer queries.

The implementation relies on the `pgvector` extension for PostgreSQL. Database migrations related to this feature are handled separately by `Features::ResponseBotService`. A future update will integrate these migrations into the default rails migrations, once compatibility with Postgres extensions across all self-hosted installation options is confirmed.

Additionally, a new GitHub action has been added to the CI pipeline to ensure the execution of specs related to this feature.

enterprise/app/models/enterprise/concerns/account.rb

@@ -0,0 +1,15 @@
+module Enterprise::Concerns::Account
+  extend ActiveSupport::Concern
+
+  included do
+    has_many :sla_policies, dependent: :destroy_async
+
+    def self.add_response_related_associations
+      has_many :response_sources, dependent: :destroy_async
+      has_many :response_documents, dependent: :destroy_async
+      has_many :responses, dependent: :destroy_async
+    end
+
+    add_response_related_associations if Features::ResponseBotService.new.vector_extension_enabled?
+  end
+end

enterprise/app/models/enterprise/concerns/inbox.rb

@@ -0,0 +1,13 @@
+module Enterprise::Concerns::Inbox
+  extend ActiveSupport::Concern
+
+  included do
+    def self.add_response_related_associations
+      has_many :response_sources, dependent: :destroy_async
+      has_many :response_documents, dependent: :destroy_async
+      has_many :responses, dependent: :destroy_async
+    end
+
+    add_response_related_associations if Features::ResponseBotService.new.vector_extension_enabled?
+  end
+end

enterprise/app/models/enterprise/enterprise_account_concern.rb

@@ -1,7 +0,0 @@
-module Enterprise::EnterpriseAccountConcern
-  extend ActiveSupport::Concern
-
-  included do
-    has_many :sla_policies, dependent: :destroy_async
-  end
-end

enterprise/app/models/enterprise/inbox.rb

@@ -5,6 +5,19 @@ module Enterprise::Inbox
     super - overloaded_agent_ids
   end
 
+  def get_responses(query)
+    embedding = Openai::EmbeddingsService.new.get_embedding(query)
+    responses.nearest_neighbors(:embedding, embedding, distance: 'cosine').first(5)
+  end
+
+  def active_bot?
+    super || response_bot_enabled?
+  end
+
+  def response_bot_enabled?
+    account.feature_enabled?('response_bot') && response_sources.any?
+  end
+
   private
 
   def get_agent_ids_over_assignment_limit(limit)

enterprise/app/models/response.rb

@@ -0,0 +1,36 @@
+# == Schema Information
+#
+# Table name: responses
+#
+#  id                   :bigint           not null, primary key
+#  answer               :text             not null
+#  embedding            :vector(1536)
+#  question             :string           not null
+#  created_at           :datetime         not null
+#  updated_at           :datetime         not null
+#  account_id           :bigint           not null
+#  response_document_id :bigint
+#
+# Indexes
+#
+#  index_responses_on_embedding             (embedding) USING ivfflat
+#  index_responses_on_response_document_id  (response_document_id)
+#
+class Response < ApplicationRecord
+  belongs_to :response_document
+  belongs_to :account
+  has_neighbors :embedding, normalize: true
+
+  before_save :update_response_embedding
+
+  def self.search(query)
+    embedding = Openai::EmbeddingsService.new.get_embedding(query)
+    nearest_neighbors(:embedding, embedding, distance: 'cosine').first(5)
+  end
+
+  private
+
+  def update_response_embedding
+    self.embedding = Openai::EmbeddingsService.new.get_embedding("#{question}: #{answer}")
+  end
+end

enterprise/app/models/response_document.rb

@@ -0,0 +1,46 @@
+# == Schema Information
+#
+# Table name: response_documents
+#
+#  id                 :bigint           not null, primary key
+#  content            :text
+#  document_link      :string
+#  document_type      :string
+#  created_at         :datetime         not null
+#  updated_at         :datetime         not null
+#  account_id         :bigint           not null
+#  document_id        :bigint
+#  response_source_id :bigint           not null
+#
+# Indexes
+#
+#  index_response_documents_on_document            (document_type,document_id)
+#  index_response_documents_on_response_source_id  (response_source_id)
+#
+class ResponseDocument < ApplicationRecord
+  has_many :responses, dependent: :destroy
+  belongs_to :account
+  belongs_to :response_source
+
+  before_validation :set_account
+  after_create :ensure_content
+  after_update :handle_content_change
+
+  private
+
+  def set_account
+    self.account = response_source.account
+  end
+
+  def ensure_content
+    return unless content.nil?
+
+    ResponseDocumentContentJob.perform_later(self)
+  end
+
+  def handle_content_change
+    return unless saved_change_to_content? && content.present?
+
+    ResponseBuilderJob.perform_later(self)
+  end
+end

enterprise/app/models/response_source.rb

@@ -0,0 +1,28 @@
+# == Schema Information
+#
+# Table name: response_sources
+#
+#  id                :bigint           not null, primary key
+#  name              :string           not null
+#  source_link       :string
+#  source_model_type :string
+#  source_type       :integer          default("external"), not null
+#  created_at        :datetime         not null
+#  updated_at        :datetime         not null
+#  account_id        :bigint           not null
+#  inbox_id          :bigint           not null
+#  source_model_id   :bigint
+#
+# Indexes
+#
+#  index_response_sources_on_source_model  (source_model_type,source_model_id)
+#
+class ResponseSource < ApplicationRecord
+  enum source_type: { external: 0, kbase: 1, inbox: 2 }
+  belongs_to :account
+  belongs_to :inbox
+  has_many :response_documents, dependent: :destroy
+  has_many :responses, through: :response_documents
+
+  accepts_nested_attributes_for :response_documents
+end