feat: Response Bot using GPT and Webpage Sources (#7518)

This commit introduces the ability to associate response sources to an inbox, allowing external webpages to be parsed by Chatwoot. The parsed data is converted into embeddings for use with GPT models when managing customer queries.

The implementation relies on the `pgvector` extension for PostgreSQL. Database migrations related to this feature are handled separately by `Features::ResponseBotService`. A future update will integrate these migrations into the default rails migrations, once compatibility with Postgres extensions across all self-hosted installation options is confirmed.

Additionally, a new GitHub action has been added to the CI pipeline to ensure the execution of specs related to this feature.

spec/enterprise/controllers/api/v1/accounts/response_sources_controller_spec.rb

@@ -0,0 +1,135 @@
+require 'rails_helper'
+
+RSpec.describe 'Response Sources API', type: :request do
+  let!(:account) { create(:account) }
+  let!(:admin) { create(:user, account: account, role: :administrator) }
+  let!(:inbox) { create(:inbox, account: account) }
+
+  before do
+    skip('Skipping since vector is not enabled in this environment') unless Features::ResponseBotService.new.vector_extension_enabled?
+  end
+
+  describe 'POST /api/v1/accounts/{account.id}/response_sources/parse' do
+    let(:valid_params) do
+      {
+        link: 'http://test.test'
+      }
+    end
+
+    context 'when it is an unauthenticated user' do
+      it 'returns unauthorized' do
+        post "/api/v1/accounts/#{account.id}/response_sources/parse", params: valid_params
+
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+
+    context 'when it is an authenticated user' do
+      it 'returns links in the webpage' do
+        crawler = double
+        allow(PageCrawlerService).to receive(:new).and_return(crawler)
+        allow(crawler).to receive(:page_links).and_return(['http://test.test'])
+
+        post "/api/v1/accounts/#{account.id}/response_sources/parse", headers: admin.create_new_auth_token,
+                                                                      params: valid_params
+        expect(response).to have_http_status(:success)
+        expect(response.parsed_body['links']).to eq(['http://test.test'])
+      end
+    end
+  end
+
+  describe 'POST /api/v1/accounts/{account.id}/response_sources' do
+    let(:valid_params) do
+      {
+        response_source: {
+          name: 'Test',
+          source_link: 'http://test.test',
+          inbox_id: inbox.id,
+          response_documents_attributes: [
+            { document_link: 'http://test1.test' },
+            { document_link: 'http://test2.test' }
+          ]
+        }
+      }
+    end
+
+    context 'when it is an unauthenticated user' do
+      it 'returns unauthorized' do
+        expect { post "/api/v1/accounts/#{account.id}/response_sources", params: valid_params }.not_to change(ResponseSource, :count)
+
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+
+    context 'when it is an authenticated user' do
+      it 'creates the response sources and documents' do
+        expect do
+          post "/api/v1/accounts/#{account.id}/response_sources", headers: admin.create_new_auth_token,
+                                                                  params: valid_params
+        end.to change(ResponseSource, :count).by(1)
+
+        expect(ResponseDocument.count).to eq(2)
+        expect(response).to have_http_status(:success)
+      end
+    end
+  end
+
+  describe 'POST /api/v1/accounts/{account.id}/response_sources/{response_source.id}/add_document' do
+    let!(:response_source) { create(:response_source, account: account, inbox: inbox) }
+    let(:valid_params) do
+      { document_link: 'http://test.test' }
+    end
+
+    context 'when it is an unauthenticated user' do
+      it 'returns unauthorized' do
+        expect do
+          post "/api/v1/accounts/#{account.id}/response_sources/#{response_source.id}/add_document",
+               params: valid_params
+        end.not_to change(ResponseDocument, :count)
+
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+
+    context 'when it is an authenticated user' do
+      it 'creates the response sources and documents' do
+        expect do
+          post "/api/v1/accounts/#{account.id}/response_sources/#{response_source.id}/add_document", headers: admin.create_new_auth_token,
+                                                                                                     params: valid_params
+        end.to change(ResponseDocument, :count).by(1)
+        expect(response).to have_http_status(:success)
+      end
+    end
+  end
+
+  describe 'POST /api/v1/accounts/{account.id}/response_sources/{response_source.id}/remove_document' do
+    let!(:response_source) { create(:response_source, account: account, inbox: inbox) }
+    let!(:response_document) { response_source.response_documents.create!(document_link: 'http://test.test') }
+    let(:valid_params) do
+      { document_id: response_document.id }
+    end
+
+    context 'when it is an unauthenticated user' do
+      it 'returns unauthorized' do
+        expect do
+          post "/api/v1/accounts/#{account.id}/response_sources/#{response_source.id}/remove_document",
+               params: valid_params
+        end.not_to change(ResponseDocument, :count)
+
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+
+    context 'when it is an authenticated user' do
+      it 'creates the response sources and documents' do
+        expect do
+          post "/api/v1/accounts/#{account.id}/response_sources/#{response_source.id}/remove_document", headers: admin.create_new_auth_token,
+                                                                                                        params: valid_params
+        end.to change(ResponseDocument, :count).by(-1)
+        expect(response).to have_http_status(:success)
+
+        expect { response_document.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+  end
+end