feat(ee): Add Captain features (#10665)

Migration Guide: https://chwt.app/v4/migration

This PR imports all the work related to Captain into the EE codebase. Captain represents the AI-based features in Chatwoot and includes the following key components:

- Assistant: An assistant has a persona, the product it would be trained on. At the moment, the data at which it is trained is from websites. Future integrations on Notion documents, PDF etc. This PR enables connecting an assistant to an inbox. The assistant would run the conversation every time before transferring it to an agent.
- Copilot for Agents: When an agent is supporting a customer, we will be able to offer additional help to lookup some data or fetch information from integrations etc via copilot.
- Conversation FAQ generator: When a conversation is resolved, the Captain integration would identify questions which were not in the knowledge base.
- CRM memory: Learns from the conversations and identifies important information about the contact.

---------

Co-authored-by: Vishnu Narayanan <vishnu@chatwoot.com>
Co-authored-by: Sojan <sojan@pepalo.com>
Co-authored-by: iamsivin <iamsivin@gmail.com>
Co-authored-by: Sivin Varghese <64252451+iamsivin@users.noreply.github.com>

db/migrate/20250104200055_create_captain_tables.rb

@@ -0,0 +1,90 @@
+class CreateCaptainTables < ActiveRecord::Migration[7.0]
+  def up
+    # Post this migration, the 'vector' extension is mandatory to run the application.
+    # If the extension is not installed, the migration will raise an error.
+    setup_vector_extension
+    create_assistants
+    create_documents
+    create_assistant_responses
+    create_old_tables
+  end
+
+  def down
+    drop_table :captain_assistant_responses if table_exists?(:captain_assistant_responses)
+    drop_table :captain_documents if table_exists?(:captain_documents)
+    drop_table :captain_assistants if table_exists?(:captain_assistants)
+    drop_table :article_embeddings if table_exists?(:article_embeddings)
+
+    # We are not disabling the extension here because it might be
+    # used by other tables which are not part of this migration.
+  end
+
+  private
+
+  def setup_vector_extension
+    return if extension_enabled?('vector')
+
+    begin
+      enable_extension 'vector'
+    rescue ActiveRecord::StatementInvalid
+      raise StandardError, "Failed to enable 'vector' extension. Read more at https://chwt.app/v4/migration"
+    end
+  end
+
+  def create_assistants
+    create_table :captain_assistants do |t|
+      t.string :name, null: false
+      t.bigint :account_id, null: false
+      t.string :description
+
+      t.timestamps
+    end
+
+    add_index :captain_assistants, :account_id
+    add_index :captain_assistants, [:account_id, :name], unique: true
+  end
+
+  def create_documents
+    create_table :captain_documents do |t|
+      t.string :name, null: false
+      t.string :external_link, null: false
+      t.text :content
+      t.bigint :assistant_id, null: false
+      t.bigint :account_id, null: false
+
+      t.timestamps
+    end
+
+    add_index :captain_documents, :account_id
+    add_index :captain_documents, :assistant_id
+    add_index :captain_documents, [:assistant_id, :external_link], unique: true
+  end
+
+  def create_assistant_responses
+    create_table :captain_assistant_responses do |t|
+      t.string :question, null: false
+      t.text :answer, null: false
+      t.vector :embedding, limit: 1536
+      t.bigint :assistant_id, null: false
+      t.bigint :document_id
+      t.bigint :account_id, null: false
+
+      t.timestamps
+    end
+
+    add_index :captain_assistant_responses, :account_id
+    add_index :captain_assistant_responses, :assistant_id
+    add_index :captain_assistant_responses, :document_id
+    add_index :captain_assistant_responses, :embedding, using: :ivfflat, name: 'vector_idx_knowledge_entries_embedding', opclass: :vector_l2_ops
+  end
+
+  def create_old_tables
+    create_table :article_embeddings, if_not_exists: true do |t|
+      t.bigint :article_id, null: false
+      t.text :term, null: false
+      t.vector :embedding, limit: 1536
+      t.timestamps
+    end
+    add_index :article_embeddings, :embedding, if_not_exists: true, using: :ivfflat, opclass: :vector_l2_ops
+  end
+end

db/migrate/20250104210328_remove_robin_tables.rb

@@ -0,0 +1,10 @@
+class RemoveRobinTables < ActiveRecord::Migration[7.0]
+  def change
+    # rubocop:disable Rails/ReversibleMigration
+    drop_table :responses if table_exists?(:responses)
+    drop_table :response_sources if table_exists?(:response_sources)
+    drop_table :response_documents if table_exists?(:response_documents)
+    drop_table :inbox_response_sources if table_exists?(:inbox_response_sources)
+    # rubocop:enable Rails/ReversibleMigration
+  end
+end

db/migrate/20250105001414_add_status_to_captain_documents.rb

@@ -0,0 +1,6 @@
+class AddStatusToCaptainDocuments < ActiveRecord::Migration[7.0]
+  def change
+    add_column :captain_documents, :status, :integer, null: false, default: 0
+    add_index :captain_documents, :status
+  end
+end

db/migrate/20250105005821_remove_not_null_from_captain_documents.rb

@@ -0,0 +1,5 @@
+class RemoveNotNullFromCaptainDocuments < ActiveRecord::Migration[7.0]
+  def change
+    change_column_null :captain_documents, :name, true
+  end
+end

db/migrate/20250107030743_add_config_to_captain_assistant.rb

@@ -0,0 +1,5 @@
+class AddConfigToCaptainAssistant < ActiveRecord::Migration[7.0]
+  def change
+    add_column :captain_assistants, :config, :jsonb, default: {}, null: false
+  end
+end

db/migrate/20250108031358_create_captain_inbox.rb

@@ -0,0 +1,11 @@
+class CreateCaptainInbox < ActiveRecord::Migration[7.0]
+  def change
+    create_table :captain_inboxes do |t|
+      t.references :captain_assistant, null: false
+      t.references :inbox, null: false
+      t.timestamps
+    end
+
+    add_index :captain_inboxes, [:captain_assistant_id, :inbox_id], unique: true
+  end
+end

db/migrate/20250108211541_remove_index_from_captain_assistants.rb

@@ -0,0 +1,5 @@
+class RemoveIndexFromCaptainAssistants < ActiveRecord::Migration[7.0]
+  def change
+    remove_index :captain_assistants, [:account_id, :name], if_exists: true
+  end
+end

db/schema.rb

@@ -10,12 +10,13 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2024_12_17_041352) do
-  # These are extensions that must be enabled in order to support this database
+ActiveRecord::Schema[7.0].define(version: 2025_01_08_211541) do
+  # These extensions should be enabled to support this database
   enable_extension "pg_stat_statements"
   enable_extension "pg_trgm"
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
+  enable_extension "vector"
 
   create_table "access_tokens", force: :cascade do |t|
     t.string "owner_type"
@@ -130,6 +131,15 @@ ActiveRecord::Schema[7.0].define(version: 2024_12_17_041352) do
     t.index ["sla_policy_id"], name: "index_applied_slas_on_sla_policy_id"
   end
 
+  create_table "article_embeddings", force: :cascade do |t|
+    t.bigint "article_id", null: false
+    t.text "term", null: false
+    t.vector "embedding", limit: 1536
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["embedding"], name: "index_article_embeddings_on_embedding", using: :ivfflat
+  end
+
   create_table "articles", force: :cascade do |t|
     t.integer "account_id", null: false
     t.integer "portal_id", null: false
@@ -235,6 +245,56 @@ ActiveRecord::Schema[7.0].define(version: 2024_12_17_041352) do
     t.datetime "updated_at", precision: nil, null: false
   end
 
+  create_table "captain_assistant_responses", force: :cascade do |t|
+    t.string "question", null: false
+    t.text "answer", null: false
+    t.vector "embedding", limit: 1536
+    t.bigint "assistant_id", null: false
+    t.bigint "document_id"
+    t.bigint "account_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["account_id"], name: "index_captain_assistant_responses_on_account_id"
+    t.index ["assistant_id"], name: "index_captain_assistant_responses_on_assistant_id"
+    t.index ["document_id"], name: "index_captain_assistant_responses_on_document_id"
+    t.index ["embedding"], name: "vector_idx_knowledge_entries_embedding", using: :ivfflat
+  end
+
+  create_table "captain_assistants", force: :cascade do |t|
+    t.string "name", null: false
+    t.bigint "account_id", null: false
+    t.string "description"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.jsonb "config", default: {}, null: false
+    t.index ["account_id"], name: "index_captain_assistants_on_account_id"
+  end
+
+  create_table "captain_documents", force: :cascade do |t|
+    t.string "name"
+    t.string "external_link", null: false
+    t.text "content"
+    t.bigint "assistant_id", null: false
+    t.bigint "account_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.integer "status", default: 0, null: false
+    t.index ["account_id"], name: "index_captain_documents_on_account_id"
+    t.index ["assistant_id", "external_link"], name: "index_captain_documents_on_assistant_id_and_external_link", unique: true
+    t.index ["assistant_id"], name: "index_captain_documents_on_assistant_id"
+    t.index ["status"], name: "index_captain_documents_on_status"
+  end
+
+  create_table "captain_inboxes", force: :cascade do |t|
+    t.bigint "captain_assistant_id", null: false
+    t.bigint "inbox_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["captain_assistant_id", "inbox_id"], name: "index_captain_inboxes_on_captain_assistant_id_and_inbox_id", unique: true
+    t.index ["captain_assistant_id"], name: "index_captain_inboxes_on_captain_assistant_id"
+    t.index ["inbox_id"], name: "index_captain_inboxes_on_inbox_id"
+  end
+
   create_table "categories", force: :cascade do |t|
     t.integer "account_id", null: false
     t.integer "portal_id", null: false