From be8205657e4ad63cbd9945ae069491119e0aa1b9 Mon Sep 17 00:00:00 2001
From: Sojan Jose <sojan@pepalo.com>
Date: Thu, 23 Jan 2025 07:44:25 +0530
Subject: [PATCH] fix: Added authentication to FireCrawl API, remove unused
 RobinAI references (#10737)

- Fixed Firecrawl webhook payloads to ensure proper data handling and
delivery.
- Removed unused Robin AI code to improve codebase cleanliness and
maintainability.
- Implement authentication for the Firecrawl endpoint to improve
security. A key is generated to secure the webhook URLs from FireCrawl.

---------

Co-authored-by: Pranav <pranavrajs@gmail.com>
---
 app/policies/response_source_policy.rb        |  17 ---
 .../inboxes/response_sources.json.jbuilder    |   3 -
 .../add_document.json.jbuilder                |   1 -
 .../response_sources/create.json.jbuilder     |   1 -
 .../remove_document.json.jbuilder             |   1 -
 .../v1/models/_response_source.json.jbuilder  |  15 --
 .../application/_navigation.html.erb          |   2 +-
 config/installation_config.yml                |   4 +
 config/routes.rb                              |   2 -
 .../super_admin/app_configs_controller.rb     |   2 +-
 .../webhooks/firecrawl_controller.rb          |  31 +++-
 .../app/dashboards/response_dashboard.rb      |  89 ------------
 .../dashboards/response_document_dashboard.rb |  84 -----------
 .../dashboards/response_source_dashboard.rb   |  86 -----------
 .../app/helpers/captain/firecrawl_helper.rb   |   9 ++
 .../app/jobs/captain/documents/crawl_job.rb   |  15 +-
 .../captain/tools/firecrawl_parser_job.rb     |   4 +-
 enterprise/app/models/captain/document.rb     |   1 +
 .../captain/tools/firecrawl_service.rb        |   8 +-
 .../response_sources/chat.html.erb            |   5 -
 .../response_sources/show.html.erb            |  71 ----------
 .../webooks/firecrawl_controller_spec.rb      | 133 ++++++++++++-----
 .../jobs/captain/documents/crawl_job_spec.rb  |  54 ++++++-
 .../tools/firecrawl_parser_job_spec.rb        |  14 +-
 .../captain/tools/firecrawl_service_spec.rb   | 134 ++++++++++++++++++
 spec/factories/inbox_response_source.rb       |   8 --
 spec/factories/response.rb                    |  11 --
 spec/factories/response_document.rb           |  10 --
 28 files changed, 345 insertions(+), 470 deletions(-)
 delete mode 100644 app/policies/response_source_policy.rb
 delete mode 100644 app/views/api/v1/accounts/inboxes/response_sources.json.jbuilder
 delete mode 100644 app/views/api/v1/accounts/response_sources/add_document.json.jbuilder
 delete mode 100644 app/views/api/v1/accounts/response_sources/create.json.jbuilder
 delete mode 100644 app/views/api/v1/accounts/response_sources/remove_document.json.jbuilder
 delete mode 100644 app/views/api/v1/models/_response_source.json.jbuilder
 delete mode 100644 enterprise/app/dashboards/response_dashboard.rb
 delete mode 100644 enterprise/app/dashboards/response_document_dashboard.rb
 delete mode 100644 enterprise/app/dashboards/response_source_dashboard.rb
 create mode 100644 enterprise/app/helpers/captain/firecrawl_helper.rb
 delete mode 100644 enterprise/app/views/super_admin/response_sources/chat.html.erb
 delete mode 100644 enterprise/app/views/super_admin/response_sources/show.html.erb
 create mode 100644 spec/enterprise/services/captain/tools/firecrawl_service_spec.rb
 delete mode 100644 spec/factories/inbox_response_source.rb
 delete mode 100644 spec/factories/response.rb
 delete mode 100644 spec/factories/response_document.rb

diff --git a/app/policies/response_source_policy.rb b/app/policies/response_source_policy.rb
deleted file mode 100644
index e6083ba22..000000000
--- a/app/policies/response_source_policy.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-class ResponseSourcePolicy < ApplicationPolicy
-  def parse?
-    @account_user.administrator?
-  end
-
-  def create?
-    @account_user.administrator?
-  end
-
-  def add_document?
-    @account_user.administrator?
-  end
-
-  def remove_document?
-    @account_user.administrator?
-  end
-end
diff --git a/app/views/api/v1/accounts/inboxes/response_sources.json.jbuilder b/app/views/api/v1/accounts/inboxes/response_sources.json.jbuilder
deleted file mode 100644
index aff6e551c..000000000
--- a/app/views/api/v1/accounts/inboxes/response_sources.json.jbuilder
+++ /dev/null
@@ -1,3 +0,0 @@
-json.array! @response_sources do |response_source|
-  json.partial! 'api/v1/models/response_source', formats: [:json], resource: response_source
-end
diff --git a/app/views/api/v1/accounts/response_sources/add_document.json.jbuilder b/app/views/api/v1/accounts/response_sources/add_document.json.jbuilder
deleted file mode 100644
index ab0af26d3..000000000
--- a/app/views/api/v1/accounts/response_sources/add_document.json.jbuilder
+++ /dev/null
@@ -1 +0,0 @@
-json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source
diff --git a/app/views/api/v1/accounts/response_sources/create.json.jbuilder b/app/views/api/v1/accounts/response_sources/create.json.jbuilder
deleted file mode 100644
index ab0af26d3..000000000
--- a/app/views/api/v1/accounts/response_sources/create.json.jbuilder
+++ /dev/null
@@ -1 +0,0 @@
-json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source
diff --git a/app/views/api/v1/accounts/response_sources/remove_document.json.jbuilder b/app/views/api/v1/accounts/response_sources/remove_document.json.jbuilder
deleted file mode 100644
index ab0af26d3..000000000
--- a/app/views/api/v1/accounts/response_sources/remove_document.json.jbuilder
+++ /dev/null
@@ -1 +0,0 @@
-json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source
diff --git a/app/views/api/v1/models/_response_source.json.jbuilder b/app/views/api/v1/models/_response_source.json.jbuilder
deleted file mode 100644
index 9753fa860..000000000
--- a/app/views/api/v1/models/_response_source.json.jbuilder
+++ /dev/null
@@ -1,15 +0,0 @@
-json.id resource.id
-json.name resource.name
-json.source_link resource.source_link
-json.source_type resource.source_type
-json.account_id resource.account_id
-json.created_at resource.created_at.to_i
-json.updated_at resource.updated_at.to_i
-json.response_documents do
-  json.array! resource.response_documents do |response_document|
-    json.id response_document.id
-    json.document_link response_document.document_link
-    json.created_at response_document.created_at.to_i
-    json.updated_at response_document.updated_at.to_i
-  end
-end
diff --git a/app/views/super_admin/application/_navigation.html.erb b/app/views/super_admin/application/_navigation.html.erb
index 13d33d682..f673c6bd1 100644
--- a/app/views/super_admin/application/_navigation.html.erb
+++ b/app/views/super_admin/application/_navigation.html.erb
@@ -32,7 +32,7 @@ as defined by the routes in the `admin/` namespace
     <ul class="my-4">
       <%= render partial: "nav_item", locals: { icon: 'icon-grid-line', url: super_admin_root_url, label: 'Dashboard' } %>
       <% Administrate::Namespace.new(namespace).resources.each do |resource| %>
-        <% next if ["account_users", "access_tokens", "installation_configs", "dashboard", "devise/sessions", "app_configs", "instance_statuses", "responses", "response_sources", "response_documents" , "settings"].include?  resource.resource %>
+        <% next if ["account_users", "access_tokens", "installation_configs", "dashboard", "devise/sessions", "app_configs", "instance_statuses", "settings"].include?  resource.resource %>
         <%= render partial: "nav_item", locals: {
             icon: sidebar_icons[resource.resource.to_sym],
             url: resource_index_route(resource),
diff --git a/config/installation_config.yml b/config/installation_config.yml
index a3b15a191..1a804b707 100644
--- a/config/installation_config.yml
+++ b/config/installation_config.yml
@@ -142,6 +142,10 @@
   display_title: 'OpenAI Model'
   description: 'The OpenAI model configured for use in Captain AI. Default: gpt-4o-mini'
   locked: false
+- name: CAPTAIN_FIRECRAWL_API_KEY
+  display_title: 'FireCrawl API Key (optional)'
+  description: 'The FireCrawl API key for the Captain AI service'
+  locked: false
 - name: CAPTAIN_CLOUD_PLAN_LIMITS
   display_title: 'Captain Cloud Plan Limits'
   description: 'The limits for the Captain AI service for different plans'
diff --git a/config/routes.rb b/config/routes.rb
index fc115f8da..ebb3804cb 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -481,8 +481,6 @@ Rails.application.routes.draw do
       end
 
       resources :access_tokens, only: [:index, :show]
-      resources :response_documents, only: [:index, :show, :new, :create, :edit, :update, :destroy]
-      resources :responses, only: [:index, :show, :new, :create, :edit, :update, :destroy]
       resources :installation_configs, only: [:index, :new, :create, :show, :edit, :update]
       resources :agent_bots, only: [:index, :new, :create, :show, :edit, :update] do
         delete :avatar, on: :member, action: :destroy_avatar
diff --git a/enterprise/app/controllers/enterprise/super_admin/app_configs_controller.rb b/enterprise/app/controllers/enterprise/super_admin/app_configs_controller.rb
index b11aab487..21617aa83 100644
--- a/enterprise/app/controllers/enterprise/super_admin/app_configs_controller.rb
+++ b/enterprise/app/controllers/enterprise/super_admin/app_configs_controller.rb
@@ -10,7 +10,7 @@ module Enterprise::SuperAdmin::AppConfigsController
     when 'internal'
       @allowed_configs = internal_config_options
     when 'captain'
-      @allowed_configs = %w[CAPTAIN_OPEN_AI_API_KEY CAPTAIN_OPEN_AI_MODEL]
+      @allowed_configs = %w[CAPTAIN_OPEN_AI_API_KEY CAPTAIN_OPEN_AI_MODEL CAPTAIN_FIRECRAWL_API_KEY]
     else
       super
     end
diff --git a/enterprise/app/controllers/enterprise/webhooks/firecrawl_controller.rb b/enterprise/app/controllers/enterprise/webhooks/firecrawl_controller.rb
index a00770123..bf64a80d7 100644
--- a/enterprise/app/controllers/enterprise/webhooks/firecrawl_controller.rb
+++ b/enterprise/app/controllers/enterprise/webhooks/firecrawl_controller.rb
@@ -1,16 +1,32 @@
 class Enterprise::Webhooks::FirecrawlController < ActionController::API
+  before_action :validate_token
+
   def process_payload
-    if crawl_page_event?
-      Captain::Tools::FirecrawlParserJob.perform_later(
-        assistant_id: permitted_params[:assistant_id],
-        payload: permitted_params[:data]
-      )
-    end
+    Captain::Tools::FirecrawlParserJob.perform_later(assistant_id: assistant.id, payload: payload) if crawl_page_event?
+
     head :ok
   end
 
   private
 
+  include Captain::FirecrawlHelper
+
+  def payload
+    permitted_params[:data]&.first&.to_h
+  end
+
+  def validate_token
+    render json: { error: 'Invalid access_token' }, status: :unauthorized if assistant_token != permitted_params[:token]
+  end
+
+  def assistant
+    @assistant ||= Captain::Assistant.find(permitted_params[:assistant_id])
+  end
+
+  def assistant_token
+    generate_firecrawl_token(assistant.id, assistant.account_id)
+  end
+
   def crawl_page_event?
     permitted_params[:type] == 'crawl.page'
   end
@@ -19,12 +35,13 @@ class Enterprise::Webhooks::FirecrawlController < ActionController::API
     params.permit(
       :type,
       :assistant_id,
+      :token,
       :success,
       :id,
       :metadata,
       :format,
       :firecrawl,
-      { data: {} }
+      data: [:markdown, { metadata: {} }]
     )
   end
 end
diff --git a/enterprise/app/dashboards/response_dashboard.rb b/enterprise/app/dashboards/response_dashboard.rb
deleted file mode 100644
index 40ba1c428..000000000
--- a/enterprise/app/dashboards/response_dashboard.rb
+++ /dev/null
@@ -1,89 +0,0 @@
-require 'administrate/base_dashboard'
-
-class ResponseDashboard < Administrate::BaseDashboard
-  # ATTRIBUTE_TYPES
-  # a hash that describes the type of each of the model's fields.
-  #
-  # Each different type represents an Administrate::Field object,
-  # which determines how the attribute is displayed
-  # on pages throughout the dashboard.
-  ATTRIBUTE_TYPES = {
-    id: Field::Number.with_options(searchable: true),
-    account: Field::BelongsToSearch.with_options(class_name: 'Account', searchable_field: [:name, :id], order: 'id DESC'),
-    response_source: Field::BelongsToSearch.with_options(class_name: 'ResponseSource', searchable_field: [:name, :id, :source_link],
-                                                         order: 'id DESC'),
-    answer: Field::Text.with_options(searchable: true),
-    question: Field::String.with_options(searchable: true),
-    status: Field::Select.with_options(searchable: false, collection: lambda { |field|
-      field.resource.class.send(field.attribute.to_s.pluralize).keys
-    }),
-    response_document: Field::BelongsToSearch.with_options(class_name: 'ResponseDocument', searchable_field: [:document_link, :content, :id],
-                                                           order: 'id DESC'),
-    created_at: Field::DateTime,
-    updated_at: Field::DateTime
-  }.freeze
-
-  # COLLECTION_ATTRIBUTES
-  # an array of attributes that will be displayed on the model's index page.
-  #
-  # By default, it's limited to four items to reduce clutter on index pages.
-  # Feel free to add, remove, or rearrange items.
-  COLLECTION_ATTRIBUTES = %i[
-    id
-    question
-    answer
-    status
-    response_document
-    response_source
-    account
-  ].freeze
-
-  # SHOW_PAGE_ATTRIBUTES
-  # an array of attributes that will be displayed on the model's show page.
-  SHOW_PAGE_ATTRIBUTES = %i[
-    id
-    status
-    question
-    answer
-    response_document
-    response_source
-    account
-    created_at
-    updated_at
-  ].freeze
-
-  # FORM_ATTRIBUTES
-  # an array of attributes that will be displayed
-  # on the model's form (`new` and `edit`) pages.
-  FORM_ATTRIBUTES = %i[
-    response_source
-    response_document
-    question
-    answer
-    status
-  ].freeze
-
-  # COLLECTION_FILTERS
-  # a hash that defines filters that can be used while searching via the search
-  # field of the dashboard.
-  #
-  # For example to add an option to search for open resources by typing "open:"
-  # in the search field:
-  #
-  #   COLLECTION_FILTERS = {
-  #     open: ->(resources) { resources.where(open: true) }
-  #   }.freeze
-  COLLECTION_FILTERS = {
-    account: ->(resources, attr) { resources.where(account_id: attr) },
-    response_source: ->(resources, attr) { resources.where(response_source_id: attr) },
-    response_document: ->(resources, attr) { resources.where(response_document_id: attr) },
-    status: ->(resources, attr) { resources.where(status: attr) }
-  }.freeze
-
-  # Overwrite this method to customize how responses are displayed
-  # across all pages of the admin dashboard.
-  #
-  def display_resource(response)
-    "Response: ##{response.id} - #{response.question}"
-  end
-end
diff --git a/enterprise/app/dashboards/response_document_dashboard.rb b/enterprise/app/dashboards/response_document_dashboard.rb
deleted file mode 100644
index fe20c842b..000000000
--- a/enterprise/app/dashboards/response_document_dashboard.rb
+++ /dev/null
@@ -1,84 +0,0 @@
-require 'administrate/base_dashboard'
-
-class ResponseDocumentDashboard < Administrate::BaseDashboard
-  # ATTRIBUTE_TYPES
-  # a hash that describes the type of each of the model's fields.
-  #
-  # Each different type represents an Administrate::Field object,
-  # which determines how the attribute is displayed
-  # on pages throughout the dashboard.
-  ATTRIBUTE_TYPES = {
-    id: Field::Number.with_options(searchable: true),
-    account: Field::BelongsToSearch.with_options(class_name: 'Account', searchable_field: [:name, :id], order: 'id DESC'),
-    content: Field::Text.with_options(searchable: true),
-    document_id: Field::Number,
-    document_link: Field::String.with_options(searchable: true),
-    document_type: Field::String,
-    response_source: Field::BelongsToSearch.with_options(class_name: 'ResponseSource', searchable_field: [:name, :id, :source_link],
-                                                         order: 'id DESC'),
-    responses: Field::HasMany,
-    created_at: Field::DateTime,
-    updated_at: Field::DateTime
-  }.freeze
-
-  # COLLECTION_ATTRIBUTES
-  # an array of attributes that will be displayed on the model's index page.
-  #
-  # By default, it's limited to four items to reduce clutter on index pages.
-  # Feel free to add, remove, or rearrange items.
-  COLLECTION_ATTRIBUTES = %i[
-    id
-    account
-    response_source
-    document_link
-  ].freeze
-
-  # SHOW_PAGE_ATTRIBUTES
-  # an array of attributes that will be displayed on the model's show page.
-  SHOW_PAGE_ATTRIBUTES = %i[
-    id
-    account
-    response_source
-    document_link
-    document_id
-    document_type
-    content
-    created_at
-    updated_at
-    responses
-  ].freeze
-
-  # FORM_ATTRIBUTES
-  # an array of attributes that will be displayed
-  # on the model's form (`new` and `edit`) pages.
-  FORM_ATTRIBUTES = %i[
-    account
-    response_source
-    document_link
-    document_id
-    document_type
-    content
-  ].freeze
-
-  # COLLECTION_FILTERS
-  # a hash that defines filters that can be used while searching via the search
-  # field of the dashboard.
-  #
-  # For example to add an option to search for open resources by typing "open:"
-  # in the search field:
-  #
-  #   COLLECTION_FILTERS = {
-  #     open: ->(resources) { resources.where(open: true) }
-  #   }.freeze
-  COLLECTION_FILTERS = {
-    account: ->(resources, attr) { resources.where(account_id: attr) },
-    response_source: ->(resources, attr) { resources.where(response_source_id: attr) }
-  }.freeze
-
-  # Overwrite this method to customize how response documents are displayed
-  # across all pages of the admin dashboard.
-  #
-  def display_resource(response_document)
-    "Document: ##{response_document.id} - #{response_document.document_link}"
-  end
-end
diff --git a/enterprise/app/dashboards/response_source_dashboard.rb b/enterprise/app/dashboards/response_source_dashboard.rb
deleted file mode 100644
index f17548643..000000000
--- a/enterprise/app/dashboards/response_source_dashboard.rb
+++ /dev/null
@@ -1,86 +0,0 @@
-require 'administrate/base_dashboard'
-
-class ResponseSourceDashboard < Administrate::BaseDashboard
-  # ATTRIBUTE_TYPES
-  # a hash that describes the type of each of the model's fields.
-  #
-  # Each different type represents an Administrate::Field object,
-  # which determines how the attribute is displayed
-  # on pages throughout the dashboard.
-  ATTRIBUTE_TYPES = {
-    id: Field::Number.with_options(searchable: true),
-    account: Field::BelongsToSearch.with_options(class_name: 'Account', searchable_field: [:name, :id], order: 'id DESC'),
-    name: Field::String.with_options(searchable: true),
-    response_documents: Field::HasMany,
-    responses: Field::HasMany,
-    source_link: Field::String.with_options(searchable: true),
-    source_model_id: Field::Number,
-    source_model_type: Field::String,
-    source_type: Field::Select.with_options(searchable: false, collection: lambda { |field|
-                                                                             field.resource.class.send(field.attribute.to_s.pluralize).keys
-                                                                           }),
-    created_at: Field::DateTime,
-    updated_at: Field::DateTime
-  }.freeze
-
-  # COLLECTION_ATTRIBUTES
-  # an array of attributes that will be displayed on the model's index page.
-  #
-  # By default, it's limited to four items to reduce clutter on index pages.
-  # Feel free to add, remove, or rearrange items.
-  COLLECTION_ATTRIBUTES = %i[
-    id
-    name
-    account
-    source_link
-  ].freeze
-
-  # SHOW_PAGE_ATTRIBUTES
-  # an array of attributes that will be displayed on the model's show page.
-  SHOW_PAGE_ATTRIBUTES = %i[
-    id
-    name
-    account
-    source_link
-    source_model_id
-    source_model_type
-    source_type
-    created_at
-    updated_at
-    response_documents
-    responses
-  ].freeze
-
-  # FORM_ATTRIBUTES
-  # an array of attributes that will be displayed
-  # on the model's form (`new` and `edit`) pages.
-  FORM_ATTRIBUTES = %i[
-    account
-    name
-    source_link
-    source_model_id
-    source_model_type
-    source_type
-  ].freeze
-
-  # COLLECTION_FILTERS
-  # a hash that defines filters that can be used while searching via the search
-  # field of the dashboard.
-  #
-  # For example to add an option to search for open resources by typing "open:"
-  # in the search field:
-  #
-  #   COLLECTION_FILTERS = {
-  #     open: ->(resources) { resources.where(open: true) }
-  #   }.freeze
-  COLLECTION_FILTERS = {
-    account: ->(resources, attr) { resources.where(account_id: attr) }
-  }.freeze
-
-  # Overwrite this method to customize how response sources are displayed
-  # across all pages of the admin dashboard.
-  #
-  def display_resource(response_source)
-    "Source: ##{response_source.id} - #{response_source.name}"
-  end
-end
diff --git a/enterprise/app/helpers/captain/firecrawl_helper.rb b/enterprise/app/helpers/captain/firecrawl_helper.rb
new file mode 100644
index 000000000..1045d9a1d
--- /dev/null
+++ b/enterprise/app/helpers/captain/firecrawl_helper.rb
@@ -0,0 +1,9 @@
+module Captain::FirecrawlHelper
+  def generate_firecrawl_token(assistant_id, account_id)
+    api_key = InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY')&.value
+    return nil unless api_key
+
+    token_base = "#{api_key[-4..]}#{assistant_id}#{account_id}"
+    Digest::SHA256.hexdigest(token_base)
+  end
+end
diff --git a/enterprise/app/jobs/captain/documents/crawl_job.rb b/enterprise/app/jobs/captain/documents/crawl_job.rb
index ae60c63da..4e1b7d18b 100644
--- a/enterprise/app/jobs/captain/documents/crawl_job.rb
+++ b/enterprise/app/jobs/captain/documents/crawl_job.rb
@@ -11,6 +11,8 @@ class Captain::Documents::CrawlJob < ApplicationJob
 
   private
 
+  include Captain::FirecrawlHelper
+
   def perform_simple_crawl(document)
     page_links = Captain::Tools::SimplePageCrawlService.new(document.external_link).page_links
 
@@ -28,13 +30,22 @@ class Captain::Documents::CrawlJob < ApplicationJob
   end
 
   def perform_firecrawl_crawl(document)
-    webhook_url = Rails.application.routes.url_helpers.enterprise_webhooks_firecrawl_url
+    captain_usage_limits = document.account.usage_limits[:captain] || {}
+    document_limit = captain_usage_limits[:documents] || {}
+    crawl_limit = [document_limit[:available] || 10, 500].min
 
     Captain::Tools::FirecrawlService
       .new
       .perform(
         document.external_link,
-        "#{webhook_url}?assistant_id=#{document.assistant_id}"
+        firecrawl_webhook_url(document),
+        crawl_limit
       )
   end
+
+  def firecrawl_webhook_url(document)
+    webhook_url = Rails.application.routes.url_helpers.enterprise_webhooks_firecrawl_url
+
+    "#{webhook_url}?assistant_id=#{document.assistant_id}&token=#{generate_firecrawl_token(document.assistant_id, document.account_id)}"
+  end
 end
diff --git a/enterprise/app/jobs/captain/tools/firecrawl_parser_job.rb b/enterprise/app/jobs/captain/tools/firecrawl_parser_job.rb
index a380a8ae6..41a50a35e 100644
--- a/enterprise/app/jobs/captain/tools/firecrawl_parser_job.rb
+++ b/enterprise/app/jobs/captain/tools/firecrawl_parser_job.rb
@@ -6,12 +6,12 @@ class Captain::Tools::FirecrawlParserJob < ApplicationJob
     metadata = payload[:metadata]
 
     document = assistant.documents.find_or_initialize_by(
-      external_link: metadata[:ogUrl]
+      external_link: metadata['url']
     )
 
     document.update!(
       content: payload[:markdown],
-      name: metadata[:ogTitle],
+      name: metadata['title'],
       status: :available
     )
   rescue StandardError => e
diff --git a/enterprise/app/models/captain/document.rb b/enterprise/app/models/captain/document.rb
index 8331d291b..d2a02f5b5 100644
--- a/enterprise/app/models/captain/document.rb
+++ b/enterprise/app/models/captain/document.rb
@@ -29,6 +29,7 @@ class Captain::Document < ApplicationRecord
 
   validates :external_link, presence: true
   validates :external_link, uniqueness: { scope: :assistant_id }
+  validates :content, length: { maximum: 200_000 }
   before_validation :ensure_account_id
 
   enum status: {
diff --git a/enterprise/app/services/captain/tools/firecrawl_service.rb b/enterprise/app/services/captain/tools/firecrawl_service.rb
index 8bfb513fc..a32f1d630 100644
--- a/enterprise/app/services/captain/tools/firecrawl_service.rb
+++ b/enterprise/app/services/captain/tools/firecrawl_service.rb
@@ -4,10 +4,10 @@ class Captain::Tools::FirecrawlService
     raise 'Missing API key' if @api_key.nil?
   end
 
-  def perform(url, webhook_url = '')
+  def perform(url, webhook_url, crawl_limit = 10)
     HTTParty.post(
       'https://api.firecrawl.dev/v1/crawl',
-      body: crawl_payload(url, webhook_url),
+      body: crawl_payload(url, webhook_url, crawl_limit),
       headers: headers
     )
   rescue StandardError => e
@@ -16,12 +16,12 @@ class Captain::Tools::FirecrawlService
 
   private
 
-  def crawl_payload(url, webhook_url)
+  def crawl_payload(url, webhook_url, crawl_limit)
     {
       url: url,
       maxDepth: 50,
       ignoreSitemap: false,
-      limit: 10,
+      limit: crawl_limit,
       webhook: webhook_url,
       scrapeOptions: {
         onlyMainContent: false,
diff --git a/enterprise/app/views/super_admin/response_sources/chat.html.erb b/enterprise/app/views/super_admin/response_sources/chat.html.erb
deleted file mode 100644
index ea2147ea4..000000000
--- a/enterprise/app/views/super_admin/response_sources/chat.html.erb
+++ /dev/null
@@ -1,5 +0,0 @@
-<% content_for :title, "Robin AI playground: #{@response_source.name}" %>
-<%= render_vue_component('PlaygroundIndex', {
-  responseSourceName: @response_source.name,
-  responseSourcePath: super_admin_response_source_path(@response_source)
-}) %>
diff --git a/enterprise/app/views/super_admin/response_sources/show.html.erb b/enterprise/app/views/super_admin/response_sources/show.html.erb
deleted file mode 100644
index 60cbda605..000000000
--- a/enterprise/app/views/super_admin/response_sources/show.html.erb
+++ /dev/null
@@ -1,71 +0,0 @@
-<%#
-# Show
-
-This view is the template for the show page.
-It renders the attributes of a resource,
-as well as a link to its edit page.
-
-## Local variables:
-
-- `page`:
-  An instance of [Administrate::Page::Show][1].
-  Contains methods for accessing the resource to be displayed on the page,
-  as well as helpers for describing how each attribute of the resource
-  should be displayed.
-
-[1]: http://www.rubydoc.info/gems/administrate/Administrate/Page/Show
-%>
-
-<% content_for(:title) { t("administrate.actions.show_resource", name: page.page_title) } %>
-
-<header class="main-content__header">
-  <h1 class="main-content__page-title">
-    <%= content_for(:title) %>
-  </h1>
-
-  <div>
-    <%= link_to(
-      "Chat",
-      [:chat, namespace, page.resource],
-      class: "button"
-    ) %>
-
-    <%= link_to(
-      "Edit",
-      [:edit, namespace, page.resource],
-      class: "button",
-    ) if accessible_action?(page.resource, :edit) %>
-
-    <%= link_to(
-      t("administrate.actions.destroy"),
-      [namespace, page.resource],
-      class: "button button--danger",
-      method: :delete,
-      data: { confirm: t("administrate.actions.confirm") }
-    ) if accessible_action?(page.resource, :destroy) %>
-  </div>
-</header>
-
-<section class="main-content__body">
-  <dl>
-    <% page.attributes.each do |title, attributes| %>
-      <fieldset class="<%= "field-unit--nested" if title.present? %>">
-        <% if title.present? %>
-          <legend><%= t "helpers.label.#{page.resource_name}.#{title}", default: title %></legend>
-        <% end %>
-
-        <% attributes.each do |attribute| %>
-          <dt class="attribute-label" id="<%= attribute.name %>">
-          <%= t(
-            "helpers.label.#{resource_name}.#{attribute.name}",
-            default: page.resource.class.human_attribute_name(attribute.name),
-          ) %>
-          </dt>
-
-          <dd class="attribute-data attribute-data--<%=attribute.html_class%>"
-              ><%= render_field attribute, page: page %></dd>
-        <% end %>
-      </fieldset>
-    <% end %>
-  </dl>
-</section>
diff --git a/spec/enterprise/controllers/enterprise/webooks/firecrawl_controller_spec.rb b/spec/enterprise/controllers/enterprise/webooks/firecrawl_controller_spec.rb
index f5810adea..382d8f69c 100644
--- a/spec/enterprise/controllers/enterprise/webooks/firecrawl_controller_spec.rb
+++ b/spec/enterprise/controllers/enterprise/webooks/firecrawl_controller_spec.rb
@@ -1,57 +1,120 @@
 require 'rails_helper'
 
 RSpec.describe 'Firecrawl Webhooks', type: :request do
-  describe 'POST /enterprise/webhooks/firecrawl?assistant_id=:assistant_id' do
-    let(:assistant_id) { 'asst_123' }
+  describe 'POST /enterprise/webhooks/firecrawl?assistant_id=:assistant_id&token=:token' do
+    let!(:api_key) { create(:installation_config, name: 'CAPTAIN_FIRECRAWL_API_KEY', value: 'test_api_key_123') }
+    let!(:account) { create(:account) }
+    let!(:assistant) { create(:captain_assistant, account: account) }
+
     let(:payload_data) do
       {
-        'markdown' => 'hello world',
-        'metadata' => {
-          'ogUrl' => 'https://example.com'
-        }
+        markdown: 'hello world',
+        metadata: { ogUrl: 'https://example.com' }
       }
     end
 
-    context 'with crawl.page event type' do
-      let(:valid_params) do
-        {
-          data: payload_data,
-          type: 'crawl.page'
-        }
+    # Generate actual token using the helper
+    let(:valid_token) do
+      token_base = "#{api_key.value[-4..]}#{assistant.id}#{assistant.account_id}"
+      Digest::SHA256.hexdigest(token_base)
+    end
+
+    context 'with valid token' do
+      context 'with crawl.page event type' do
+        let(:valid_params) do
+          {
+            type: 'crawl.page',
+            data: [payload_data]
+          }
+        end
+
+        it 'processes the webhook and returns success' do
+          expect(Captain::Tools::FirecrawlParserJob).to receive(:perform_later)
+            .with(
+              assistant_id: assistant.id,
+              payload: payload_data
+            )
+
+          post(
+            "/enterprise/webhooks/firecrawl?assistant_id=#{assistant.id}&token=#{valid_token}",
+            params: valid_params,
+            as: :json
+          )
+          expect(response).to have_http_status(:ok)
+          expect(response.body).to be_empty
+        end
       end
 
-      it 'processes the webhook and returns success' do
-        expect(Captain::Tools::FirecrawlParserJob).to(
-          receive(:perform_later)
-          .with(
-            assistant_id: assistant_id,
-            payload: payload_data
-          )
-        )
+      context 'with crawl.completed event type' do
+        let(:valid_params) do
+          {
+            type: 'crawl.completed'
+          }
+        end
 
-        post("/enterprise/webhooks/firecrawl?assistant_id=#{assistant_id}",
-             params: valid_params,
-             as: :json)
+        it 'returns success without enqueuing job' do
+          expect(Captain::Tools::FirecrawlParserJob).not_to receive(:perform_later)
 
-        expect(response).to have_http_status(:ok)
-        expect(response.body).to be_empty
+          post("/enterprise/webhooks/firecrawl?assistant_id=#{assistant.id}&token=#{valid_token}",
+               params: valid_params,
+               as: :json)
+
+          expect(response).to have_http_status(:ok)
+          expect(response.body).to be_empty
+        end
       end
     end
 
-    context 'with crawl.completed event type' do
-      let(:valid_params) do
-        { type: 'crawl.completed' }
+    context 'with invalid token' do
+      let(:invalid_params) do
+        {
+          type: 'crawl.page',
+          data: [payload_data]
+        }
       end
 
-      it 'returns success without enqueuing job' do
-        expect(Captain::Tools::FirecrawlParserJob).not_to receive(:perform_later)
-
-        post("/enterprise/webhooks/firecrawl?assistant_id=#{assistant_id}",
-             params: valid_params,
+      it 'returns unauthorized status' do
+        post("/enterprise/webhooks/firecrawl?assistant_id=#{assistant.id}&token=invalid_token",
+             params: invalid_params,
              as: :json)
 
-        expect(response).to have_http_status(:ok)
-        expect(response.body).to be_empty
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+
+    context 'with invalid assistant_id' do
+      context 'with non-existent assistant_id' do
+        it 'returns not found status' do
+          post("/enterprise/webhooks/firecrawl?assistant_id=invalid_id&token=#{valid_token}",
+               params: { type: 'crawl.page', data: [payload_data] },
+               as: :json)
+
+          expect(response).to have_http_status(:not_found)
+        end
+      end
+
+      context 'with nil assistant_id' do
+        it 'returns not found status' do
+          post("/enterprise/webhooks/firecrawl?token=#{valid_token}",
+               params: { type: 'crawl.page', data: [payload_data] },
+               as: :json)
+
+          expect(response).to have_http_status(:not_found)
+        end
+      end
+    end
+
+    context 'when CAPTAIN_FIRECRAWL_API_KEY is not configured' do
+      before do
+        api_key.destroy
+      end
+
+      it 'returns unauthorized status' do
+        post("/enterprise/webhooks/firecrawl?assistant_id=#{assistant.id}&token=#{valid_token}",
+             params: { type: 'crawl.page', data: [payload_data] },
+             as: :json)
+
+        expect(response).to have_http_status(:unauthorized)
       end
     end
   end
diff --git a/spec/enterprise/jobs/captain/documents/crawl_job_spec.rb b/spec/enterprise/jobs/captain/documents/crawl_job_spec.rb
index dde5d4b6a..13876a282 100644
--- a/spec/enterprise/jobs/captain/documents/crawl_job_spec.rb
+++ b/spec/enterprise/jobs/captain/documents/crawl_job_spec.rb
@@ -8,21 +8,61 @@ RSpec.describe Captain::Documents::CrawlJob, type: :job do
   describe '#perform' do
     context 'when CAPTAIN_FIRECRAWL_API_KEY is configured' do
       let(:firecrawl_service) { instance_double(Captain::Tools::FirecrawlService) }
+      let(:account) { document.account }
+      let(:token) { Digest::SHA256.hexdigest("-key#{document.assistant_id}#{document.account_id}") }
 
       before do
         allow(Captain::Tools::FirecrawlService).to receive(:new).and_return(firecrawl_service)
         allow(firecrawl_service).to receive(:perform)
+        create(:installation_config, name: 'CAPTAIN_FIRECRAWL_API_KEY', value: 'test-key')
       end
 
-      it 'uses FirecrawlService to crawl the page' do
-        create(:installation_config, name: 'CAPTAIN_FIRECRAWL_API_KEY', value: 'test-key')
+      context 'with account usage limits' do
+        before do
+          allow(account).to receive(:usage_limits).and_return({ captain: { documents: { available: 20 } } })
+        end
 
-        expect(firecrawl_service).to receive(:perform).with(
-          document.external_link,
-          "#{webhook_url}?assistant_id=#{assistant_id}"
-        )
+        it 'uses FirecrawlService with the correct crawl limit' do
+          expect(firecrawl_service).to receive(:perform).with(
+            document.external_link,
+            "#{webhook_url}?assistant_id=#{assistant_id}&token=#{token}",
+            20
+          )
 
-        described_class.perform_now(document)
+          described_class.perform_now(document)
+        end
+      end
+
+      context 'when crawl limit exceeds maximum' do
+        before do
+          allow(account).to receive(:usage_limits).and_return({ captain: { documents: { available: 1000 } } })
+        end
+
+        it 'caps the crawl limit at 500' do
+          expect(firecrawl_service).to receive(:perform).with(
+            document.external_link,
+            "#{webhook_url}?assistant_id=#{assistant_id}&token=#{token}",
+            500
+          )
+
+          described_class.perform_now(document)
+        end
+      end
+
+      context 'with no usage limits configured' do
+        before do
+          allow(account).to receive(:usage_limits).and_return({})
+        end
+
+        it 'uses default crawl limit of 10' do
+          expect(firecrawl_service).to receive(:perform).with(
+            document.external_link,
+            "#{webhook_url}?assistant_id=#{assistant_id}&token=#{token}",
+            10
+          )
+
+          described_class.perform_now(document)
+        end
       end
     end
 
diff --git a/spec/enterprise/jobs/captain/tools/firecrawl_parser_job_spec.rb b/spec/enterprise/jobs/captain/tools/firecrawl_parser_job_spec.rb
index deeacf383..f2a42c7a2 100644
--- a/spec/enterprise/jobs/captain/tools/firecrawl_parser_job_spec.rb
+++ b/spec/enterprise/jobs/captain/tools/firecrawl_parser_job_spec.rb
@@ -7,9 +7,9 @@ RSpec.describe Captain::Tools::FirecrawlParserJob, type: :job do
       {
         markdown: 'Launch Week I is here! 🚀',
         metadata: {
-          title: 'Home - Firecrawl',
-          ogTitle: 'Firecrawl',
-          ogUrl: 'https://www.firecrawl.dev/'
+          'title' => 'Home - Firecrawl',
+          'ogTitle' => 'Firecrawl',
+          'url' => 'https://www.firecrawl.dev/'
         }
       }
     end
@@ -22,8 +22,8 @@ RSpec.describe Captain::Tools::FirecrawlParserJob, type: :job do
       document = assistant.documents.last
       expect(document).to have_attributes(
         content: payload[:markdown],
-        name: payload[:metadata][:ogTitle],
-        external_link: payload[:metadata][:ogUrl],
+        name: payload[:metadata]['title'],
+        external_link: payload[:metadata]['url'],
         status: 'available'
       )
     end
@@ -32,7 +32,7 @@ RSpec.describe Captain::Tools::FirecrawlParserJob, type: :job do
       existing_document = create(:captain_document,
                                  assistant: assistant,
                                  account: assistant.account,
-                                 external_link: payload[:metadata][:ogUrl],
+                                 external_link: payload[:metadata]['url'],
                                  content: 'old content',
                                  name: 'old title',
                                  status: :in_progress)
@@ -44,7 +44,7 @@ RSpec.describe Captain::Tools::FirecrawlParserJob, type: :job do
       existing_document.reload
       expect(existing_document).to have_attributes(
         content: payload[:markdown],
-        name: payload[:metadata][:ogTitle],
+        name: payload[:metadata]['title'],
         status: 'available'
       )
     end
diff --git a/spec/enterprise/services/captain/tools/firecrawl_service_spec.rb b/spec/enterprise/services/captain/tools/firecrawl_service_spec.rb
new file mode 100644
index 000000000..8b358a655
--- /dev/null
+++ b/spec/enterprise/services/captain/tools/firecrawl_service_spec.rb
@@ -0,0 +1,134 @@
+require 'rails_helper'
+
+RSpec.describe Captain::Tools::FirecrawlService do
+  let(:api_key) { 'test-api-key' }
+  let(:url) { 'https://example.com' }
+  let(:webhook_url) { 'https://webhook.example.com/callback' }
+  let(:crawl_limit) { 15 }
+
+  before do
+    create(:installation_config, name: 'CAPTAIN_FIRECRAWL_API_KEY', value: api_key)
+  end
+
+  describe '#initialize' do
+    context 'when API key is configured' do
+      it 'initializes successfully' do
+        expect { described_class.new }.not_to raise_error
+      end
+    end
+
+    context 'when API key is missing' do
+      before do
+        InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY').destroy
+      end
+
+      it 'raises an error' do
+        expect { described_class.new }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+
+    context 'when API key is nil' do
+      before do
+        InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY').update(value: nil)
+      end
+
+      it 'raises an error' do
+        expect { described_class.new }.to raise_error('Missing API key')
+      end
+    end
+  end
+
+  describe '#perform' do
+    let(:service) { described_class.new }
+    let(:expected_payload) do
+      {
+        url: url,
+        maxDepth: 50,
+        ignoreSitemap: false,
+        limit: crawl_limit,
+        webhook: webhook_url,
+        scrapeOptions: {
+          onlyMainContent: false,
+          formats: ['markdown'],
+          excludeTags: ['iframe']
+        }
+      }.to_json
+    end
+
+    let(:expected_headers) do
+      {
+        'Authorization' => "Bearer #{api_key}",
+        'Content-Type' => 'application/json'
+      }
+    end
+
+    context 'when the API call is successful' do
+      before do
+        stub_request(:post, 'https://api.firecrawl.dev/v1/crawl')
+          .with(
+            body: expected_payload,
+            headers: expected_headers
+          )
+          .to_return(status: 200, body: '{"status": "success"}')
+      end
+
+      it 'makes a POST request with correct parameters' do
+        service.perform(url, webhook_url, crawl_limit)
+
+        expect(WebMock).to have_requested(:post, 'https://api.firecrawl.dev/v1/crawl')
+          .with(
+            body: expected_payload,
+            headers: expected_headers
+          )
+      end
+
+      it 'uses default crawl limit when not specified' do
+        default_payload = expected_payload.gsub(crawl_limit.to_s, '10')
+
+        stub_request(:post, 'https://api.firecrawl.dev/v1/crawl')
+          .with(
+            body: default_payload,
+            headers: expected_headers
+          )
+          .to_return(status: 200, body: '{"status": "success"}')
+
+        service.perform(url, webhook_url)
+
+        expect(WebMock).to have_requested(:post, 'https://api.firecrawl.dev/v1/crawl')
+          .with(
+            body: default_payload,
+            headers: expected_headers
+          )
+      end
+    end
+
+    context 'when the API call fails' do
+      before do
+        stub_request(:post, 'https://api.firecrawl.dev/v1/crawl')
+          .to_raise(StandardError.new('Connection failed'))
+      end
+
+      it 'raises an error with the failure message' do
+        expect { service.perform(url, webhook_url, crawl_limit) }
+          .to raise_error('Failed to crawl URL: Connection failed')
+      end
+    end
+
+    context 'when the API returns an error response' do
+      before do
+        stub_request(:post, 'https://api.firecrawl.dev/v1/crawl')
+          .to_return(status: 422, body: '{"error": "Invalid URL"}')
+      end
+
+      it 'makes the request but does not raise an error' do
+        expect { service.perform(url, webhook_url, crawl_limit) }.not_to raise_error
+
+        expect(WebMock).to have_requested(:post, 'https://api.firecrawl.dev/v1/crawl')
+          .with(
+            body: expected_payload,
+            headers: expected_headers
+          )
+      end
+    end
+  end
+end
diff --git a/spec/factories/inbox_response_source.rb b/spec/factories/inbox_response_source.rb
deleted file mode 100644
index d8b9d18c6..000000000
--- a/spec/factories/inbox_response_source.rb
+++ /dev/null
@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-FactoryBot.define do
-  factory :inbox_response_source do
-    inbox
-    response_source
-  end
-end
diff --git a/spec/factories/response.rb b/spec/factories/response.rb
deleted file mode 100644
index b4d3e15e8..000000000
--- a/spec/factories/response.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-FactoryBot.define do
-  factory :response do
-    response_document
-    response_source
-    question { Faker::Lorem.sentence }
-    answer { Faker::Lorem.paragraph }
-    account
-  end
-end
diff --git a/spec/factories/response_document.rb b/spec/factories/response_document.rb
deleted file mode 100644
index 666e64e0a..000000000
--- a/spec/factories/response_document.rb
+++ /dev/null
@@ -1,10 +0,0 @@
-# frozen_string_literal: true
-
-FactoryBot.define do
-  factory :response_document do
-    response_source
-    content { Faker::Lorem.paragraph }
-    document_link { Faker::Internet.url }
-    account
-  end
-end