fix: Added authentication to FireCrawl API, remove unused RobinAI references (#10737)

- Fixed Firecrawl webhook payloads to ensure proper data handling and
delivery.
- Removed unused Robin AI code to improve codebase cleanliness and
maintainability.
- Implement authentication for the Firecrawl endpoint to improve
security. A key is generated to secure the webhook URLs from FireCrawl.

---------

Co-authored-by: Pranav <pranavrajs@gmail.com>

enterprise/app/controllers/enterprise/super_admin/app_configs_controller.rb

@@ -10,7 +10,7 @@ module Enterprise::SuperAdmin::AppConfigsController
     when 'internal'
       @allowed_configs = internal_config_options
     when 'captain'
-      @allowed_configs = %w[CAPTAIN_OPEN_AI_API_KEY CAPTAIN_OPEN_AI_MODEL]
+      @allowed_configs = %w[CAPTAIN_OPEN_AI_API_KEY CAPTAIN_OPEN_AI_MODEL CAPTAIN_FIRECRAWL_API_KEY]
     else
       super
     end

enterprise/app/controllers/enterprise/webhooks/firecrawl_controller.rb

@@ -1,16 +1,32 @@
 class Enterprise::Webhooks::FirecrawlController < ActionController::API
+  before_action :validate_token
+
   def process_payload
-    if crawl_page_event?
-      Captain::Tools::FirecrawlParserJob.perform_later(
-        assistant_id: permitted_params[:assistant_id],
-        payload: permitted_params[:data]
-      )
-    end
+    Captain::Tools::FirecrawlParserJob.perform_later(assistant_id: assistant.id, payload: payload) if crawl_page_event?
+
     head :ok
   end
 
   private
 
+  include Captain::FirecrawlHelper
+
+  def payload
+    permitted_params[:data]&.first&.to_h
+  end
+
+  def validate_token
+    render json: { error: 'Invalid access_token' }, status: :unauthorized if assistant_token != permitted_params[:token]
+  end
+
+  def assistant
+    @assistant ||= Captain::Assistant.find(permitted_params[:assistant_id])
+  end
+
+  def assistant_token
+    generate_firecrawl_token(assistant.id, assistant.account_id)
+  end
+
   def crawl_page_event?
     permitted_params[:type] == 'crawl.page'
   end
@@ -19,12 +35,13 @@ class Enterprise::Webhooks::FirecrawlController < ActionController::API
     params.permit(
       :type,
       :assistant_id,
+      :token,
       :success,
       :id,
       :metadata,
       :format,
       :firecrawl,
-      { data: {} }
+      data: [:markdown, { metadata: {} }]
     )
   end
 end