feat: Add support for account abuse detection (#11001)

This PR adds service to automate account abuse detection. Currently
based on the signup name and URL, could potentially add more context
such as usage analysis, message metadata etc.

enterprise/app/services/captain/copilot/chat_service.rb

@@ -1,6 +1,6 @@
 require 'openai'
 
-class Captain::Copilot::ChatService < Captain::Llm::BaseOpenAiService
+class Captain::Copilot::ChatService < Llm::BaseOpenAiService
   include Captain::ChatHelper
 
   def initialize(assistant, config)

enterprise/app/services/captain/llm/assistant_chat_service.rb

@@ -1,6 +1,6 @@
 require 'openai'
 
-class Captain::Llm::AssistantChatService < Captain::Llm::BaseOpenAiService
+class Captain::Llm::AssistantChatService < Llm::BaseOpenAiService
   include Captain::ChatHelper
 
   def initialize(assistant: nil)

enterprise/app/services/captain/llm/contact_attributes_service.rb

@@ -1,4 +1,4 @@
-class Captain::Llm::ContactAttributesService < Captain::Llm::BaseOpenAiService
+class Captain::Llm::ContactAttributesService < Llm::BaseOpenAiService
   def initialize(assistant, conversation)
     super()
     @assistant = assistant

enterprise/app/services/captain/llm/contact_notes_service.rb

@@ -1,4 +1,4 @@
-class Captain::Llm::ContactNotesService < Captain::Llm::BaseOpenAiService
+class Captain::Llm::ContactNotesService < Llm::BaseOpenAiService
   def initialize(assistant, conversation)
     super()
     @assistant = assistant

enterprise/app/services/captain/llm/conversation_faq_service.rb

@@ -1,4 +1,4 @@
-class Captain::Llm::ConversationFaqService < Captain::Llm::BaseOpenAiService
+class Captain::Llm::ConversationFaqService < Llm::BaseOpenAiService
   DISTANCE_THRESHOLD = 0.3
 
   def initialize(assistant, conversation)

enterprise/app/services/captain/llm/embedding_service.rb

@@ -1,6 +1,6 @@
 require 'openai'
 
-class Captain::Llm::EmbeddingService < Captain::Llm::BaseOpenAiService
+class Captain::Llm::EmbeddingService < Llm::BaseOpenAiService
   class EmbeddingsError < StandardError; end
 
   DEFAULT_MODEL = 'text-embedding-3-small'.freeze

enterprise/app/services/captain/llm/faq_generator_service.rb

@@ -1,4 +1,4 @@
-class Captain::Llm::FaqGeneratorService < Captain::Llm::BaseOpenAiService
+class Captain::Llm::FaqGeneratorService < Llm::BaseOpenAiService
   def initialize(content)
     super()
     @content = content

enterprise/app/services/internal/account_analysis/account_updater_service.rb

@@ -0,0 +1,54 @@
+class Internal::AccountAnalysis::AccountUpdaterService
+  def initialize(account)
+    @account = account
+  end
+
+  def update_with_analysis(analysis, error_message = nil)
+    if error_message
+      save_error(error_message)
+      notify_on_discord
+      return
+    end
+
+    save_analysis_results(analysis)
+    flag_account_if_needed(analysis)
+  end
+
+  private
+
+  def save_error(error_message)
+    @account.internal_attributes['security_flagged'] = true
+    @account.internal_attributes['security_flag_reason'] = "Error: #{error_message}"
+    @account.save
+  end
+
+  def save_analysis_results(analysis)
+    @account.internal_attributes['last_threat_scan_at'] = Time.current
+    @account.internal_attributes['last_threat_scan_level'] = analysis['threat_level']
+    @account.internal_attributes['last_threat_scan_summary'] = analysis['threat_summary']
+    @account.internal_attributes['last_threat_scan_recommendation'] = analysis['recommendation']
+    @account.save!
+  end
+
+  def flag_account_if_needed(analysis)
+    return if analysis['threat_level'] == 'none'
+
+    if %w[high medium].include?(analysis['threat_level']) ||
+       analysis['illegal_activities_detected'] == true ||
+       analysis['recommendation'] == 'block'
+
+      @account.internal_attributes['security_flagged'] = true
+      @account.internal_attributes['security_flag_reason'] = "Threat detected: #{analysis['threat_summary']}"
+      @account.save!
+
+      Rails.logger.info("Flagging account #{@account.id} due to threat level: #{analysis['threat_level']}")
+    end
+
+    notify_on_discord
+  end
+
+  def notify_on_discord
+    Rails.logger.info("Account #{@account.id} has been flagged for security review")
+    Internal::AccountAnalysis::DiscordNotifierService.new.notify_flagged_account(@account)
+  end
+end

enterprise/app/services/internal/account_analysis/content_evaluator_service.rb

@@ -0,0 +1,73 @@
+class Internal::AccountAnalysis::ContentEvaluatorService < Llm::BaseOpenAiService
+  def initialize
+    super()
+
+    @model = 'gpt-4o-mini'.freeze
+  end
+
+  def evaluate(content)
+    return default_evaluation if content.blank?
+
+    begin
+      response = send_to_llm(content)
+      evaluation = handle_response(response)
+      log_evaluation_results(evaluation)
+      evaluation
+    rescue StandardError => e
+      handle_evaluation_error(e)
+    end
+  end
+
+  private
+
+  def send_to_llm(content)
+    Rails.logger.info('Sending content to LLM for security evaluation')
+    @client.chat(
+      parameters: {
+        model: @model,
+        messages: llm_messages(content),
+        response_format: { type: 'json_object' }
+      }
+    )
+  end
+
+  def handle_response(response)
+    return default_evaluation if response.nil?
+
+    parsed = JSON.parse(response.dig('choices', 0, 'message', 'content').strip)
+
+    {
+      'threat_level' => parsed['threat_level'] || 'unknown',
+      'threat_summary' => parsed['threat_summary'] || 'No threat summary provided',
+      'detected_threats' => parsed['detected_threats'] || [],
+      'illegal_activities_detected' => parsed['illegal_activities_detected'] || false,
+      'recommendation' => parsed['recommendation'] || 'review'
+    }
+  end
+
+  def default_evaluation(error_type = nil)
+    {
+      'threat_level' => 'unknown',
+      'threat_summary' => 'Failed to complete content evaluation',
+      'detected_threats' => error_type ? [error_type] : [],
+      'illegal_activities_detected' => false,
+      'recommendation' => 'review'
+    }
+  end
+
+  def log_evaluation_results(evaluation)
+    Rails.logger.info("LLM evaluation - Level: #{evaluation['threat_level']}, Illegal activities: #{evaluation['illegal_activities_detected']}")
+  end
+
+  def handle_evaluation_error(error)
+    Rails.logger.error("Error evaluating content: #{error.message}")
+    default_evaluation('evaluation_failure')
+  end
+
+  def llm_messages(content)
+    [
+      { role: 'system', content: 'You are a security analysis system that evaluates content for potential threats and scams.' },
+      { role: 'user', content: Internal::AccountAnalysis::PromptsService.threat_analyser(content.to_s[0...10_000]) }
+    ]
+  end
+end

enterprise/app/services/internal/account_analysis/discord_notifier_service.rb

@@ -0,0 +1,47 @@
+class Internal::AccountAnalysis::DiscordNotifierService
+  def notify_flagged_account(account)
+    if webhook_url.blank?
+      Rails.logger.error('Cannot send Discord notification: No webhook URL configured')
+      return
+    end
+
+    HTTParty.post(
+      webhook_url,
+      body: build_message(account).to_json,
+      headers: { 'Content-Type' => 'application/json' }
+    )
+
+    Rails.logger.info("Discord notification sent for flagged account #{account.id}")
+  rescue StandardError => e
+    Rails.logger.error("Error sending Discord notification: #{e.message}")
+  end
+
+  private
+
+  def build_message(account)
+    analysis = account.internal_attributes
+    user = account.users.order(id: :asc).first
+
+    content = <<~MESSAGE
+      ---
+      An account has been flagged in our security system with the following details:
+
+      🆔 **Account Details:**
+      Account ID: #{account.id}
+      User Email: #{user&.email || 'N/A'}
+      Threat Level: #{analysis['last_threat_scan_level']}
+
+      🔎 **System Recommendation:** #{analysis['last_threat_scan_recommendation']}
+      #{analysis['illegal_activities_detected'] ? '⚠️ Potential illegal activities detected' : 'No illegal activities detected'}
+
+      📝 **Findings:**
+      #{analysis['last_threat_scan_summary']}
+    MESSAGE
+
+    { content: content }
+  end
+
+  def webhook_url
+    @webhook_url ||= InstallationConfig.find_by(name: 'ACCOUNT_SECURITY_NOTIFICATION_WEBHOOK_URL')&.value
+  end
+end

enterprise/app/services/internal/account_analysis/prompts_service.rb

@@ -0,0 +1,31 @@
+class Internal::AccountAnalysis::PromptsService
+  class << self
+    def threat_analyser(content)
+      <<~PROMPT
+        Analyze the following website content for potential security threats, scams, or illegal activities.
+
+        Focus on identifying:
+        1. Phishing attempts
+        2. Fraudulent business practices
+        3. Malware distribution
+        4. Illegal product/service offerings
+        5. Money laundering indicators
+        6. Identity theft schemes
+
+        Always classify websites under construction or without content to be a medium.
+
+        Website content:
+        #{content}
+
+        Provide your analysis in the following JSON format:
+        {
+          "threat_level": "none|low|medium|high",
+          "threat_summary": "Brief summary of findings",
+          "detected_threats": ["threat1", "threat2"],
+          "illegal_activities_detected": true|false,
+          "recommendation": "approve|review|block"
+        }
+      PROMPT
+    end
+  end
+end

enterprise/app/services/internal/account_analysis/threat_analyser_service.rb

@@ -0,0 +1,43 @@
+class Internal::AccountAnalysis::ThreatAnalyserService
+  def initialize(account)
+    @account = account
+    @user = account.users.order(id: :asc).first
+    @domain = extract_domain_from_email(@user&.email)
+  end
+
+  def perform
+    if @domain.blank?
+      Rails.logger.info("Skipping threat analysis for account #{@account.id}: No domain found")
+      return
+    end
+
+    website_content = Internal::AccountAnalysis::WebsiteScraperService.new(@domain).perform
+    if website_content.blank?
+      Rails.logger.info("Skipping threat analysis for account #{@account.id}: No website content found")
+      Internal::AccountAnalysis::AccountUpdaterService.new(@account).update_with_analysis(nil, 'Scraping error: No content found')
+      return
+    end
+
+    content = <<~MESSAGE
+      Domain: #{@domain}
+      Content: #{website_content}
+    MESSAGE
+    threat_analysis = Internal::AccountAnalysis::ContentEvaluatorService.new.evaluate(content)
+    Rails.logger.info("Completed threat analysis: level=#{threat_analysis['threat_level']} for account-id: #{@account.id}")
+
+    Internal::AccountAnalysis::AccountUpdaterService.new(@account).update_with_analysis(threat_analysis)
+
+    threat_analysis
+  end
+
+  private
+
+  def extract_domain_from_email(email)
+    return nil if email.blank?
+
+    email.split('@').last
+  rescue StandardError => e
+    Rails.logger.error("Error extracting domain from email #{email}: #{e.message}")
+    nil
+  end
+end

enterprise/app/services/internal/account_analysis/website_scraper_service.rb

@@ -0,0 +1,32 @@
+class Internal::AccountAnalysis::WebsiteScraperService
+  def initialize(domain)
+    @domain = domain
+  end
+
+  def perform
+    return nil if @domain.blank?
+
+    Rails.logger.info("Scraping website: #{external_link}")
+
+    begin
+      response = HTTParty.get(external_link, follow_redirects: true)
+      response.to_s
+    rescue StandardError => e
+      Rails.logger.error("Error scraping website for domain #{@domain}: #{e.message}")
+      nil
+    end
+  end
+
+  private
+
+  def external_link
+    sanitize_url(@domain)
+  end
+
+  def sanitize_url(domain)
+    url = domain
+    url = "https://#{domain}" unless domain.start_with?('http://', 'https://')
+    Rails.logger.info("Sanitized URL: #{url}")
+    url
+  end
+end

enterprise/app/services/llm/base_open_ai_service.rb

@@ -1,4 +1,4 @@
-class Captain::Llm::BaseOpenAiService
+class Llm::BaseOpenAiService
   DEFAULT_MODEL = 'gpt-4o-mini'.freeze
 
   def initialize