feat: Add support for account abuse detection (#11001)
This PR adds service to automate account abuse detection. Currently based on the signup name and URL, could potentially add more context such as usage analysis, message metadata etc.
This commit is contained in:
Pranav
@@ -0,0 +1,130 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Internal::AccountAnalysis::AccountUpdaterService do
|
||||
let(:account) { create(:account) }
|
||||
let(:service) { described_class.new(account) }
|
||||
let(:discord_notifier) { instance_double(Internal::AccountAnalysis::DiscordNotifierService, notify_flagged_account: true) }
|
||||
|
||||
before do
|
||||
allow(Internal::AccountAnalysis::DiscordNotifierService).to receive(:new).and_return(discord_notifier)
|
||||
allow(Rails.logger).to receive(:info)
|
||||
end
|
||||
|
||||
describe '#update_with_analysis' do
|
||||
context 'when error_message is provided' do
|
||||
it 'saves the error and notifies Discord' do
|
||||
service.update_with_analysis({}, 'Analysis failed')
|
||||
|
||||
expect(account.internal_attributes['security_flagged']).to be true
|
||||
expect(account.internal_attributes['security_flag_reason']).to eq('Error: Analysis failed')
|
||||
expect(discord_notifier).to have_received(:notify_flagged_account).with(account)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when analysis is successful' do
|
||||
let(:analysis) do
|
||||
{
|
||||
'threat_level' => 'none',
|
||||
'threat_summary' => 'No threats detected',
|
||||
'recommendation' => 'allow'
|
||||
}
|
||||
end
|
||||
|
||||
it 'saves the analysis results' do
|
||||
allow(Time).to receive(:current).and_return('2023-01-01 12:00:00')
|
||||
|
||||
service.update_with_analysis(analysis)
|
||||
|
||||
expect(account.internal_attributes['last_threat_scan_at']).to eq('2023-01-01 12:00:00')
|
||||
expect(account.internal_attributes['last_threat_scan_level']).to eq('none')
|
||||
expect(account.internal_attributes['last_threat_scan_summary']).to eq('No threats detected')
|
||||
expect(account.internal_attributes['last_threat_scan_recommendation']).to eq('allow')
|
||||
end
|
||||
|
||||
it 'does not flag the account when threat level is none' do
|
||||
service.update_with_analysis(analysis)
|
||||
|
||||
expect(account.internal_attributes).not_to include('security_flagged')
|
||||
expect(discord_notifier).not_to have_received(:notify_flagged_account)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when analysis detects high threat level' do
|
||||
let(:analysis) do
|
||||
{
|
||||
'threat_level' => 'high',
|
||||
'threat_summary' => 'Suspicious activity detected',
|
||||
'recommendation' => 'review',
|
||||
'illegal_activities_detected' => false
|
||||
}
|
||||
end
|
||||
|
||||
it 'flags the account and notifies Discord' do
|
||||
service.update_with_analysis(analysis)
|
||||
|
||||
expect(account.internal_attributes['security_flagged']).to be true
|
||||
expect(account.internal_attributes['security_flag_reason']).to eq('Threat detected: Suspicious activity detected')
|
||||
expect(discord_notifier).to have_received(:notify_flagged_account).with(account)
|
||||
expect(Rails.logger).to have_received(:info).with("Flagging account #{account.id} due to threat level: high")
|
||||
expect(Rails.logger).to have_received(:info).with("Account #{account.id} has been flagged for security review")
|
||||
end
|
||||
end
|
||||
|
||||
context 'when analysis detects medium threat level' do
|
||||
let(:analysis) do
|
||||
{
|
||||
'threat_level' => 'medium',
|
||||
'threat_summary' => 'Potential issues found',
|
||||
'recommendation' => 'review',
|
||||
'illegal_activities_detected' => false
|
||||
}
|
||||
end
|
||||
|
||||
it 'flags the account and notifies Discord' do
|
||||
service.update_with_analysis(analysis)
|
||||
|
||||
expect(account.internal_attributes['security_flagged']).to be true
|
||||
expect(account.internal_attributes['security_flag_reason']).to eq('Threat detected: Potential issues found')
|
||||
expect(discord_notifier).to have_received(:notify_flagged_account).with(account)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when analysis detects illegal activities' do
|
||||
let(:analysis) do
|
||||
{
|
||||
'threat_level' => 'low',
|
||||
'threat_summary' => 'Minor issues found',
|
||||
'recommendation' => 'review',
|
||||
'illegal_activities_detected' => true
|
||||
}
|
||||
end
|
||||
|
||||
it 'flags the account and notifies Discord' do
|
||||
service.update_with_analysis(analysis)
|
||||
|
||||
expect(account.internal_attributes['security_flagged']).to be true
|
||||
expect(account.internal_attributes['security_flag_reason']).to eq('Threat detected: Minor issues found')
|
||||
expect(discord_notifier).to have_received(:notify_flagged_account).with(account)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when analysis recommends blocking' do
|
||||
let(:analysis) do
|
||||
{
|
||||
'threat_level' => 'low',
|
||||
'threat_summary' => 'Minor issues found',
|
||||
'recommendation' => 'block',
|
||||
'illegal_activities_detected' => false
|
||||
}
|
||||
end
|
||||
|
||||
it 'flags the account and notifies Discord' do
|
||||
service.update_with_analysis(analysis)
|
||||
|
||||
expect(account.internal_attributes['security_flagged']).to be true
|
||||
expect(account.internal_attributes['security_flag_reason']).to eq('Threat detected: Minor issues found')
|
||||
expect(discord_notifier).to have_received(:notify_flagged_account).with(account)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,111 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Internal::AccountAnalysis::ContentEvaluatorService do
|
||||
let(:service) { described_class.new }
|
||||
let(:content) { 'This is some test content' }
|
||||
|
||||
before do
|
||||
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
|
||||
end
|
||||
|
||||
describe '#evaluate' do
|
||||
context 'when content is present' do
|
||||
let(:llm_response) do
|
||||
{
|
||||
'choices' => [
|
||||
{
|
||||
'message' => {
|
||||
'content' => {
|
||||
'threat_level' => 'low',
|
||||
'threat_summary' => 'No significant threats detected',
|
||||
'detected_threats' => ['minor_concern'],
|
||||
'illegal_activities_detected' => false,
|
||||
'recommendation' => 'approve'
|
||||
}.to_json
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
end
|
||||
|
||||
before do
|
||||
allow(service).to receive(:send_to_llm).and_return(llm_response)
|
||||
allow(Rails.logger).to receive(:info)
|
||||
end
|
||||
|
||||
it 'returns the evaluation results' do
|
||||
result = service.evaluate(content)
|
||||
|
||||
expect(result).to include(
|
||||
'threat_level' => 'low',
|
||||
'threat_summary' => 'No significant threats detected',
|
||||
'detected_threats' => ['minor_concern'],
|
||||
'illegal_activities_detected' => false,
|
||||
'recommendation' => 'approve'
|
||||
)
|
||||
end
|
||||
|
||||
it 'logs the evaluation results' do
|
||||
service.evaluate(content)
|
||||
|
||||
expect(Rails.logger).to have_received(:info).with('LLM evaluation - Level: low, Illegal activities: false')
|
||||
end
|
||||
end
|
||||
|
||||
context 'when content is blank' do
|
||||
let(:blank_content) { '' }
|
||||
|
||||
it 'returns the default evaluation without calling the LLM' do
|
||||
expect(service).not_to receive(:send_to_llm)
|
||||
|
||||
result = service.evaluate(blank_content)
|
||||
|
||||
expect(result).to include(
|
||||
'threat_level' => 'unknown',
|
||||
'threat_summary' => 'Failed to complete content evaluation',
|
||||
'detected_threats' => [],
|
||||
'illegal_activities_detected' => false,
|
||||
'recommendation' => 'review'
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when LLM response is nil' do
|
||||
before do
|
||||
allow(service).to receive(:send_to_llm).and_return(nil)
|
||||
end
|
||||
|
||||
it 'returns the default evaluation' do
|
||||
result = service.evaluate(content)
|
||||
|
||||
expect(result).to include(
|
||||
'threat_level' => 'unknown',
|
||||
'threat_summary' => 'Failed to complete content evaluation',
|
||||
'detected_threats' => [],
|
||||
'illegal_activities_detected' => false,
|
||||
'recommendation' => 'review'
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when error occurs during evaluation' do
|
||||
before do
|
||||
allow(service).to receive(:send_to_llm).and_raise(StandardError.new('Test error'))
|
||||
allow(Rails.logger).to receive(:error)
|
||||
end
|
||||
|
||||
it 'logs the error and returns default evaluation with error type' do
|
||||
result = service.evaluate(content)
|
||||
|
||||
expect(Rails.logger).to have_received(:error).with('Error evaluating content: Test error')
|
||||
expect(result).to include(
|
||||
'threat_level' => 'unknown',
|
||||
'threat_summary' => 'Failed to complete content evaluation',
|
||||
'detected_threats' => ['evaluation_failure'],
|
||||
'illegal_activities_detected' => false,
|
||||
'recommendation' => 'review'
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,73 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Internal::AccountAnalysis::DiscordNotifierService do
|
||||
let(:service) { described_class.new }
|
||||
let(:webhook_url) { 'https://discord.com/api/webhooks/123456789/some-token' }
|
||||
let(:account) do
|
||||
create(
|
||||
:account,
|
||||
internal_attributes: {
|
||||
'last_threat_scan_level' => 'high',
|
||||
'last_threat_scan_recommendation' => 'review',
|
||||
'illegal_activities_detected' => true,
|
||||
'last_threat_scan_summary' => 'Suspicious activity detected'
|
||||
}
|
||||
)
|
||||
end
|
||||
let!(:user) { create(:user, account: account) }
|
||||
|
||||
before do
|
||||
allow(Rails.logger).to receive(:info)
|
||||
allow(Rails.logger).to receive(:error)
|
||||
end
|
||||
|
||||
describe '#notify_flagged_account' do
|
||||
context 'when webhook URL is configured' do
|
||||
before do
|
||||
create(:installation_config, name: 'ACCOUNT_SECURITY_NOTIFICATION_WEBHOOK_URL', value: webhook_url)
|
||||
stub_request(:post, webhook_url).to_return(status: 200)
|
||||
end
|
||||
|
||||
it 'sends notification to Discord webhook' do
|
||||
service.notify_flagged_account(account)
|
||||
expect(WebMock).to have_requested(:post, webhook_url)
|
||||
.with(
|
||||
body: hash_including(
|
||||
content: include(
|
||||
"Account ID: #{account.id}",
|
||||
"User Email: #{user.email}",
|
||||
'Threat Level: high',
|
||||
'**System Recommendation:** review',
|
||||
'⚠️ Potential illegal activities detected',
|
||||
'Suspicious activity detected'
|
||||
)
|
||||
)
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when webhook URL is not configured' do
|
||||
it 'logs error and does not make HTTP request' do
|
||||
service.notify_flagged_account(account)
|
||||
|
||||
expect(Rails.logger).to have_received(:error)
|
||||
.with('Cannot send Discord notification: No webhook URL configured')
|
||||
expect(WebMock).not_to have_requested(:post, webhook_url)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when HTTP request fails' do
|
||||
before do
|
||||
create(:installation_config, name: 'ACCOUNT_SECURITY_NOTIFICATION_WEBHOOK_URL', value: webhook_url)
|
||||
stub_request(:post, webhook_url).to_raise(StandardError.new('Connection failed'))
|
||||
end
|
||||
|
||||
it 'catches exception and logs error' do
|
||||
service.notify_flagged_account(account)
|
||||
|
||||
expect(Rails.logger).to have_received(:error)
|
||||
.with('Error sending Discord notification: Connection failed')
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,62 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Internal::AccountAnalysis::ThreatAnalyserService do
|
||||
subject { described_class.new(account) }
|
||||
|
||||
let(:account) { create(:account) }
|
||||
let(:user) { create(:user, email: 'test@example.com', account: account) }
|
||||
let(:website_scraper) { instance_double(Internal::AccountAnalysis::WebsiteScraperService) }
|
||||
let(:content_evaluator) { instance_double(Internal::AccountAnalysis::ContentEvaluatorService) }
|
||||
let(:account_updater) { instance_double(Internal::AccountAnalysis::AccountUpdaterService) }
|
||||
let(:website_content) { 'This is the website content' }
|
||||
let(:threat_analysis) { { 'threat_level' => 'medium' } }
|
||||
|
||||
before do
|
||||
user
|
||||
|
||||
allow(Internal::AccountAnalysis::WebsiteScraperService).to receive(:new).with('example.com').and_return(website_scraper)
|
||||
allow(Internal::AccountAnalysis::ContentEvaluatorService).to receive(:new).and_return(content_evaluator)
|
||||
allow(Internal::AccountAnalysis::AccountUpdaterService).to receive(:new).with(account).and_return(account_updater)
|
||||
end
|
||||
|
||||
describe '#perform' do
|
||||
before do
|
||||
allow(website_scraper).to receive(:perform).and_return(website_content)
|
||||
allow(content_evaluator).to receive(:evaluate).and_return(threat_analysis)
|
||||
allow(account_updater).to receive(:update_with_analysis)
|
||||
allow(Rails.logger).to receive(:info)
|
||||
end
|
||||
|
||||
it 'performs threat analysis and updates the account' do
|
||||
expected_content = <<~MESSAGE
|
||||
Domain: example.com
|
||||
Content: This is the website content
|
||||
MESSAGE
|
||||
|
||||
expect(website_scraper).to receive(:perform)
|
||||
expect(content_evaluator).to receive(:evaluate).with(expected_content)
|
||||
expect(account_updater).to receive(:update_with_analysis).with(threat_analysis)
|
||||
expect(Rails.logger).to receive(:info).with("Completed threat analysis: level=medium for account-id: #{account.id}")
|
||||
|
||||
result = subject.perform
|
||||
expect(result).to eq(threat_analysis)
|
||||
end
|
||||
|
||||
context 'when website content is blank' do
|
||||
before do
|
||||
allow(website_scraper).to receive(:perform).and_return(nil)
|
||||
end
|
||||
|
||||
it 'logs info and updates account with error' do
|
||||
expect(Rails.logger).to receive(:info).with("Skipping threat analysis for account #{account.id}: No website content found")
|
||||
expect(account_updater).to receive(:update_with_analysis).with(nil, 'Scraping error: No content found')
|
||||
expect(content_evaluator).not_to receive(:evaluate)
|
||||
|
||||
result = subject.perform
|
||||
expect(result).to be_nil
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,45 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Internal::AccountAnalysis::WebsiteScraperService do
|
||||
describe '#perform' do
|
||||
let(:service) { described_class.new(domain) }
|
||||
let(:html_content) { '<html><body>This is sample website content</body></html>' }
|
||||
|
||||
before do
|
||||
allow(Rails.logger).to receive(:info)
|
||||
allow(Rails.logger).to receive(:error)
|
||||
end
|
||||
|
||||
context 'when domain is nil' do
|
||||
let(:domain) { nil }
|
||||
|
||||
it 'returns nil' do
|
||||
expect(service.perform).to be_nil
|
||||
end
|
||||
end
|
||||
|
||||
context 'when domain is present' do
|
||||
let(:domain) { 'example.com' }
|
||||
|
||||
before do
|
||||
allow(HTTParty).to receive(:get).and_return(html_content)
|
||||
end
|
||||
|
||||
it 'returns the stripped and normalized content' do
|
||||
expect(service.perform).to eq(html_content)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when an error occurs' do
|
||||
let(:domain) { 'example.com' }
|
||||
|
||||
before do
|
||||
allow(HTTParty).to receive(:get).and_raise(StandardError.new('Error'))
|
||||
end
|
||||
|
||||
it 'returns nil' do
|
||||
expect(service.perform).to be_nil
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user