LeadMagnet/leadchat
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat: legacy features to ruby llm (#12994)

Browse Source
This commit is contained in:
Aakash Bakhle
2025-12-11 14:17:28 +05:30
committed by GitHub
parent f2054e703a
commit 1de8d3e56d
39 changed files with 860 additions and 755 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Gemfile.lock
View File

@@ -827,6 +827,7 @@ GEM
faraday-net_http (>= 1)
faraday-retry (>= 1)
marcel (~> 1.0)
ruby_llm-schema (~> 0.2.1)
zeitwerk (~> 2)
ruby_llm-schema (0.2.5)
ruby_parser (3.20.0)

15
app/models/message.rb
View File

@@ -254,6 +254,21 @@ class Message < ApplicationRecord
Messages::SearchDataPresenter.new(self).search_data
end
# Returns message content suitable for LLM consumption
# Falls back to audio transcription or attachment placeholder when content is nil
def content_for_llm
return content if content.present?
audio_transcription = attachments
.where(file_type: :audio)
.filter_map { |att| att.meta&.dig('transcribed_text') }
.join(' ')
.presence
return "[Voice Message] #{audio_transcription}" if audio_transcription.present?
'[Attachment]' if attachments.any?
end
private
def prevent_message_flooding

2
app/services/llm_formatter/conversation_llm_formatter.rb
View File

@@ -48,7 +48,7 @@ class LlmFormatter::ConversationLlmFormatter < LlmFormatter::DefaultLlmFormatter
'Bot'
end
sender = "[Private Note] #{sender}" if message.private?
"#{sender}: #{message.content}\n"
"#{sender}: #{message.content_for_llm}\n"
end
def build_attributes

4
config/initializers/ai_agents.rb
View File

@@ -4,8 +4,8 @@ require 'agents'
Rails.application.config.after_initialize do
api_key = InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_API_KEY')&.value
model = InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || OpenAiConstants::DEFAULT_MODEL
api_endpoint = InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_ENDPOINT')&.value || OpenAiConstants::DEFAULT_ENDPOINT
model = InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || LlmConstants::DEFAULT_MODEL
api_endpoint = InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_ENDPOINT')&.value || LlmConstants::OPENAI_API_ENDPOINT
if api_key.present?
Agents.configure do |config|

2
enterprise/app/controllers/enterprise/public/api/v1/portals/articles_controller.rb
View File

@@ -3,7 +3,7 @@ module Enterprise::Public::Api::V1::Portals::ArticlesController
def search_articles
if @portal.account.feature_enabled?('help_center_embedding_search')
@articles = @articles.vector_search(list_params) if list_params[:query].present?
@articles = @articles.vector_search(list_params.merge(account_id: @portal.account_id)) if list_params[:query].present?
else
super
end

12
enterprise/app/helpers/captain/chat_helper.rb
View File

@@ -21,20 +21,18 @@ module Captain::ChatHelper
def build_chat
llm_chat = chat(model: @model, temperature: temperature)
llm_chat.with_params(response_format: { type: 'json_object' })
llm_chat = llm_chat.with_params(response_format: { type: 'json_object' })
llm_chat = setup_tools(llm_chat)
setup_system_instructions(llm_chat)
llm_chat = setup_system_instructions(llm_chat)
setup_event_handlers(llm_chat)
llm_chat
end
def setup_tools(chat)
def setup_tools(llm_chat)
@tools&.each do |tool|
chat.with_tool(tool)
llm_chat = llm_chat.with_tool(tool)
end
chat
llm_chat
end
def setup_system_instructions(chat)

2
enterprise/app/jobs/captain/documents/response_builder_job.rb
View File

@@ -26,7 +26,7 @@ class Captain::Documents::ResponseBuilderJob < ApplicationJob
end
def generate_standard_faqs(document)
Captain::Llm::FaqGeneratorService.new(document.content, document.account.locale_english_name).generate
Captain::Llm::FaqGeneratorService.new(document.content, document.account.locale_english_name, account_id: document.account_id).generate
end
def build_paginated_service(document, options)

3
enterprise/app/jobs/captain/llm/update_embedding_job.rb
View File

@@ -2,7 +2,8 @@ class Captain::Llm::UpdateEmbeddingJob < ApplicationJob
queue_as :low
def perform(record, content)
embedding = Captain::Llm::EmbeddingService.new.get_embedding(content)
account_id = record.account_id
embedding = Captain::Llm::EmbeddingService.new(account_id: account_id).get_embedding(content)
record.update!(embedding: embedding)
end
end

2
enterprise/app/models/article_embedding.rb
View File

@@ -19,6 +19,8 @@ class ArticleEmbedding < ApplicationRecord
after_commit :update_response_embedding
delegate :account_id, to: :article
private
def update_response_embedding

4
enterprise/app/models/captain/assistant_response.rb
View File

@@ -44,8 +44,8 @@ class Captain::AssistantResponse < ApplicationRecord
enum status: { pending: 0, approved: 1 }
def self.search(query)
embedding = Captain::Llm::EmbeddingService.new.get_embedding(query)
def self.search(query, account_id: nil)
embedding = Captain::Llm::EmbeddingService.new(account_id: account_id).get_embedding(query)
nearest_neighbors(:embedding, embedding, distance: 'cosine').limit(5)
end

2
enterprise/app/models/concerns/agentable.rb
View File

@@ -43,7 +43,7 @@ module Concerns::Agentable
end
def agent_model
InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || OpenAiConstants::DEFAULT_MODEL
InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || LlmConstants::DEFAULT_MODEL
end
def agent_response_schema

2
enterprise/app/models/enterprise/concerns/article.rb
View File

@@ -11,7 +11,7 @@ module Enterprise::Concerns::Article
add_article_embedding_association
def self.vector_search(params)
embedding = Captain::Llm::EmbeddingService.new.get_embedding(params['query'])
embedding = Captain::Llm::EmbeddingService.new(account_id: params[:account_id]).get_embedding(params['query'])
records = joins(
:category
).search_by_category_slug(

44
enterprise/app/services/captain/llm/contact_attributes_service.rb
View File

@@ -1,4 +1,5 @@
class Captain::Llm::ContactAttributesService < Llm::LegacyBaseOpenAiService
class Captain::Llm::ContactAttributesService < Llm::BaseAiService
include Integrations::LlmInstrumentation
def initialize(assistant, conversation)
super()
@assistant = assistant
@@ -17,33 +18,38 @@ class Captain::Llm::ContactAttributesService < Llm::LegacyBaseOpenAiService
attr_reader :content
def generate_attributes
response = @client.chat(parameters: chat_parameters)
parse_response(response)
rescue OpenAI::Error => e
Rails.logger.error "OpenAI API Error: #{e.message}"
response = instrument_llm_call(instrumentation_params) do
chat
.with_params(response_format: { type: 'json_object' })
.with_instructions(system_prompt)
.ask(@content)
end
parse_response(response.content)
rescue RubyLLM::Error => e
ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception
[]
end
def chat_parameters
prompt = Captain::Llm::SystemPromptsService.attributes_generator
def instrumentation_params
{
span_name: 'llm.captain.contact_attributes',
model: @model,
response_format: { type: 'json_object' },
temperature: @temperature,
account_id: @conversation.account_id,
feature_name: 'contact_attributes',
messages: [
{
role: 'system',
content: prompt
},
{
role: 'user',
content: content
}
]
{ role: 'system', content: system_prompt },
{ role: 'user', content: @content }
],
metadata: { assistant_id: @assistant.id, contact_id: @contact.id }
}
end
def parse_response(response)
content = response.dig('choices', 0, 'message', 'content')
def system_prompt
Captain::Llm::SystemPromptsService.attributes_generator
end
def parse_response(content)
return [] if content.nil?
JSON.parse(content.strip).fetch('attributes', [])

51
enterprise/app/services/captain/llm/contact_notes_service.rb
View File

@@ -1,4 +1,5 @@
class Captain::Llm::ContactNotesService < Llm::LegacyBaseOpenAiService
class Captain::Llm::ContactNotesService < Llm::BaseAiService
include Integrations::LlmInstrumentation
def initialize(assistant, conversation)
super()
@assistant = assistant
@@ -18,38 +19,42 @@ class Captain::Llm::ContactNotesService < Llm::LegacyBaseOpenAiService
attr_reader :content
def generate_notes
response = @client.chat(parameters: chat_parameters)
parse_response(response)
rescue OpenAI::Error => e
Rails.logger.error "OpenAI API Error: #{e.message}"
response = instrument_llm_call(instrumentation_params) do
chat
.with_params(response_format: { type: 'json_object' })
.with_instructions(system_prompt)
.ask(@content)
end
parse_response(response.content)
rescue RubyLLM::Error => e
ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception
[]
end
def chat_parameters
account_language = @conversation.account.locale_english_name
prompt = Captain::Llm::SystemPromptsService.notes_generator(account_language)
def instrumentation_params
{
span_name: 'llm.captain.contact_notes',
model: @model,
response_format: { type: 'json_object' },
temperature: @temperature,
account_id: @conversation.account_id,
feature_name: 'contact_notes',
messages: [
{
role: 'system',
content: prompt
},
{
role: 'user',
content: content
}
]
{ role: 'system', content: system_prompt },
{ role: 'user', content: @content }
],
metadata: { assistant_id: @assistant.id, contact_id: @contact.id }
}
end
def parse_response(response)
content = response.dig('choices', 0, 'message', 'content')
return [] if content.nil?
def system_prompt
account_language = @conversation.account.locale_english_name
Captain::Llm::SystemPromptsService.notes_generator(account_language)
end
JSON.parse(content.strip).fetch('notes', [])
def parse_response(response)
return [] if response.nil?
JSON.parse(response.strip).fetch('notes', [])
rescue JSON::ParserError => e
Rails.logger.error "Error in parsing GPT processed response: #{e.message}"
[]

54
enterprise/app/services/captain/llm/conversation_faq_service.rb
View File

@@ -1,4 +1,5 @@
class Captain::Llm::ConversationFaqService < Llm::LegacyBaseOpenAiService
class Captain::Llm::ConversationFaqService < Llm::BaseAiService
include Integrations::LlmInstrumentation
DISTANCE_THRESHOLD = 0.3
def initialize(assistant, conversation)
@@ -35,7 +36,7 @@ class Captain::Llm::ConversationFaqService < Llm::LegacyBaseOpenAiService
faqs.each do |faq|
combined_text = "#{faq['question']}: #{faq['answer']}"
embedding = Captain::Llm::EmbeddingService.new.get_embedding(combined_text)
embedding = Captain::Llm::EmbeddingService.new(account_id: @conversation.account_id).get_embedding(combined_text)
similar_faqs = find_similar_faqs(embedding)
if similar_faqs.any?
@@ -81,38 +82,43 @@ class Captain::Llm::ConversationFaqService < Llm::LegacyBaseOpenAiService
end
def generate
response = @client.chat(parameters: chat_parameters)
parse_response(response)
rescue OpenAI::Error => e
Rails.logger.error "OpenAI API Error: #{e.message}"
response = instrument_llm_call(instrumentation_params) do
chat
.with_params(response_format: { type: 'json_object' })
.with_instructions(system_prompt)
.ask(@content)
end
parse_response(response.content)
rescue RubyLLM::Error => e
Rails.logger.error "LLM API Error: #{e.message}"
[]
end
def chat_parameters
account_language = @conversation.account.locale_english_name
prompt = Captain::Llm::SystemPromptsService.conversation_faq_generator(account_language)
def instrumentation_params
{
span_name: 'llm.captain.conversation_faq',
model: @model,
response_format: { type: 'json_object' },
temperature: @temperature,
account_id: @conversation.account_id,
conversation_id: @conversation.id,
feature_name: 'conversation_faq',
messages: [
{
role: 'system',
content: prompt
},
{
role: 'user',
content: content
}
]
{ role: 'system', content: system_prompt },
{ role: 'user', content: @content }
],
metadata: { assistant_id: @assistant.id }
}
end
def parse_response(response)
content = response.dig('choices', 0, 'message', 'content')
return [] if content.nil?
def system_prompt
account_language = @conversation.account.locale_english_name
Captain::Llm::SystemPromptsService.conversation_faq_generator(account_language)
end
JSON.parse(content.strip).fetch('faqs', [])
def parse_response(response)
return [] if response.nil?
JSON.parse(response.strip).fetch('faqs', [])
rescue JSON::ParserError => e
Rails.logger.error "Error in parsing GPT processed response: #{e.message}"
[]

42
enterprise/app/services/captain/llm/embedding_service.rb
View File

@@ -1,22 +1,38 @@
require 'openai'
class Captain::Llm::EmbeddingService
include Integrations::LlmInstrumentation
class Captain::Llm::EmbeddingService < Llm::LegacyBaseOpenAiService
class EmbeddingsError < StandardError; end
def self.embedding_model
@embedding_model = InstallationConfig.find_by(name: 'CAPTAIN_EMBEDDING_MODEL')&.value.presence || OpenAiConstants::DEFAULT_EMBEDDING_MODEL
def initialize(account_id: nil)
Llm::Config.initialize!
@account_id = account_id
@embedding_model = InstallationConfig.find_by(name: 'CAPTAIN_EMBEDDING_MODEL')&.value.presence || LlmConstants::DEFAULT_EMBEDDING_MODEL
end
def get_embedding(content, model: self.class.embedding_model)
response = @client.embeddings(
parameters: {
model: model,
input: content
}
)
def self.embedding_model
InstallationConfig.find_by(name: 'CAPTAIN_EMBEDDING_MODEL')&.value.presence || LlmConstants::DEFAULT_EMBEDDING_MODEL
end
response.dig('data', 0, 'embedding')
rescue StandardError => e
def get_embedding(content, model: @embedding_model)
return [] if content.blank?
instrument_embedding_call(instrumentation_params(content, model)) do
RubyLLM.embed(content, model: model).vectors
end
rescue RubyLLM::Error => e
Rails.logger.error "Embedding API Error: #{e.message}"
raise EmbeddingsError, "Failed to create an embedding: #{e.message}"
end
private
def instrumentation_params(content, model)
{
span_name: 'llm.captain.embedding',
model: model,
input: content,
feature_name: 'embedding',
account_id: @account_id
}
end
end

46
enterprise/app/services/captain/llm/faq_generator_service.rb
View File

@@ -1,15 +1,24 @@
class Captain::Llm::FaqGeneratorService < Llm::LegacyBaseOpenAiService
def initialize(content, language = 'english')
class Captain::Llm::FaqGeneratorService < Llm::BaseAiService
include Integrations::LlmInstrumentation
def initialize(content, language = 'english', account_id: nil)
super()
@language = language
@content = content
@account_id = account_id
end
def generate
response = @client.chat(parameters: chat_parameters)
parse_response(response)
rescue OpenAI::Error => e
Rails.logger.error "OpenAI API Error: #{e.message}"
response = instrument_llm_call(instrumentation_params) do
chat
.with_params(response_format: { type: 'json_object' })
.with_instructions(system_prompt)
.ask(@content)
end
parse_response(response.content)
rescue RubyLLM::Error => e
Rails.logger.error "LLM API Error: #{e.message}"
[]
end
@@ -17,26 +26,25 @@ class Captain::Llm::FaqGeneratorService < Llm::LegacyBaseOpenAiService
attr_reader :content, :language
def chat_parameters
prompt = Captain::Llm::SystemPromptsService.faq_generator(language)
def system_prompt
Captain::Llm::SystemPromptsService.faq_generator(language)
end
def instrumentation_params
{
span_name: 'llm.captain.faq_generator',
model: @model,
response_format: { type: 'json_object' },
temperature: @temperature,
feature_name: 'faq_generator',
account_id: @account_id,
messages: [
{
role: 'system',
content: prompt
},
{
role: 'user',
content: content
}
{ role: 'system', content: system_prompt },
{ role: 'user', content: @content }
]
}
end
def parse_response(response)
content = response.dig('choices', 0, 'message', 'content')
def parse_response(content)
return [] if content.nil?
JSON.parse(content.strip).fetch('faqs', [])

49
enterprise/app/services/captain/llm/paginated_faq_generator_service.rb
View File

@@ -1,4 +1,6 @@
class Captain::Llm::PaginatedFaqGeneratorService < Llm::LegacyBaseOpenAiService
include Integrations::LlmInstrumentation
# Default pages per chunk - easily configurable
DEFAULT_PAGES_PER_CHUNK = 10
MAX_ITERATIONS = 20 # Safety limit to prevent infinite loops
@@ -13,7 +15,7 @@ class Captain::Llm::PaginatedFaqGeneratorService < Llm::LegacyBaseOpenAiService
@max_pages = options[:max_pages] # Optional limit from UI
@total_pages_processed = 0
@iterations_completed = 0
@model = OpenAiConstants::PDF_PROCESSING_MODEL
@model = LlmConstants::PDF_PROCESSING_MODEL
end
def generate
@@ -43,7 +45,19 @@ class Captain::Llm::PaginatedFaqGeneratorService < Llm::LegacyBaseOpenAiService
private
def generate_standard_faqs
response = @client.chat(parameters: standard_chat_parameters)
params = standard_chat_parameters
instrumentation_params = {
span_name: 'llm.faq_generation',
account_id: @document&.account_id,
feature_name: 'faq_generation',
model: @model,
messages: params[:messages]
}
response = instrument_llm_call(instrumentation_params) do
@client.chat(parameters: params)
end
parse_response(response)
rescue OpenAI::Error => e
Rails.logger.error I18n.t('captain.documents.openai_api_error', error: e.message)
@@ -84,7 +98,13 @@ class Captain::Llm::PaginatedFaqGeneratorService < Llm::LegacyBaseOpenAiService
def process_page_chunk(start_page, end_page)
params = build_chunk_parameters(start_page, end_page)
response = @client.chat(parameters: params)
instrumentation_params = build_instrumentation_params(params, start_page, end_page)
response = instrument_llm_call(instrumentation_params) do
@client.chat(parameters: params)
end
result = parse_chunk_response(response)
{ faqs: result['faqs'] || [], has_content: result['has_content'] != false }
rescue OpenAI::Error => e
@@ -180,21 +200,26 @@ class Captain::Llm::PaginatedFaqGeneratorService < Llm::LegacyBaseOpenAiService
def similarity_score(str1, str2)
words1 = str1.downcase.split(/\W+/).reject(&:empty?)
words2 = str2.downcase.split(/\W+/).reject(&:empty?)
common_words = words1 & words2
total_words = (words1 + words2).uniq.size
return 0 if total_words.zero?
common_words.size.to_f / total_words
end
def determine_stop_reason(last_chunk_result)
return 'Maximum iterations reached' if @iterations_completed >= MAX_ITERATIONS
return 'Maximum pages processed' if @max_pages && @total_pages_processed >= @max_pages
return 'No content found in last chunk' if last_chunk_result[:faqs].empty?
return 'End of document reached' if last_chunk_result[:has_content] == false
'Unknown'
def build_instrumentation_params(params, start_page, end_page)
{
span_name: 'llm.paginated_faq_generation',
account_id: @document&.account_id,
feature_name: 'paginated_faq_generation',
model: @model,
messages: params[:messages],
metadata: {
document_id: @document&.id,
start_page: start_page,
end_page: end_page,
iteration: @iterations_completed + 1
}
}
end
end

33
enterprise/app/services/captain/llm/pdf_processing_service.rb
View File

@@ -1,4 +1,6 @@
class Captain::Llm::PdfProcessingService < Llm::LegacyBaseOpenAiService
include Integrations::LlmInstrumentation
def initialize(document)
super()
@document = document
@@ -19,13 +21,30 @@ class Captain::Llm::PdfProcessingService < Llm::LegacyBaseOpenAiService
def upload_pdf_to_openai
with_tempfile do |temp_file|
response = @client.files.upload(
parameters: {
file: temp_file,
purpose: 'assistants'
}
)
response['id']
instrument_file_upload do
response = @client.files.upload(
parameters: {
file: temp_file,
purpose: 'assistants'
}
)
response['id']
end
end
end
def instrument_file_upload(&)
return yield unless ChatwootApp.otel_enabled?
tracer.in_span('llm.file.upload') do |span|
span.set_attribute('gen_ai.provider', 'openai')
span.set_attribute('file.purpose', 'assistants')
span.set_attribute(ATTR_LANGFUSE_USER_ID, document.account_id.to_s)
span.set_attribute(ATTR_LANGFUSE_TAGS, ['pdf_upload'].to_json)
span.set_attribute(format(ATTR_LANGFUSE_METADATA, 'document_id'), document.id.to_s)
file_id = yield
span.set_attribute('file.id', file_id) if file_id
file_id
end
end

37
enterprise/app/services/captain/onboarding/website_analyzer_service.rb
View File

@@ -1,4 +1,5 @@
class Captain::Onboarding::WebsiteAnalyzerService < Llm::LegacyBaseOpenAiService
class Captain::Onboarding::WebsiteAnalyzerService < Llm::BaseAiService
include Integrations::LlmInstrumentation
MAX_CONTENT_LENGTH = 8000
def initialize(website_url)
@@ -57,19 +58,29 @@ class Captain::Onboarding::WebsiteAnalyzerService < Llm::LegacyBaseOpenAiService
end
def extract_business_info
prompt = build_analysis_prompt
response = instrument_llm_call(instrumentation_params) do
chat
.with_params(response_format: { type: 'json_object' }, max_tokens: 1000)
.with_temperature(0.1)
.with_instructions(build_analysis_prompt)
.ask(@website_content)
end
response = client.chat(
parameters: {
model: model,
messages: [{ role: 'user', content: prompt }],
response_format: { type: 'json_object' },
temperature: 0.1,
max_tokens: 1000
}
)
parse_llm_response(response.content)
end
parse_llm_response(response.dig('choices', 0, 'message', 'content'))
def instrumentation_params
{
span_name: 'llm.captain.website_analyzer',
model: @model,
temperature: 0.1,
feature_name: 'website_analyzer',
messages: [
{ role: 'system', content: build_analysis_prompt },
{ role: 'user', content: @website_content }
],
metadata: { website_url: @website_url }
}
end
def build_analysis_prompt
@@ -95,7 +106,7 @@ class Captain::Onboarding::WebsiteAnalyzerService < Llm::LegacyBaseOpenAiService
end
def parse_llm_response(response_text)
parsed_response = JSON.parse(response_text)
parsed_response = JSON.parse(response_text.strip)
{
success: true,

80
enterprise/app/services/internal/account_analysis/content_evaluator_service.rb
View File

@@ -1,48 +1,59 @@
class Internal::AccountAnalysis::ContentEvaluatorService < Llm::LegacyBaseOpenAiService
def initialize
super()
class Internal::AccountAnalysis::ContentEvaluatorService
include Integrations::LlmInstrumentation
@model = 'gpt-4o-mini'.freeze
def initialize
Llm::Config.initialize!
end
def evaluate(content)
return default_evaluation if content.blank?
begin
response = send_to_llm(content)
evaluation = handle_response(response)
log_evaluation_results(evaluation)
evaluation
rescue StandardError => e
handle_evaluation_error(e)
moderation_result = instrument_moderation_call(instrumentation_params(content)) do
RubyLLM.moderate(content.to_s[0...10_000])
end
build_evaluation(moderation_result)
rescue StandardError => e
handle_evaluation_error(e)
end
private
def send_to_llm(content)
Rails.logger.info('Sending content to LLM for security evaluation')
@client.chat(
parameters: {
model: @model,
messages: llm_messages(content),
response_format: { type: 'json_object' }
}
)
def instrumentation_params(content)
{
span_name: 'llm.internal.content_moderation',
model: 'text-moderation-latest',
input: content,
feature_name: 'content_evaluator'
}
end
def handle_response(response)
return default_evaluation if response.nil?
def build_evaluation(result)
flagged = result.flagged?
categories = result.flagged_categories
parsed = JSON.parse(response.dig('choices', 0, 'message', 'content').strip)
{
'threat_level' => parsed['threat_level'] || 'unknown',
'threat_summary' => parsed['threat_summary'] || 'No threat summary provided',
'detected_threats' => parsed['detected_threats'] || [],
'illegal_activities_detected' => parsed['illegal_activities_detected'] || false,
'recommendation' => parsed['recommendation'] || 'review'
evaluation = {
'threat_level' => flagged ? determine_threat_level(result) : 'safe',
'threat_summary' => flagged ? "Content flagged for: #{categories.join(', ')}" : 'No threats detected',
'detected_threats' => categories,
'illegal_activities_detected' => categories.any? { |c| c.include?('violence') || c.include?('self-harm') },
'recommendation' => flagged ? 'review' : 'approve'
}
log_evaluation_results(evaluation)
evaluation
end
def determine_threat_level(result)
scores = result.category_scores
max_score = scores.values.max || 0
case max_score
when 0.8.. then 'critical'
when 0.5..0.8 then 'high'
when 0.2..0.5 then 'medium'
else 'low'
end
end
def default_evaluation(error_type = nil)
@@ -56,18 +67,11 @@ class Internal::AccountAnalysis::ContentEvaluatorService < Llm::LegacyBaseOpenAi
end
def log_evaluation_results(evaluation)
Rails.logger.info("LLM evaluation - Level: #{evaluation['threat_level']}, Illegal activities: #{evaluation['illegal_activities_detected']}")
Rails.logger.info("Moderation evaluation - Level: #{evaluation['threat_level']}, Threats: #{evaluation['detected_threats'].join(', ')}")
end
def handle_evaluation_error(error)
Rails.logger.error("Error evaluating content: #{error.message}")
default_evaluation('evaluation_failure')
end
def llm_messages(content)
[
{ role: 'system', content: 'You are a security analysis system that evaluates content for potential threats and scams.' },
{ role: 'user', content: Internal::AccountAnalysis::PromptsService.threat_analyser(content.to_s[0...10_000]) }
]
end
end

2
enterprise/app/services/llm/base_ai_service.rb
View File

@@ -14,8 +14,6 @@ class Llm::BaseAiService
setup_temperature
end
# Returns a configured RubyLLM chat instance.
# Subclasses can override model/temperature via instance variables or pass them explicitly.
def chat(model: @model, temperature: @temperature)
RubyLLM.chat(model: model).with_temperature(temperature)
end

7
enterprise/app/services/llm/legacy_base_open_ai_service.rb
View File

@@ -1,8 +1,11 @@
# frozen_string_literal: true
# DEPRECATED: This class uses the legacy OpenAI Ruby gem directly.
# New features should use Llm::BaseAiService with RubyLLM instead.
# This class will be removed once all services are migrated to RubyLLM.
# Only used for PDF/file operations that require OpenAI's files API:
# - Captain::Llm::PdfProcessingService (files.upload for assistants)
# - Captain::Llm::PaginatedFaqGeneratorService (uses file_id from uploaded files)
#
# For all other LLM operations, use Llm::BaseAiService with RubyLLM instead.
class Llm::LegacyBaseOpenAiService
DEFAULT_MODEL = 'gpt-4o-mini'

25
enterprise/app/services/messages/audio_transcription_service.rb
View File

@@ -1,4 +1,8 @@
class Messages::AudioTranscriptionService < Llm::LegacyBaseOpenAiService
class Messages::AudioTranscriptionService< Llm::LegacyBaseOpenAiService
include Integrations::LlmInstrumentation
WHISPER_MODEL = 'whisper-1'.freeze
attr_reader :attachment, :message, :account
def initialize(attachment)
@@ -46,7 +50,7 @@ class Messages::AudioTranscriptionService < Llm::LegacyBaseOpenAiService
temp_file_path = fetch_audio_file
response_text = nil
transcribed_text = nil
File.open(temp_file_path, 'rb') do |file|
response = @client.audio.transcribe(
@@ -56,14 +60,23 @@ class Messages::AudioTranscriptionService < Llm::LegacyBaseOpenAiService
temperature: 0.4
}
)
response_text = response['text']
transcribed_text = response['text']
end
FileUtils.rm_f(temp_file_path)
update_transcription(response_text)
response_text
update_transcription(transcribed_text)
transcribed_text
end
def instrumentation_params(file_path)
{
span_name: 'llm.messages.audio_transcription',
model: WHISPER_MODEL,
account_id: account&.id,
feature_name: 'audio_transcription',
file_path: file_path
}
end
def update_transcription(transcribed_text)

137
enterprise/lib/captain/agent.rb
View File

@@ -1,137 +0,0 @@
require 'openai'
class Captain::Agent
attr_reader :name, :tools, :prompt, :persona, :goal, :secrets
def initialize(name:, config:)
@name = name
@prompt = construct_prompt(config)
@tools = prepare_tools(config[:tools] || [])
@messages = config[:messages] || []
@max_iterations = config[:max_iterations] || 10
@llm = Captain::LlmService.new(api_key: config[:secrets][:OPENAI_API_KEY])
@logger = Rails.logger
@logger.info(@prompt)
end
def execute(input, context)
setup_messages(input, context)
result = {}
@max_iterations.times do |iteration|
push_to_messages(role: 'system', content: 'Provide a final answer') if iteration == @max_iterations - 1
result = @llm.call(@messages, functions)
handle_llm_result(result)
break if result[:stop]
end
result[:output]
end
def register_tool(tool)
@tools << tool
end
private
def setup_messages(input, context)
if @messages.empty?
push_to_messages({ role: 'system', content: @prompt })
push_to_messages({ role: 'assistant', content: context }) if context.present?
end
push_to_messages({ role: 'user', content: input })
end
def handle_llm_result(result)
if result[:tool_call]
tool_result = execute_tool(result[:tool_call])
push_to_messages({ role: 'assistant', content: tool_result })
else
push_to_messages({ role: 'assistant', content: result[:output] })
end
result[:output]
end
def execute_tool(tool_call)
function_name = tool_call['function']['name']
arguments = JSON.parse(tool_call['function']['arguments'])
tool = @tools.find { |t| t.name == function_name }
tool.execute(arguments, {})
rescue StandardError => e
"Tool execution failed: #{e.message}"
end
def construct_prompt(config)
return config[:prompt] if config[:prompt]
<<~PROMPT
Persona: #{config[:persona]}
Objective: #{config[:goal]}
Guidelines:
- Persistently work towards achieving the stated objective without deviation.
- Use only the provided tools to complete the task. Avoid inventing or assuming function names.
- Set `'stop': true` once the objective is fully achieved.
- DO NOT return tool usage as the final result.
- If sufficient information is available to deliver result, compile and present it to the user.
- Always return a final result and ENSURE the final result is formatted in Markdown.
Output Structure:
1. **Tool Usage:**
- If a relevant function is identified, call it directly without unnecessary explanations.
2. **Final Answer:**
When ready to provide a complete response, follow this JSON format:
```json
{
"thought_process": "Explain the reasoning and steps taken to arrive at the final result.",
"result": "Provide the complete response in clear, structured text.",
"stop": true
}
PROMPT
end
def prepare_tools(tools = [])
tools.map do |_, tool|
Captain::Tool.new(
name: tool['name'],
config: {
description: tool['description'],
properties: tool['properties'],
secrets: tool['secrets'],
implementation: tool['implementation']
}
)
end
end
def functions
@tools.map do |tool|
properties = {}
tool.properties.each do |property_name, property_details|
properties[property_name] = {
type: property_details[:type],
description: property_details[:description]
}
end
required = tool.properties.select { |_, details| details[:required] == true }.keys
{
type: 'function',
function: {
name: tool.name,
description: tool.description,
parameters: { type: 'object', properties: properties, required: required }
}
}
end
end
def push_to_messages(message)
@logger.info("\n\n\nMessage: #{message}\n\n\n")
@messages << message
end
end

64
enterprise/lib/captain/llm_service.rb
View File

@@ -1,64 +0,0 @@
require 'openai'
class Captain::LlmService
def initialize(config)
@client = OpenAI::Client.new(
access_token: config[:api_key],
log_errors: Rails.env.development?
)
@logger = Rails.logger
end
def call(messages, functions = [])
openai_params = {
model: 'gpt-4o',
response_format: { type: 'json_object' },
messages: messages
}
openai_params[:tools] = functions if functions.any?
response = @client.chat(parameters: openai_params)
handle_response(response)
rescue StandardError => e
handle_error(e)
end
private
def handle_response(response)
if response['choices'][0]['message']['tool_calls']
handle_tool_calls(response)
else
handle_direct_response(response)
end
end
def handle_tool_calls(response)
tool_call = response['choices'][0]['message']['tool_calls'][0]
{
tool_call: tool_call,
output: nil,
stop: false
}
end
def handle_direct_response(response)
content = response.dig('choices', 0, 'message', 'content').strip
parsed = JSON.parse(content)
{
output: parsed['result'] || parsed['thought_process'],
stop: parsed['stop'] || false
}
rescue JSON::ParserError => e
handle_error(e, content)
end
def handle_error(error, content = nil)
@logger.error("LLM call failed: #{error.message}")
@logger.error(error.backtrace.join("\n"))
@logger.error("Content: #{content}") if content
{ output: 'Error occurred, retrying', stop: false }
end
end

66
enterprise/lib/captain/tool.rb
View File

@@ -1,66 +0,0 @@
class Captain::Tool
class InvalidImplementationError < StandardError; end
class InvalidSecretsError < StandardError; end
class ExecutionError < StandardError; end
REQUIRED_PROPERTIES = %w[name description properties secrets].freeze
attr_reader :name, :description, :properties, :secrets, :implementation, :memory
def initialize(name:, config:)
@name = name
@description = config[:description]
@properties = config[:properties]
@secrets = config[:secrets] || []
@implementation = config[:implementation]
@memory = config[:memory] || {}
end
def register_method(&block)
@implementation = block
end
def execute(input, provided_secrets = {})
validate_secrets!(provided_secrets)
validate_input!(input)
raise ExecutionError, 'No implementation registered' unless @implementation
instance_exec(input, provided_secrets, memory, &@implementation)
rescue StandardError => e
raise ExecutionError, "Execution failed: #{e.message}"
end
private
def validate_config!(config)
missing_keys = REQUIRED_PROPERTIES - config.keys
return if missing_keys.empty?
raise InvalidImplementationError,
"Missing required properties: #{missing_keys.join(', ')}"
end
def validate_secrets!(provided_secrets)
required_secrets = secrets.map!(&:to_sym)
missing_secrets = required_secrets - provided_secrets.keys
return if missing_secrets.empty?
raise InvalidSecretsError, "Missing required secrets: #{missing_secrets.join(', ')}"
end
def validate_input!(input)
properties.each do |property, constraints|
validate_property!(input, property, constraints)
end
end
def validate_property!(input, property, constraints)
value = input[property.to_sym]
raise ArgumentError, "Missing required property: #{property}" if constraints['required'] && value.nil?
true
end
end

95
lib/integrations/llm_instrumentation.rb
View File

@@ -7,14 +7,6 @@ module Integrations::LlmInstrumentation
include Integrations::LlmInstrumentationHelpers
include Integrations::LlmInstrumentationSpans
PROVIDER_PREFIXES = {
'openai' => %w[gpt- o1 o3 o4 text-embedding- whisper- tts-],
'anthropic' => %w[claude-],
'google' => %w[gemini-],
'mistral' => %w[mistral- codestral-],
'deepseek' => %w[deepseek-]
}.freeze
def instrument_llm_call(params)
return yield unless ChatwootApp.otel_enabled?
@@ -66,16 +58,57 @@ module Integrations::LlmInstrumentation
end
end
def determine_provider(model_name)
return 'openai' if model_name.blank?
def instrument_embedding_call(params)
return yield unless ChatwootApp.otel_enabled?
model = model_name.to_s.downcase
PROVIDER_PREFIXES.each do |provider, prefixes|
return provider if prefixes.any? { |prefix| model.start_with?(prefix) }
instrument_with_span(params[:span_name] || 'llm.embedding', params) do |span, track_result|
set_embedding_span_attributes(span, params)
result = yield
track_result.call(result)
set_embedding_result_attributes(span, result)
result
end
end
'openai'
def instrument_audio_transcription(params)
return yield unless ChatwootApp.otel_enabled?
instrument_with_span(params[:span_name] || 'llm.audio.transcription', params) do |span, track_result|
set_audio_transcription_span_attributes(span, params)
result = yield
track_result.call(result)
set_transcription_result_attributes(span, result)
result
end
end
def instrument_moderation_call(params)
return yield unless ChatwootApp.otel_enabled?
instrument_with_span(params[:span_name] || 'llm.moderation', params) do |span, track_result|
set_moderation_span_attributes(span, params)
result = yield
track_result.call(result)
set_moderation_result_attributes(span, result)
result
end
end
def instrument_with_span(span_name, params, &)
result = nil
executed = false
tracer.in_span(span_name) do |span|
track_result = lambda do |r|
executed = true
result = r
end
yield(span, track_result)
end
rescue StandardError => e
ChatwootExceptionTracker.new(e, account: resolve_account(params)).capture_exception
raise unless executed
result
end
private
@@ -86,36 +119,4 @@ module Integrations::LlmInstrumentation
nil
end
def setup_span_attributes(span, params)
set_request_attributes(span, params)
set_prompt_messages(span, params[:messages])
set_metadata_attributes(span, params)
end
def record_completion(span, result)
if result.respond_to?(:content)
span.set_attribute(ATTR_GEN_AI_COMPLETION_ROLE, result.role.to_s) if result.respond_to?(:role)
span.set_attribute(ATTR_GEN_AI_COMPLETION_CONTENT, result.content.to_s)
elsif result.is_a?(Hash)
set_completion_attributes(span, result) if result.is_a?(Hash)
end
end
def set_request_attributes(span, params)
provider = determine_provider(params[:model])
span.set_attribute(ATTR_GEN_AI_PROVIDER, provider)
span.set_attribute(ATTR_GEN_AI_REQUEST_MODEL, params[:model])
span.set_attribute(ATTR_GEN_AI_REQUEST_TEMPERATURE, params[:temperature]) if params[:temperature]
end
def set_prompt_messages(span, messages)
messages.each_with_index do |msg, idx|
role = msg[:role] || msg['role']
content = msg[:content] || msg['content']
span.set_attribute(format(ATTR_GEN_AI_PROMPT_ROLE, idx), role)
span.set_attribute(format(ATTR_GEN_AI_PROMPT_CONTENT, idx), content.to_s)
end
end
end

88
lib/integrations/llm_instrumentation_completion_helpers.rb Normal file
View File

@@ -0,0 +1,88 @@
# frozen_string_literal: true
module Integrations::LlmInstrumentationCompletionHelpers
include Integrations::LlmInstrumentationConstants
private
def set_embedding_span_attributes(span, params)
span.set_attribute(ATTR_GEN_AI_PROVIDER, determine_provider(params[:model]))
span.set_attribute(ATTR_GEN_AI_REQUEST_MODEL, params[:model])
span.set_attribute('embedding.input_length', params[:input]&.length || 0)
span.set_attribute(ATTR_LANGFUSE_OBSERVATION_INPUT, params[:input].to_s)
set_common_span_metadata(span, params)
end
def set_audio_transcription_span_attributes(span, params)
span.set_attribute(ATTR_GEN_AI_PROVIDER, 'openai')
span.set_attribute(ATTR_GEN_AI_REQUEST_MODEL, params[:model] || 'whisper-1')
span.set_attribute('audio.duration_seconds', params[:duration]) if params[:duration]
span.set_attribute(ATTR_LANGFUSE_OBSERVATION_INPUT, params[:file_path].to_s) if params[:file_path]
set_common_span_metadata(span, params)
end
def set_moderation_span_attributes(span, params)
span.set_attribute(ATTR_GEN_AI_PROVIDER, 'openai')
span.set_attribute(ATTR_GEN_AI_REQUEST_MODEL, params[:model] || 'text-moderation-latest')
span.set_attribute('moderation.input_length', params[:input]&.length || 0)
span.set_attribute(ATTR_LANGFUSE_OBSERVATION_INPUT, params[:input].to_s)
set_common_span_metadata(span, params)
end
def set_common_span_metadata(span, params)
span.set_attribute(ATTR_LANGFUSE_USER_ID, params[:account_id].to_s) if params[:account_id]
span.set_attribute(ATTR_LANGFUSE_TAGS, [params[:feature_name]].to_json) if params[:feature_name]
end
def set_embedding_result_attributes(span, result)
span.set_attribute('embedding.dimensions', result&.length || 0) if result.is_a?(Array)
span.set_attribute(ATTR_LANGFUSE_OBSERVATION_OUTPUT, "[#{result&.length || 0} dimensions]")
end
def set_transcription_result_attributes(span, result)
transcribed_text = result.respond_to?(:text) ? result.text : result.to_s
span.set_attribute('transcription.length', transcribed_text&.length || 0)
span.set_attribute(ATTR_LANGFUSE_OBSERVATION_OUTPUT, transcribed_text.to_s)
end
def set_moderation_result_attributes(span, result)
span.set_attribute('moderation.flagged', result.flagged?) if result.respond_to?(:flagged?)
span.set_attribute('moderation.categories', result.flagged_categories.to_json) if result.respond_to?(:flagged_categories)
output = {
flagged: result.respond_to?(:flagged?) ? result.flagged? : nil,
categories: result.respond_to?(:flagged_categories) ? result.flagged_categories : []
}
span.set_attribute(ATTR_LANGFUSE_OBSERVATION_OUTPUT, output.to_json)
end
def set_completion_attributes(span, result)
set_completion_message(span, result)
set_usage_metrics(span, result)
set_error_attributes(span, result)
end
def set_completion_message(span, result)
message = result[:message] || result.dig('choices', 0, 'message', 'content')
return if message.blank?
span.set_attribute(ATTR_GEN_AI_COMPLETION_ROLE, 'assistant')
span.set_attribute(ATTR_GEN_AI_COMPLETION_CONTENT, message)
end
def set_usage_metrics(span, result)
usage = result[:usage] || result['usage']
return if usage.blank?
span.set_attribute(ATTR_GEN_AI_USAGE_INPUT_TOKENS, usage['prompt_tokens']) if usage['prompt_tokens']
span.set_attribute(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, usage['completion_tokens']) if usage['completion_tokens']
span.set_attribute(ATTR_GEN_AI_USAGE_TOTAL_TOKENS, usage['total_tokens']) if usage['total_tokens']
end
def set_error_attributes(span, result)
error = result[:error] || result['error']
return if error.blank?
span.set_attribute(ATTR_GEN_AI_RESPONSE_ERROR, error.to_json)
span.status = OpenTelemetry::Trace::Status.error(error.to_s.truncate(1000))
end
end

58
lib/integrations/llm_instrumentation_helpers.rb
View File

@@ -2,38 +2,52 @@
module Integrations::LlmInstrumentationHelpers
include Integrations::LlmInstrumentationConstants
include Integrations::LlmInstrumentationCompletionHelpers
def determine_provider(model_name)
return 'openai' if model_name.blank?
model = model_name.to_s.downcase
LlmConstants::PROVIDER_PREFIXES.each do |provider, prefixes|
return provider if prefixes.any? { |prefix| model.start_with?(prefix) }
end
'openai'
end
private
def set_completion_attributes(span, result)
set_completion_message(span, result)
set_usage_metrics(span, result)
set_error_attributes(span, result)
def setup_span_attributes(span, params)
set_request_attributes(span, params)
set_prompt_messages(span, params[:messages])
set_metadata_attributes(span, params)
end
def set_completion_message(span, result)
message = result[:message] || result.dig('choices', 0, 'message', 'content')
return if message.blank?
span.set_attribute(ATTR_GEN_AI_COMPLETION_ROLE, 'assistant')
span.set_attribute(ATTR_GEN_AI_COMPLETION_CONTENT, message)
def record_completion(span, result)
if result.respond_to?(:content)
span.set_attribute(ATTR_GEN_AI_COMPLETION_ROLE, result.role.to_s) if result.respond_to?(:role)
span.set_attribute(ATTR_GEN_AI_COMPLETION_CONTENT, result.content.to_s)
elsif result.is_a?(Hash)
set_completion_attributes(span, result)
end
end
def set_usage_metrics(span, result)
usage = result[:usage] || result['usage']
return if usage.blank?
span.set_attribute(ATTR_GEN_AI_USAGE_INPUT_TOKENS, usage['prompt_tokens']) if usage['prompt_tokens']
span.set_attribute(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, usage['completion_tokens']) if usage['completion_tokens']
span.set_attribute(ATTR_GEN_AI_USAGE_TOTAL_TOKENS, usage['total_tokens']) if usage['total_tokens']
def set_request_attributes(span, params)
provider = determine_provider(params[:model])
span.set_attribute(ATTR_GEN_AI_PROVIDER, provider)
span.set_attribute(ATTR_GEN_AI_REQUEST_MODEL, params[:model])
span.set_attribute(ATTR_GEN_AI_REQUEST_TEMPERATURE, params[:temperature]) if params[:temperature]
end
def set_error_attributes(span, result)
error = result[:error] || result['error']
return if error.blank?
def set_prompt_messages(span, messages)
messages.each_with_index do |msg, idx|
role = msg[:role] || msg['role']
content = msg[:content] || msg['content']
span.set_attribute(ATTR_GEN_AI_RESPONSE_ERROR, error.to_json)
span.status = OpenTelemetry::Trace::Status.error(error.to_s.truncate(1000))
span.set_attribute(format(ATTR_GEN_AI_PROMPT_ROLE, idx), role)
span.set_attribute(format(ATTR_GEN_AI_PROMPT_CONTENT, idx), content.to_s)
end
end
def set_metadata_attributes(span, params)

1
lib/integrations/llm_instrumentation_spans.rb
View File

@@ -1,7 +1,6 @@
# frozen_string_literal: true
require 'opentelemetry_config'
require_relative 'llm_instrumentation_constants'
module Integrations::LlmInstrumentationSpans
include Integrations::LlmInstrumentationConstants

27
lib/integrations/openai/processor_service.rb
View File

@@ -77,21 +77,22 @@ class Integrations::Openai::ProcessorService < Integrations::LlmBaseService
end
def add_message_if_within_limit(character_count, message, messages, in_array_format)
if valid_message?(message, character_count)
add_message_to_list(message, messages, in_array_format)
character_count += message.content.length
content = message.content_for_llm
if valid_message?(content, character_count)
add_message_to_list(message, messages, in_array_format, content)
character_count += content.length
[character_count, true]
else
[character_count, false]
end
end
def valid_message?(message, character_count)
message.content.present? && character_count + message.content.length <= TOKEN_LIMIT
def valid_message?(content, character_count)
content.present? && character_count + content.length <= TOKEN_LIMIT
end
def add_message_to_list(message, messages, in_array_format)
formatted_message = format_message(message, in_array_format)
def add_message_to_list(message, messages, in_array_format, content)
formatted_message = format_message(message, in_array_format, content)
messages.prepend(formatted_message)
end
@@ -99,17 +100,17 @@ class Integrations::Openai::ProcessorService < Integrations::LlmBaseService
in_array_format ? [] : ''
end
def format_message(message, in_array_format)
in_array_format ? format_message_in_array(message) : format_message_in_string(message)
def format_message(message, in_array_format, content)
in_array_format ? format_message_in_array(message, content) : format_message_in_string(message, content)
end
def format_message_in_array(message)
{ role: (message.incoming? ? 'user' : 'assistant'), content: message.content }
def format_message_in_array(message, content)
{ role: (message.incoming? ? 'user' : 'assistant'), content: content }
end
def format_message_in_string(message)
def format_message_in_string(message, content)
sender_type = message.incoming? ? 'Customer' : 'Agent'
"#{sender_type} #{message.sender&.name} : #{message.content}\n"
"#{sender_type} #{message.sender&.name} : #{content}\n"
end
def summarize_body

17
lib/llm_constants.rb Normal file
View File

@@ -0,0 +1,17 @@
# frozen_string_literal: true
module LlmConstants
DEFAULT_MODEL = 'gpt-4.1-mini'
DEFAULT_EMBEDDING_MODEL = 'text-embedding-3-small'
PDF_PROCESSING_MODEL = 'gpt-4.1-mini'
OPENAI_API_ENDPOINT = 'https://api.openai.com'
PROVIDER_PREFIXES = {
'openai' => %w[gpt- o1 o3 o4 text-embedding- whisper- tts-],
'anthropic' => %w[claude-],
'google' => %w[gemini-],
'mistral' => %w[mistral- codestral-],
'deepseek' => %w[deepseek-]
}.freeze
end

8
lib/open_ai_constants.rb
View File

@@ -1,8 +0,0 @@
# frozen_string_literal: true
module OpenAiConstants
DEFAULT_MODEL = 'gpt-4.1-mini'
DEFAULT_ENDPOINT = 'https://api.openai.com'
DEFAULT_EMBEDDING_MODEL = 'text-embedding-3-small'
PDF_PROCESSING_MODEL = 'gpt-4.1-mini'
end

6
spec/enterprise/jobs/captain/documents/response_builder_job_spec.rb
View File

@@ -13,7 +13,7 @@ RSpec.describe Captain::Documents::ResponseBuilderJob, type: :job do
before do
allow(Captain::Llm::FaqGeneratorService).to receive(:new)
.with(document.content, document.account.locale_english_name)
.with(document.content, document.account.locale_english_name, account_id: document.account_id)
.and_return(faq_generator)
allow(faq_generator).to receive(:generate).and_return(faqs)
end
@@ -52,7 +52,7 @@ RSpec.describe Captain::Documents::ResponseBuilderJob, type: :job do
before do
allow(Captain::Llm::FaqGeneratorService).to receive(:new)
.with(spanish_document.content, 'portuguese')
.with(spanish_document.content, 'portuguese', account_id: spanish_document.account_id)
.and_return(spanish_faq_generator)
allow(spanish_faq_generator).to receive(:generate).and_return(faqs)
end
@@ -61,7 +61,7 @@ RSpec.describe Captain::Documents::ResponseBuilderJob, type: :job do
described_class.new.perform(spanish_document)
expect(Captain::Llm::FaqGeneratorService).to have_received(:new)
.with(spanish_document.content, 'portuguese')
.with(spanish_document.content, 'portuguese', account_id: spanish_document.account_id)
end
end

130
spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb
View File

@@ -4,49 +4,42 @@ RSpec.describe Captain::Llm::ConversationFaqService do
let(:captain_assistant) { create(:captain_assistant) }
let(:conversation) { create(:conversation, first_reply_created_at: Time.zone.now) }
let(:service) { described_class.new(captain_assistant, conversation) }
let(:client) { instance_double(OpenAI::Client) }
let(:embedding_service) { instance_double(Captain::Llm::EmbeddingService) }
let(:mock_chat) { instance_double(RubyLLM::Chat) }
let(:sample_faqs) do
[
{ 'question' => 'What is the purpose?', 'answer' => 'To help users.' },
{ 'question' => 'How does it work?', 'answer' => 'Through AI.' }
]
end
let(:mock_response) do
instance_double(RubyLLM::Message, content: { faqs: sample_faqs }.to_json)
end
before do
create(:installation_config) { create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key') }
allow(OpenAI::Client).to receive(:new).and_return(client)
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
allow(Captain::Llm::EmbeddingService).to receive(:new).and_return(embedding_service)
allow(RubyLLM).to receive(:chat).and_return(mock_chat)
allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
allow(mock_chat).to receive(:with_params).and_return(mock_chat)
allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
describe '#generate_and_deduplicate' do
let(:sample_faqs) do
[
{ 'question' => 'What is the purpose?', 'answer' => 'To help users.' },
{ 'question' => 'How does it work?', 'answer' => 'Through AI.' }
]
end
let(:openai_response) do
{
'choices' => [
{
'message' => {
'content' => { faqs: sample_faqs }.to_json
}
}
]
}
end
context 'when successful' do
before do
allow(client).to receive(:chat).and_return(openai_response)
allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3])
allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([])
end
it 'creates new FAQs' do
it 'creates new FAQs for valid conversation content' do
expect do
service.generate_and_deduplicate
end.to change(captain_assistant.responses, :count).by(2)
end
it 'saves the correct FAQ content' do
it 'saves FAQs with pending status linked to conversation' do
service.generate_and_deduplicate
expect(
captain_assistant.responses.pluck(:question, :answer, :status, :documentable_id)
@@ -63,6 +56,11 @@ RSpec.describe Captain::Llm::ConversationFaqService do
it 'returns an empty array without generating FAQs' do
expect(service.generate_and_deduplicate).to eq([])
end
it 'does not call the LLM API' do
expect(RubyLLM).not_to receive(:chat)
service.generate_and_deduplicate
end
end
context 'when finding duplicates' do
@@ -70,9 +68,6 @@ RSpec.describe Captain::Llm::ConversationFaqService do
create(:captain_assistant_response, assistant: captain_assistant, question: 'Similar question', answer: 'Similar answer')
end
let(:similar_neighbor) do
# Using OpenStruct here to mock as the Captain:AssistantResponse does not implement
# neighbor_distance as a method or attribute rather it is returned directly
# from SQL query in neighbor gem
OpenStruct.new(
id: 1,
question: existing_response.question,
@@ -82,87 +77,78 @@ RSpec.describe Captain::Llm::ConversationFaqService do
end
before do
allow(client).to receive(:chat).and_return(openai_response)
allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3])
allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([similar_neighbor])
end
it 'filters out duplicate FAQs' do
it 'filters out duplicate FAQs based on embedding similarity' do
expect do
service.generate_and_deduplicate
end.not_to change(captain_assistant.responses, :count)
end
end
context 'when OpenAI API fails' do
context 'when LLM API fails' do
before do
allow(client).to receive(:chat).and_raise(OpenAI::Error.new('API Error'))
allow(mock_chat).to receive(:ask).and_raise(RubyLLM::Error.new(nil, 'API Error'))
allow(Rails.logger).to receive(:error)
end
it 'handles the error and returns empty array' do
expect(Rails.logger).to receive(:error).with('OpenAI API Error: API Error')
it 'returns empty array and logs the error' do
expect(Rails.logger).to receive(:error).with('LLM API Error: API Error')
expect(service.generate_and_deduplicate).to eq([])
end
end
context 'when JSON parsing fails' do
let(:invalid_response) do
{
'choices' => [
{
'message' => {
'content' => 'invalid json'
}
}
]
}
instance_double(RubyLLM::Message, content: 'invalid json')
end
before do
allow(client).to receive(:chat).and_return(invalid_response)
allow(mock_chat).to receive(:ask).and_return(invalid_response)
end
it 'handles JSON parsing errors' do
it 'handles JSON parsing errors gracefully' do
expect(Rails.logger).to receive(:error).with(/Error in parsing GPT processed response:/)
expect(service.generate_and_deduplicate).to eq([])
end
end
context 'when response content is nil' do
let(:nil_response) do
instance_double(RubyLLM::Message, content: nil)
end
before do
allow(mock_chat).to receive(:ask).and_return(nil_response)
end
it 'returns empty array' do
expect(service.generate_and_deduplicate).to eq([])
end
end
end
describe '#chat_parameters' do
it 'includes correct model and response format' do
params = service.send(:chat_parameters)
expect(params[:model]).to eq('gpt-4o-mini')
expect(params[:response_format]).to eq({ type: 'json_object' })
end
it 'includes system prompt and conversation content' do
allow(Captain::Llm::SystemPromptsService).to receive(:conversation_faq_generator).and_return('system prompt')
params = service.send(:chat_parameters)
expect(params[:messages]).to include(
{ role: 'system', content: 'system prompt' },
{ role: 'user', content: conversation.to_llm_text }
)
end
describe 'language handling' do
context 'when conversation has different language' do
let(:account) { create(:account, locale: 'fr') }
let(:conversation) do
create(:conversation, account: account,
first_reply_created_at: Time.zone.now)
create(:conversation, account: account, first_reply_created_at: Time.zone.now)
end
it 'includes system prompt with correct language' do
allow(Captain::Llm::SystemPromptsService).to receive(:conversation_faq_generator)
before do
allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3])
allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([])
end
it 'uses account language for system prompt' do
expect(Captain::Llm::SystemPromptsService).to receive(:conversation_faq_generator)
.with('french')
.and_return('system prompt in french')
.at_least(:once)
.and_call_original
params = service.send(:chat_parameters)
expect(params[:messages]).to include(
{ role: 'system', content: 'system prompt in french' }
)
service.generate_and_deduplicate
end
end
end

110
spec/enterprise/services/captain/llm/faq_generator_service_spec.rb
View File

@@ -4,58 +4,40 @@ RSpec.describe Captain::Llm::FaqGeneratorService do
let(:content) { 'Sample content for FAQ generation' }
let(:language) { 'english' }
let(:service) { described_class.new(content, language) }
let(:client) { instance_double(OpenAI::Client) }
let(:mock_chat) { instance_double(RubyLLM::Chat) }
let(:sample_faqs) do
[
{ 'question' => 'What is this service?', 'answer' => 'It generates FAQs.' },
{ 'question' => 'How does it work?', 'answer' => 'Using AI technology.' }
]
end
let(:mock_response) do
instance_double(RubyLLM::Message, content: { faqs: sample_faqs }.to_json)
end
before do
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
allow(OpenAI::Client).to receive(:new).and_return(client)
allow(RubyLLM).to receive(:chat).and_return(mock_chat)
allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
allow(mock_chat).to receive(:with_params).and_return(mock_chat)
allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
describe '#generate' do
let(:sample_faqs) do
[
{ 'question' => 'What is this service?', 'answer' => 'It generates FAQs.' },
{ 'question' => 'How does it work?', 'answer' => 'Using AI technology.' }
]
end
let(:openai_response) do
{
'choices' => [
{
'message' => {
'content' => { faqs: sample_faqs }.to_json
}
}
]
}
end
context 'when successful' do
before do
allow(client).to receive(:chat).and_return(openai_response)
allow(Captain::Llm::SystemPromptsService).to receive(:faq_generator).and_return('system prompt')
end
it 'returns parsed FAQs' do
it 'returns parsed FAQs from the LLM response' do
result = service.generate
expect(result).to eq(sample_faqs)
end
it 'calls OpenAI client with chat parameters' do
expect(client).to receive(:chat).with(parameters: hash_including(
model: 'gpt-4o-mini',
response_format: { type: 'json_object' },
messages: array_including(
hash_including(role: 'system'),
hash_including(role: 'user', content: content)
)
))
it 'sends content to LLM with JSON response format' do
expect(mock_chat).to receive(:with_params).with(response_format: { type: 'json_object' }).and_return(mock_chat)
service.generate
end
it 'calls SystemPromptsService with correct language' do
expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with(language)
it 'uses SystemPromptsService with the specified language' do
expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with(language).at_least(:once).and_call_original
service.generate
end
end
@@ -63,23 +45,57 @@ RSpec.describe Captain::Llm::FaqGeneratorService do
context 'with different language' do
let(:language) { 'spanish' }
before do
allow(client).to receive(:chat).and_return(openai_response)
end
it 'passes the correct language to SystemPromptsService' do
expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with('spanish')
expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with('spanish').at_least(:once).and_call_original
service.generate
end
end
context 'when OpenAI API fails' do
context 'when LLM API fails' do
before do
allow(client).to receive(:chat).and_raise(OpenAI::Error.new('API Error'))
allow(mock_chat).to receive(:ask).and_raise(RubyLLM::Error.new(nil, 'API Error'))
allow(Rails.logger).to receive(:error)
end
it 'handles the error and returns empty array' do
expect(Rails.logger).to receive(:error).with('OpenAI API Error: API Error')
it 'returns empty array and logs the error' do
expect(Rails.logger).to receive(:error).with('LLM API Error: API Error')
expect(service.generate).to eq([])
end
end
context 'when response content is nil' do
let(:nil_response) { instance_double(RubyLLM::Message, content: nil) }
before do
allow(mock_chat).to receive(:ask).and_return(nil_response)
end
it 'returns empty array' do
expect(service.generate).to eq([])
end
end
context 'when JSON parsing fails' do
let(:invalid_response) { instance_double(RubyLLM::Message, content: 'invalid json') }
before do
allow(mock_chat).to receive(:ask).and_return(invalid_response)
end
it 'logs error and returns empty array' do
expect(Rails.logger).to receive(:error).with(/Error in parsing GPT processed response:/)
expect(service.generate).to eq([])
end
end
context 'when response is missing faqs key' do
let(:missing_key_response) { instance_double(RubyLLM::Message, content: '{"data": []}') }
before do
allow(mock_chat).to receive(:ask).and_return(missing_key_response)
end
it 'returns empty array via KeyError rescue' do
expect(service.generate).to eq([])
end
end

93
spec/enterprise/services/captain/onboarding/website_analyzer_service_spec.rb
View File

@@ -4,40 +4,38 @@ RSpec.describe Captain::Onboarding::WebsiteAnalyzerService do
let(:website_url) { 'https://example.com' }
let(:service) { described_class.new(website_url) }
let(:mock_crawler) { instance_double(Captain::Tools::SimplePageCrawlService) }
let(:mock_client) { instance_double(OpenAI::Client) }
let(:mock_chat) { instance_double(RubyLLM::Chat) }
let(:business_info) do
{
'business_name' => 'Example Corp',
'suggested_assistant_name' => 'Alex from Example Corp',
'description' => 'You specialize in helping customers with business solutions and support'
}
end
let(:mock_response) do
instance_double(RubyLLM::Message, content: business_info.to_json)
end
before do
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
allow(Captain::Tools::SimplePageCrawlService).to receive(:new).and_return(mock_crawler)
allow(service).to receive(:client).and_return(mock_client)
allow(service).to receive(:model).and_return('gpt-3.5-turbo')
allow(RubyLLM).to receive(:chat).and_return(mock_chat)
allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
allow(mock_chat).to receive(:with_params).and_return(mock_chat)
allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
describe '#analyze' do
context 'when website content is available and OpenAI call is successful' do
let(:openai_response) do
{
'choices' => [{
'message' => {
'content' => {
'business_name' => 'Example Corp',
'suggested_assistant_name' => 'Alex from Example Corp',
'description' => 'You specialize in helping customers with business solutions and support'
}.to_json
}
}]
}
end
context 'when website content is available and LLM call is successful' do
before do
allow(mock_crawler).to receive(:body_text_content).and_return('Welcome to Example Corp')
allow(mock_crawler).to receive(:page_title).and_return('Example Corp - Home')
allow(mock_crawler).to receive(:meta_description).and_return('Leading provider of business solutions')
allow(mock_crawler).to receive(:favicon_url).and_return('https://example.com/favicon.ico')
allow(mock_client).to receive(:chat).and_return(openai_response)
end
it 'returns success' do
it 'returns successful analysis with extracted business info' do
result = service.analyze
expect(result[:success]).to be true
@@ -49,14 +47,19 @@ RSpec.describe Captain::Onboarding::WebsiteAnalyzerService do
favicon_url: 'https://example.com/favicon.ico'
)
end
it 'uses low temperature for deterministic analysis' do
expect(mock_chat).to receive(:with_temperature).with(0.1).and_return(mock_chat)
service.analyze
end
end
context 'when website content is errored' do
context 'when website content fetch raises an error' do
before do
allow(mock_crawler).to receive(:body_text_content).and_raise(StandardError, 'Network error')
end
it 'returns error' do
it 'returns error response' do
result = service.analyze
expect(result[:success]).to be false
@@ -64,14 +67,14 @@ RSpec.describe Captain::Onboarding::WebsiteAnalyzerService do
end
end
context 'when website content is unavailable' do
context 'when website content is empty' do
before do
allow(mock_crawler).to receive(:body_text_content).and_return('')
allow(mock_crawler).to receive(:page_title).and_return('')
allow(mock_crawler).to receive(:meta_description).and_return('')
end
it 'returns error' do
it 'returns error for unavailable content' do
result = service.analyze
expect(result[:success]).to be false
@@ -79,21 +82,57 @@ RSpec.describe Captain::Onboarding::WebsiteAnalyzerService do
end
end
context 'when OpenAI error' do
context 'when LLM call fails' do
before do
allow(mock_crawler).to receive(:body_text_content).and_return('Welcome to Example Corp')
allow(mock_crawler).to receive(:page_title).and_return('Example Corp - Home')
allow(mock_crawler).to receive(:meta_description).and_return('Leading provider of business solutions')
allow(mock_crawler).to receive(:favicon_url).and_return('https://example.com/favicon.ico')
allow(mock_client).to receive(:chat).and_raise(StandardError, 'API error')
allow(mock_chat).to receive(:ask).and_raise(StandardError, 'API error')
end
it 'returns error' do
it 'returns error response with message' do
result = service.analyze
expect(result[:success]).to be false
expect(result[:error]).to eq('API error')
end
end
context 'when LLM returns invalid JSON' do
let(:invalid_response) { instance_double(RubyLLM::Message, content: 'not valid json') }
before do
allow(mock_crawler).to receive(:body_text_content).and_return('Welcome to Example Corp')
allow(mock_crawler).to receive(:page_title).and_return('Example Corp - Home')
allow(mock_crawler).to receive(:meta_description).and_return('Leading provider of business solutions')
allow(mock_crawler).to receive(:favicon_url).and_return('https://example.com/favicon.ico')
allow(mock_chat).to receive(:ask).and_return(invalid_response)
end
it 'returns error for parsing failure' do
result = service.analyze
expect(result[:success]).to be false
expect(result[:error]).to eq('Failed to parse business information from website')
end
end
context 'when URL normalization is needed' do
let(:website_url) { 'example.com' }
before do
allow(mock_crawler).to receive(:body_text_content).and_return('Welcome')
allow(mock_crawler).to receive(:page_title).and_return('Example')
allow(mock_crawler).to receive(:meta_description).and_return('Description')
allow(mock_crawler).to receive(:favicon_url).and_return(nil)
end
it 'normalizes URL by adding https prefix' do
result = service.analyze
expect(result[:data][:website_url]).to eq('https://example.com')
end
end
end
end

194
spec/enterprise/services/internal/account_analysis/content_evaluator_service_spec.rb
View File

@@ -3,60 +3,103 @@ require 'rails_helper'
RSpec.describe Internal::AccountAnalysis::ContentEvaluatorService do
let(:service) { described_class.new }
let(:content) { 'This is some test content' }
let(:mock_moderation_result) do
instance_double(
RubyLLM::Moderation,
flagged?: false,
flagged_categories: [],
category_scores: {}
)
end
before do
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
allow(RubyLLM).to receive(:moderate).and_return(mock_moderation_result)
end
describe '#evaluate' do
context 'when content is present' do
let(:llm_response) do
{
'choices' => [
{
'message' => {
'content' => {
'threat_level' => 'low',
'threat_summary' => 'No significant threats detected',
'detected_threats' => ['minor_concern'],
'illegal_activities_detected' => false,
'recommendation' => 'approve'
}.to_json
}
}
]
}
end
before do
allow(service).to receive(:send_to_llm).and_return(llm_response)
allow(Rails.logger).to receive(:info)
end
it 'returns the evaluation results' do
context 'when content is safe' do
it 'returns safe evaluation with approval recommendation' do
result = service.evaluate(content)
expect(result).to include(
'threat_level' => 'low',
'threat_summary' => 'No significant threats detected',
'detected_threats' => ['minor_concern'],
'threat_level' => 'safe',
'threat_summary' => 'No threats detected',
'detected_threats' => [],
'illegal_activities_detected' => false,
'recommendation' => 'approve'
)
end
it 'logs the evaluation results' do
expect(Rails.logger).to receive(:info).with('Moderation evaluation - Level: safe, Threats: ')
service.evaluate(content)
end
end
expect(Rails.logger).to have_received(:info).with('LLM evaluation - Level: low, Illegal activities: false')
context 'when content is flagged' do
let(:mock_moderation_result) do
instance_double(
RubyLLM::Moderation,
flagged?: true,
flagged_categories: %w[harassment hate],
category_scores: { 'harassment' => 0.6, 'hate' => 0.3 }
)
end
it 'returns flagged evaluation with review recommendation' do
result = service.evaluate(content)
expect(result).to include(
'threat_level' => 'high',
'threat_summary' => 'Content flagged for: harassment, hate',
'detected_threats' => %w[harassment hate],
'illegal_activities_detected' => false,
'recommendation' => 'review'
)
end
end
context 'when content contains violence' do
let(:mock_moderation_result) do
instance_double(
RubyLLM::Moderation,
flagged?: true,
flagged_categories: ['violence'],
category_scores: { 'violence' => 0.9 }
)
end
it 'marks illegal activities detected for violence' do
result = service.evaluate(content)
expect(result['illegal_activities_detected']).to be true
expect(result['threat_level']).to eq('critical')
end
end
context 'when content contains self-harm' do
let(:mock_moderation_result) do
instance_double(
RubyLLM::Moderation,
flagged?: true,
flagged_categories: ['self-harm'],
category_scores: { 'self-harm' => 0.85 }
)
end
it 'marks illegal activities detected for self-harm' do
result = service.evaluate(content)
expect(result['illegal_activities_detected']).to be true
end
end
context 'when content is blank' do
let(:blank_content) { '' }
it 'returns the default evaluation without calling the LLM' do
expect(service).not_to receive(:send_to_llm)
it 'returns default evaluation without calling moderation API' do
expect(RubyLLM).not_to receive(:moderate)
result = service.evaluate(blank_content)
@@ -70,34 +113,16 @@ RSpec.describe Internal::AccountAnalysis::ContentEvaluatorService do
end
end
context 'when LLM response is nil' do
before do
allow(service).to receive(:send_to_llm).and_return(nil)
end
it 'returns the default evaluation' do
result = service.evaluate(content)
expect(result).to include(
'threat_level' => 'unknown',
'threat_summary' => 'Failed to complete content evaluation',
'detected_threats' => [],
'illegal_activities_detected' => false,
'recommendation' => 'review'
)
end
end
context 'when error occurs during evaluation' do
before do
allow(service).to receive(:send_to_llm).and_raise(StandardError.new('Test error'))
allow(Rails.logger).to receive(:error)
allow(RubyLLM).to receive(:moderate).and_raise(StandardError.new('Test error'))
end
it 'logs the error and returns default evaluation with error type' do
it 'logs error and returns default evaluation with error type' do
expect(Rails.logger).to receive(:error).with('Error evaluating content: Test error')
result = service.evaluate(content)
expect(Rails.logger).to have_received(:error).with('Error evaluating content: Test error')
expect(result).to include(
'threat_level' => 'unknown',
'threat_summary' => 'Failed to complete content evaluation',
@@ -107,5 +132,68 @@ RSpec.describe Internal::AccountAnalysis::ContentEvaluatorService do
)
end
end
context 'with threat level determination' do
it 'returns critical for scores >= 0.8' do
mock_result = instance_double(
RubyLLM::Moderation,
flagged?: true,
flagged_categories: ['harassment'],
category_scores: { 'harassment' => 0.85 }
)
allow(RubyLLM).to receive(:moderate).and_return(mock_result)
result = service.evaluate(content)
expect(result['threat_level']).to eq('critical')
end
it 'returns high for scores between 0.5 and 0.8' do
mock_result = instance_double(
RubyLLM::Moderation,
flagged?: true,
flagged_categories: ['harassment'],
category_scores: { 'harassment' => 0.65 }
)
allow(RubyLLM).to receive(:moderate).and_return(mock_result)
result = service.evaluate(content)
expect(result['threat_level']).to eq('high')
end
it 'returns medium for scores between 0.2 and 0.5' do
mock_result = instance_double(
RubyLLM::Moderation,
flagged?: true,
flagged_categories: ['harassment'],
category_scores: { 'harassment' => 0.35 }
)
allow(RubyLLM).to receive(:moderate).and_return(mock_result)
result = service.evaluate(content)
expect(result['threat_level']).to eq('medium')
end
it 'returns low for scores below 0.2' do
mock_result = instance_double(
RubyLLM::Moderation,
flagged?: true,
flagged_categories: ['harassment'],
category_scores: { 'harassment' => 0.15 }
)
allow(RubyLLM).to receive(:moderate).and_return(mock_result)
result = service.evaluate(content)
expect(result['threat_level']).to eq('low')
end
end
context 'with content truncation' do
let(:long_content) { 'a' * 15_000 }
it 'truncates content to 10000 characters before sending to moderation' do
expect(RubyLLM).to receive(:moderate).with('a' * 10_000).and_return(mock_moderation_result)
service.evaluate(long_content)
end
end
end
end