diff --git a/Gemfile.lock b/Gemfile.lock index b42472ef4..15ed841ac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -827,6 +827,7 @@ GEM faraday-net_http (>= 1) faraday-retry (>= 1) marcel (~> 1.0) + ruby_llm-schema (~> 0.2.1) zeitwerk (~> 2) ruby_llm-schema (0.2.5) ruby_parser (3.20.0) diff --git a/app/models/message.rb b/app/models/message.rb index 2faba1215..0bf7a176b 100644 --- a/app/models/message.rb +++ b/app/models/message.rb @@ -254,6 +254,21 @@ class Message < ApplicationRecord Messages::SearchDataPresenter.new(self).search_data end + # Returns message content suitable for LLM consumption + # Falls back to audio transcription or attachment placeholder when content is nil + def content_for_llm + return content if content.present? + + audio_transcription = attachments + .where(file_type: :audio) + .filter_map { |att| att.meta&.dig('transcribed_text') } + .join(' ') + .presence + return "[Voice Message] #{audio_transcription}" if audio_transcription.present? + + '[Attachment]' if attachments.any? + end + private def prevent_message_flooding diff --git a/app/services/llm_formatter/conversation_llm_formatter.rb b/app/services/llm_formatter/conversation_llm_formatter.rb index 4e0bd7013..38d7c9e26 100644 --- a/app/services/llm_formatter/conversation_llm_formatter.rb +++ b/app/services/llm_formatter/conversation_llm_formatter.rb @@ -48,7 +48,7 @@ class LlmFormatter::ConversationLlmFormatter < LlmFormatter::DefaultLlmFormatter 'Bot' end sender = "[Private Note] #{sender}" if message.private? - "#{sender}: #{message.content}\n" + "#{sender}: #{message.content_for_llm}\n" end def build_attributes diff --git a/config/initializers/ai_agents.rb b/config/initializers/ai_agents.rb index 37bdd589f..be8a8a9cc 100644 --- a/config/initializers/ai_agents.rb +++ b/config/initializers/ai_agents.rb @@ -4,8 +4,8 @@ require 'agents' Rails.application.config.after_initialize do api_key = InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_API_KEY')&.value - model = InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || OpenAiConstants::DEFAULT_MODEL - api_endpoint = InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_ENDPOINT')&.value || OpenAiConstants::DEFAULT_ENDPOINT + model = InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || LlmConstants::DEFAULT_MODEL + api_endpoint = InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_ENDPOINT')&.value || LlmConstants::OPENAI_API_ENDPOINT if api_key.present? Agents.configure do |config| diff --git a/enterprise/app/controllers/enterprise/public/api/v1/portals/articles_controller.rb b/enterprise/app/controllers/enterprise/public/api/v1/portals/articles_controller.rb index 240da0dca..77162dad0 100644 --- a/enterprise/app/controllers/enterprise/public/api/v1/portals/articles_controller.rb +++ b/enterprise/app/controllers/enterprise/public/api/v1/portals/articles_controller.rb @@ -3,7 +3,7 @@ module Enterprise::Public::Api::V1::Portals::ArticlesController def search_articles if @portal.account.feature_enabled?('help_center_embedding_search') - @articles = @articles.vector_search(list_params) if list_params[:query].present? + @articles = @articles.vector_search(list_params.merge(account_id: @portal.account_id)) if list_params[:query].present? else super end diff --git a/enterprise/app/helpers/captain/chat_helper.rb b/enterprise/app/helpers/captain/chat_helper.rb index a60e3015f..9bbbd86a5 100644 --- a/enterprise/app/helpers/captain/chat_helper.rb +++ b/enterprise/app/helpers/captain/chat_helper.rb @@ -21,20 +21,18 @@ module Captain::ChatHelper def build_chat llm_chat = chat(model: @model, temperature: temperature) - llm_chat.with_params(response_format: { type: 'json_object' }) + llm_chat = llm_chat.with_params(response_format: { type: 'json_object' }) llm_chat = setup_tools(llm_chat) - setup_system_instructions(llm_chat) + llm_chat = setup_system_instructions(llm_chat) setup_event_handlers(llm_chat) - - llm_chat end - def setup_tools(chat) + def setup_tools(llm_chat) @tools&.each do |tool| - chat.with_tool(tool) + llm_chat = llm_chat.with_tool(tool) end - chat + llm_chat end def setup_system_instructions(chat) diff --git a/enterprise/app/jobs/captain/documents/response_builder_job.rb b/enterprise/app/jobs/captain/documents/response_builder_job.rb index b22fa5bc1..553cec110 100644 --- a/enterprise/app/jobs/captain/documents/response_builder_job.rb +++ b/enterprise/app/jobs/captain/documents/response_builder_job.rb @@ -26,7 +26,7 @@ class Captain::Documents::ResponseBuilderJob < ApplicationJob end def generate_standard_faqs(document) - Captain::Llm::FaqGeneratorService.new(document.content, document.account.locale_english_name).generate + Captain::Llm::FaqGeneratorService.new(document.content, document.account.locale_english_name, account_id: document.account_id).generate end def build_paginated_service(document, options) diff --git a/enterprise/app/jobs/captain/llm/update_embedding_job.rb b/enterprise/app/jobs/captain/llm/update_embedding_job.rb index 20d10f8f5..72ab540bd 100644 --- a/enterprise/app/jobs/captain/llm/update_embedding_job.rb +++ b/enterprise/app/jobs/captain/llm/update_embedding_job.rb @@ -2,7 +2,8 @@ class Captain::Llm::UpdateEmbeddingJob < ApplicationJob queue_as :low def perform(record, content) - embedding = Captain::Llm::EmbeddingService.new.get_embedding(content) + account_id = record.account_id + embedding = Captain::Llm::EmbeddingService.new(account_id: account_id).get_embedding(content) record.update!(embedding: embedding) end end diff --git a/enterprise/app/models/article_embedding.rb b/enterprise/app/models/article_embedding.rb index 14665a24b..4dafcfe06 100644 --- a/enterprise/app/models/article_embedding.rb +++ b/enterprise/app/models/article_embedding.rb @@ -19,6 +19,8 @@ class ArticleEmbedding < ApplicationRecord after_commit :update_response_embedding + delegate :account_id, to: :article + private def update_response_embedding diff --git a/enterprise/app/models/captain/assistant_response.rb b/enterprise/app/models/captain/assistant_response.rb index 7ab2878a3..12dcab1cc 100644 --- a/enterprise/app/models/captain/assistant_response.rb +++ b/enterprise/app/models/captain/assistant_response.rb @@ -44,8 +44,8 @@ class Captain::AssistantResponse < ApplicationRecord enum status: { pending: 0, approved: 1 } - def self.search(query) - embedding = Captain::Llm::EmbeddingService.new.get_embedding(query) + def self.search(query, account_id: nil) + embedding = Captain::Llm::EmbeddingService.new(account_id: account_id).get_embedding(query) nearest_neighbors(:embedding, embedding, distance: 'cosine').limit(5) end diff --git a/enterprise/app/models/concerns/agentable.rb b/enterprise/app/models/concerns/agentable.rb index dab76a726..e5b0b8eef 100644 --- a/enterprise/app/models/concerns/agentable.rb +++ b/enterprise/app/models/concerns/agentable.rb @@ -43,7 +43,7 @@ module Concerns::Agentable end def agent_model - InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || OpenAiConstants::DEFAULT_MODEL + InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || LlmConstants::DEFAULT_MODEL end def agent_response_schema diff --git a/enterprise/app/models/enterprise/concerns/article.rb b/enterprise/app/models/enterprise/concerns/article.rb index d3a94d7b7..4527bfdf6 100644 --- a/enterprise/app/models/enterprise/concerns/article.rb +++ b/enterprise/app/models/enterprise/concerns/article.rb @@ -11,7 +11,7 @@ module Enterprise::Concerns::Article add_article_embedding_association def self.vector_search(params) - embedding = Captain::Llm::EmbeddingService.new.get_embedding(params['query']) + embedding = Captain::Llm::EmbeddingService.new(account_id: params[:account_id]).get_embedding(params['query']) records = joins( :category ).search_by_category_slug( diff --git a/enterprise/app/services/captain/llm/contact_attributes_service.rb b/enterprise/app/services/captain/llm/contact_attributes_service.rb index f2c48de57..803c06f09 100644 --- a/enterprise/app/services/captain/llm/contact_attributes_service.rb +++ b/enterprise/app/services/captain/llm/contact_attributes_service.rb @@ -1,4 +1,5 @@ -class Captain::Llm::ContactAttributesService < Llm::LegacyBaseOpenAiService +class Captain::Llm::ContactAttributesService < Llm::BaseAiService + include Integrations::LlmInstrumentation def initialize(assistant, conversation) super() @assistant = assistant @@ -17,33 +18,38 @@ class Captain::Llm::ContactAttributesService < Llm::LegacyBaseOpenAiService attr_reader :content def generate_attributes - response = @client.chat(parameters: chat_parameters) - parse_response(response) - rescue OpenAI::Error => e - Rails.logger.error "OpenAI API Error: #{e.message}" + response = instrument_llm_call(instrumentation_params) do + chat + .with_params(response_format: { type: 'json_object' }) + .with_instructions(system_prompt) + .ask(@content) + end + parse_response(response.content) + rescue RubyLLM::Error => e + ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception [] end - def chat_parameters - prompt = Captain::Llm::SystemPromptsService.attributes_generator + def instrumentation_params { + span_name: 'llm.captain.contact_attributes', model: @model, - response_format: { type: 'json_object' }, + temperature: @temperature, + account_id: @conversation.account_id, + feature_name: 'contact_attributes', messages: [ - { - role: 'system', - content: prompt - }, - { - role: 'user', - content: content - } - ] + { role: 'system', content: system_prompt }, + { role: 'user', content: @content } + ], + metadata: { assistant_id: @assistant.id, contact_id: @contact.id } } end - def parse_response(response) - content = response.dig('choices', 0, 'message', 'content') + def system_prompt + Captain::Llm::SystemPromptsService.attributes_generator + end + + def parse_response(content) return [] if content.nil? JSON.parse(content.strip).fetch('attributes', []) diff --git a/enterprise/app/services/captain/llm/contact_notes_service.rb b/enterprise/app/services/captain/llm/contact_notes_service.rb index 4ee4fcb5e..d37f9fea6 100644 --- a/enterprise/app/services/captain/llm/contact_notes_service.rb +++ b/enterprise/app/services/captain/llm/contact_notes_service.rb @@ -1,4 +1,5 @@ -class Captain::Llm::ContactNotesService < Llm::LegacyBaseOpenAiService +class Captain::Llm::ContactNotesService < Llm::BaseAiService + include Integrations::LlmInstrumentation def initialize(assistant, conversation) super() @assistant = assistant @@ -18,38 +19,42 @@ class Captain::Llm::ContactNotesService < Llm::LegacyBaseOpenAiService attr_reader :content def generate_notes - response = @client.chat(parameters: chat_parameters) - parse_response(response) - rescue OpenAI::Error => e - Rails.logger.error "OpenAI API Error: #{e.message}" + response = instrument_llm_call(instrumentation_params) do + chat + .with_params(response_format: { type: 'json_object' }) + .with_instructions(system_prompt) + .ask(@content) + end + parse_response(response.content) + rescue RubyLLM::Error => e + ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception [] end - def chat_parameters - account_language = @conversation.account.locale_english_name - prompt = Captain::Llm::SystemPromptsService.notes_generator(account_language) - + def instrumentation_params { + span_name: 'llm.captain.contact_notes', model: @model, - response_format: { type: 'json_object' }, + temperature: @temperature, + account_id: @conversation.account_id, + feature_name: 'contact_notes', messages: [ - { - role: 'system', - content: prompt - }, - { - role: 'user', - content: content - } - ] + { role: 'system', content: system_prompt }, + { role: 'user', content: @content } + ], + metadata: { assistant_id: @assistant.id, contact_id: @contact.id } } end - def parse_response(response) - content = response.dig('choices', 0, 'message', 'content') - return [] if content.nil? + def system_prompt + account_language = @conversation.account.locale_english_name + Captain::Llm::SystemPromptsService.notes_generator(account_language) + end - JSON.parse(content.strip).fetch('notes', []) + def parse_response(response) + return [] if response.nil? + + JSON.parse(response.strip).fetch('notes', []) rescue JSON::ParserError => e Rails.logger.error "Error in parsing GPT processed response: #{e.message}" [] diff --git a/enterprise/app/services/captain/llm/conversation_faq_service.rb b/enterprise/app/services/captain/llm/conversation_faq_service.rb index 47d1433bd..e9152bc99 100644 --- a/enterprise/app/services/captain/llm/conversation_faq_service.rb +++ b/enterprise/app/services/captain/llm/conversation_faq_service.rb @@ -1,4 +1,5 @@ -class Captain::Llm::ConversationFaqService < Llm::LegacyBaseOpenAiService +class Captain::Llm::ConversationFaqService < Llm::BaseAiService + include Integrations::LlmInstrumentation DISTANCE_THRESHOLD = 0.3 def initialize(assistant, conversation) @@ -35,7 +36,7 @@ class Captain::Llm::ConversationFaqService < Llm::LegacyBaseOpenAiService faqs.each do |faq| combined_text = "#{faq['question']}: #{faq['answer']}" - embedding = Captain::Llm::EmbeddingService.new.get_embedding(combined_text) + embedding = Captain::Llm::EmbeddingService.new(account_id: @conversation.account_id).get_embedding(combined_text) similar_faqs = find_similar_faqs(embedding) if similar_faqs.any? @@ -81,38 +82,43 @@ class Captain::Llm::ConversationFaqService < Llm::LegacyBaseOpenAiService end def generate - response = @client.chat(parameters: chat_parameters) - parse_response(response) - rescue OpenAI::Error => e - Rails.logger.error "OpenAI API Error: #{e.message}" + response = instrument_llm_call(instrumentation_params) do + chat + .with_params(response_format: { type: 'json_object' }) + .with_instructions(system_prompt) + .ask(@content) + end + parse_response(response.content) + rescue RubyLLM::Error => e + Rails.logger.error "LLM API Error: #{e.message}" [] end - def chat_parameters - account_language = @conversation.account.locale_english_name - prompt = Captain::Llm::SystemPromptsService.conversation_faq_generator(account_language) - + def instrumentation_params { + span_name: 'llm.captain.conversation_faq', model: @model, - response_format: { type: 'json_object' }, + temperature: @temperature, + account_id: @conversation.account_id, + conversation_id: @conversation.id, + feature_name: 'conversation_faq', messages: [ - { - role: 'system', - content: prompt - }, - { - role: 'user', - content: content - } - ] + { role: 'system', content: system_prompt }, + { role: 'user', content: @content } + ], + metadata: { assistant_id: @assistant.id } } end - def parse_response(response) - content = response.dig('choices', 0, 'message', 'content') - return [] if content.nil? + def system_prompt + account_language = @conversation.account.locale_english_name + Captain::Llm::SystemPromptsService.conversation_faq_generator(account_language) + end - JSON.parse(content.strip).fetch('faqs', []) + def parse_response(response) + return [] if response.nil? + + JSON.parse(response.strip).fetch('faqs', []) rescue JSON::ParserError => e Rails.logger.error "Error in parsing GPT processed response: #{e.message}" [] diff --git a/enterprise/app/services/captain/llm/embedding_service.rb b/enterprise/app/services/captain/llm/embedding_service.rb index 5190ed28a..2fac54594 100644 --- a/enterprise/app/services/captain/llm/embedding_service.rb +++ b/enterprise/app/services/captain/llm/embedding_service.rb @@ -1,22 +1,38 @@ -require 'openai' +class Captain::Llm::EmbeddingService + include Integrations::LlmInstrumentation -class Captain::Llm::EmbeddingService < Llm::LegacyBaseOpenAiService class EmbeddingsError < StandardError; end - def self.embedding_model - @embedding_model = InstallationConfig.find_by(name: 'CAPTAIN_EMBEDDING_MODEL')&.value.presence || OpenAiConstants::DEFAULT_EMBEDDING_MODEL + def initialize(account_id: nil) + Llm::Config.initialize! + @account_id = account_id + @embedding_model = InstallationConfig.find_by(name: 'CAPTAIN_EMBEDDING_MODEL')&.value.presence || LlmConstants::DEFAULT_EMBEDDING_MODEL end - def get_embedding(content, model: self.class.embedding_model) - response = @client.embeddings( - parameters: { - model: model, - input: content - } - ) + def self.embedding_model + InstallationConfig.find_by(name: 'CAPTAIN_EMBEDDING_MODEL')&.value.presence || LlmConstants::DEFAULT_EMBEDDING_MODEL + end - response.dig('data', 0, 'embedding') - rescue StandardError => e + def get_embedding(content, model: @embedding_model) + return [] if content.blank? + + instrument_embedding_call(instrumentation_params(content, model)) do + RubyLLM.embed(content, model: model).vectors + end + rescue RubyLLM::Error => e + Rails.logger.error "Embedding API Error: #{e.message}" raise EmbeddingsError, "Failed to create an embedding: #{e.message}" end + + private + + def instrumentation_params(content, model) + { + span_name: 'llm.captain.embedding', + model: model, + input: content, + feature_name: 'embedding', + account_id: @account_id + } + end end diff --git a/enterprise/app/services/captain/llm/faq_generator_service.rb b/enterprise/app/services/captain/llm/faq_generator_service.rb index 634385b36..b22a631b3 100644 --- a/enterprise/app/services/captain/llm/faq_generator_service.rb +++ b/enterprise/app/services/captain/llm/faq_generator_service.rb @@ -1,15 +1,24 @@ -class Captain::Llm::FaqGeneratorService < Llm::LegacyBaseOpenAiService - def initialize(content, language = 'english') +class Captain::Llm::FaqGeneratorService < Llm::BaseAiService + include Integrations::LlmInstrumentation + + def initialize(content, language = 'english', account_id: nil) super() @language = language @content = content + @account_id = account_id end def generate - response = @client.chat(parameters: chat_parameters) - parse_response(response) - rescue OpenAI::Error => e - Rails.logger.error "OpenAI API Error: #{e.message}" + response = instrument_llm_call(instrumentation_params) do + chat + .with_params(response_format: { type: 'json_object' }) + .with_instructions(system_prompt) + .ask(@content) + end + + parse_response(response.content) + rescue RubyLLM::Error => e + Rails.logger.error "LLM API Error: #{e.message}" [] end @@ -17,26 +26,25 @@ class Captain::Llm::FaqGeneratorService < Llm::LegacyBaseOpenAiService attr_reader :content, :language - def chat_parameters - prompt = Captain::Llm::SystemPromptsService.faq_generator(language) + def system_prompt + Captain::Llm::SystemPromptsService.faq_generator(language) + end + + def instrumentation_params { + span_name: 'llm.captain.faq_generator', model: @model, - response_format: { type: 'json_object' }, + temperature: @temperature, + feature_name: 'faq_generator', + account_id: @account_id, messages: [ - { - role: 'system', - content: prompt - }, - { - role: 'user', - content: content - } + { role: 'system', content: system_prompt }, + { role: 'user', content: @content } ] } end - def parse_response(response) - content = response.dig('choices', 0, 'message', 'content') + def parse_response(content) return [] if content.nil? JSON.parse(content.strip).fetch('faqs', []) diff --git a/enterprise/app/services/captain/llm/paginated_faq_generator_service.rb b/enterprise/app/services/captain/llm/paginated_faq_generator_service.rb index ebfef4b5c..149152107 100644 --- a/enterprise/app/services/captain/llm/paginated_faq_generator_service.rb +++ b/enterprise/app/services/captain/llm/paginated_faq_generator_service.rb @@ -1,4 +1,6 @@ class Captain::Llm::PaginatedFaqGeneratorService < Llm::LegacyBaseOpenAiService + include Integrations::LlmInstrumentation + # Default pages per chunk - easily configurable DEFAULT_PAGES_PER_CHUNK = 10 MAX_ITERATIONS = 20 # Safety limit to prevent infinite loops @@ -13,7 +15,7 @@ class Captain::Llm::PaginatedFaqGeneratorService < Llm::LegacyBaseOpenAiService @max_pages = options[:max_pages] # Optional limit from UI @total_pages_processed = 0 @iterations_completed = 0 - @model = OpenAiConstants::PDF_PROCESSING_MODEL + @model = LlmConstants::PDF_PROCESSING_MODEL end def generate @@ -43,7 +45,19 @@ class Captain::Llm::PaginatedFaqGeneratorService < Llm::LegacyBaseOpenAiService private def generate_standard_faqs - response = @client.chat(parameters: standard_chat_parameters) + params = standard_chat_parameters + instrumentation_params = { + span_name: 'llm.faq_generation', + account_id: @document&.account_id, + feature_name: 'faq_generation', + model: @model, + messages: params[:messages] + } + + response = instrument_llm_call(instrumentation_params) do + @client.chat(parameters: params) + end + parse_response(response) rescue OpenAI::Error => e Rails.logger.error I18n.t('captain.documents.openai_api_error', error: e.message) @@ -84,7 +98,13 @@ class Captain::Llm::PaginatedFaqGeneratorService < Llm::LegacyBaseOpenAiService def process_page_chunk(start_page, end_page) params = build_chunk_parameters(start_page, end_page) - response = @client.chat(parameters: params) + + instrumentation_params = build_instrumentation_params(params, start_page, end_page) + + response = instrument_llm_call(instrumentation_params) do + @client.chat(parameters: params) + end + result = parse_chunk_response(response) { faqs: result['faqs'] || [], has_content: result['has_content'] != false } rescue OpenAI::Error => e @@ -180,21 +200,26 @@ class Captain::Llm::PaginatedFaqGeneratorService < Llm::LegacyBaseOpenAiService def similarity_score(str1, str2) words1 = str1.downcase.split(/\W+/).reject(&:empty?) words2 = str2.downcase.split(/\W+/).reject(&:empty?) - common_words = words1 & words2 total_words = (words1 + words2).uniq.size - return 0 if total_words.zero? common_words.size.to_f / total_words end - def determine_stop_reason(last_chunk_result) - return 'Maximum iterations reached' if @iterations_completed >= MAX_ITERATIONS - return 'Maximum pages processed' if @max_pages && @total_pages_processed >= @max_pages - return 'No content found in last chunk' if last_chunk_result[:faqs].empty? - return 'End of document reached' if last_chunk_result[:has_content] == false - - 'Unknown' + def build_instrumentation_params(params, start_page, end_page) + { + span_name: 'llm.paginated_faq_generation', + account_id: @document&.account_id, + feature_name: 'paginated_faq_generation', + model: @model, + messages: params[:messages], + metadata: { + document_id: @document&.id, + start_page: start_page, + end_page: end_page, + iteration: @iterations_completed + 1 + } + } end end diff --git a/enterprise/app/services/captain/llm/pdf_processing_service.rb b/enterprise/app/services/captain/llm/pdf_processing_service.rb index 55177b78d..82e3e9fbc 100644 --- a/enterprise/app/services/captain/llm/pdf_processing_service.rb +++ b/enterprise/app/services/captain/llm/pdf_processing_service.rb @@ -1,4 +1,6 @@ class Captain::Llm::PdfProcessingService < Llm::LegacyBaseOpenAiService + include Integrations::LlmInstrumentation + def initialize(document) super() @document = document @@ -19,13 +21,30 @@ class Captain::Llm::PdfProcessingService < Llm::LegacyBaseOpenAiService def upload_pdf_to_openai with_tempfile do |temp_file| - response = @client.files.upload( - parameters: { - file: temp_file, - purpose: 'assistants' - } - ) - response['id'] + instrument_file_upload do + response = @client.files.upload( + parameters: { + file: temp_file, + purpose: 'assistants' + } + ) + response['id'] + end + end + end + + def instrument_file_upload(&) + return yield unless ChatwootApp.otel_enabled? + + tracer.in_span('llm.file.upload') do |span| + span.set_attribute('gen_ai.provider', 'openai') + span.set_attribute('file.purpose', 'assistants') + span.set_attribute(ATTR_LANGFUSE_USER_ID, document.account_id.to_s) + span.set_attribute(ATTR_LANGFUSE_TAGS, ['pdf_upload'].to_json) + span.set_attribute(format(ATTR_LANGFUSE_METADATA, 'document_id'), document.id.to_s) + file_id = yield + span.set_attribute('file.id', file_id) if file_id + file_id end end diff --git a/enterprise/app/services/captain/onboarding/website_analyzer_service.rb b/enterprise/app/services/captain/onboarding/website_analyzer_service.rb index 02ba35571..a5879ba33 100644 --- a/enterprise/app/services/captain/onboarding/website_analyzer_service.rb +++ b/enterprise/app/services/captain/onboarding/website_analyzer_service.rb @@ -1,4 +1,5 @@ -class Captain::Onboarding::WebsiteAnalyzerService < Llm::LegacyBaseOpenAiService +class Captain::Onboarding::WebsiteAnalyzerService < Llm::BaseAiService + include Integrations::LlmInstrumentation MAX_CONTENT_LENGTH = 8000 def initialize(website_url) @@ -57,19 +58,29 @@ class Captain::Onboarding::WebsiteAnalyzerService < Llm::LegacyBaseOpenAiService end def extract_business_info - prompt = build_analysis_prompt + response = instrument_llm_call(instrumentation_params) do + chat + .with_params(response_format: { type: 'json_object' }, max_tokens: 1000) + .with_temperature(0.1) + .with_instructions(build_analysis_prompt) + .ask(@website_content) + end - response = client.chat( - parameters: { - model: model, - messages: [{ role: 'user', content: prompt }], - response_format: { type: 'json_object' }, - temperature: 0.1, - max_tokens: 1000 - } - ) + parse_llm_response(response.content) + end - parse_llm_response(response.dig('choices', 0, 'message', 'content')) + def instrumentation_params + { + span_name: 'llm.captain.website_analyzer', + model: @model, + temperature: 0.1, + feature_name: 'website_analyzer', + messages: [ + { role: 'system', content: build_analysis_prompt }, + { role: 'user', content: @website_content } + ], + metadata: { website_url: @website_url } + } end def build_analysis_prompt @@ -95,7 +106,7 @@ class Captain::Onboarding::WebsiteAnalyzerService < Llm::LegacyBaseOpenAiService end def parse_llm_response(response_text) - parsed_response = JSON.parse(response_text) + parsed_response = JSON.parse(response_text.strip) { success: true, diff --git a/enterprise/app/services/internal/account_analysis/content_evaluator_service.rb b/enterprise/app/services/internal/account_analysis/content_evaluator_service.rb index 586e9fad0..7be5647a8 100644 --- a/enterprise/app/services/internal/account_analysis/content_evaluator_service.rb +++ b/enterprise/app/services/internal/account_analysis/content_evaluator_service.rb @@ -1,48 +1,59 @@ -class Internal::AccountAnalysis::ContentEvaluatorService < Llm::LegacyBaseOpenAiService - def initialize - super() +class Internal::AccountAnalysis::ContentEvaluatorService + include Integrations::LlmInstrumentation - @model = 'gpt-4o-mini'.freeze + def initialize + Llm::Config.initialize! end def evaluate(content) return default_evaluation if content.blank? - begin - response = send_to_llm(content) - evaluation = handle_response(response) - log_evaluation_results(evaluation) - evaluation - rescue StandardError => e - handle_evaluation_error(e) + moderation_result = instrument_moderation_call(instrumentation_params(content)) do + RubyLLM.moderate(content.to_s[0...10_000]) end + + build_evaluation(moderation_result) + rescue StandardError => e + handle_evaluation_error(e) end private - def send_to_llm(content) - Rails.logger.info('Sending content to LLM for security evaluation') - @client.chat( - parameters: { - model: @model, - messages: llm_messages(content), - response_format: { type: 'json_object' } - } - ) + def instrumentation_params(content) + { + span_name: 'llm.internal.content_moderation', + model: 'text-moderation-latest', + input: content, + feature_name: 'content_evaluator' + } end - def handle_response(response) - return default_evaluation if response.nil? + def build_evaluation(result) + flagged = result.flagged? + categories = result.flagged_categories - parsed = JSON.parse(response.dig('choices', 0, 'message', 'content').strip) - - { - 'threat_level' => parsed['threat_level'] || 'unknown', - 'threat_summary' => parsed['threat_summary'] || 'No threat summary provided', - 'detected_threats' => parsed['detected_threats'] || [], - 'illegal_activities_detected' => parsed['illegal_activities_detected'] || false, - 'recommendation' => parsed['recommendation'] || 'review' + evaluation = { + 'threat_level' => flagged ? determine_threat_level(result) : 'safe', + 'threat_summary' => flagged ? "Content flagged for: #{categories.join(', ')}" : 'No threats detected', + 'detected_threats' => categories, + 'illegal_activities_detected' => categories.any? { |c| c.include?('violence') || c.include?('self-harm') }, + 'recommendation' => flagged ? 'review' : 'approve' } + + log_evaluation_results(evaluation) + evaluation + end + + def determine_threat_level(result) + scores = result.category_scores + max_score = scores.values.max || 0 + + case max_score + when 0.8.. then 'critical' + when 0.5..0.8 then 'high' + when 0.2..0.5 then 'medium' + else 'low' + end end def default_evaluation(error_type = nil) @@ -56,18 +67,11 @@ class Internal::AccountAnalysis::ContentEvaluatorService < Llm::LegacyBaseOpenAi end def log_evaluation_results(evaluation) - Rails.logger.info("LLM evaluation - Level: #{evaluation['threat_level']}, Illegal activities: #{evaluation['illegal_activities_detected']}") + Rails.logger.info("Moderation evaluation - Level: #{evaluation['threat_level']}, Threats: #{evaluation['detected_threats'].join(', ')}") end def handle_evaluation_error(error) Rails.logger.error("Error evaluating content: #{error.message}") default_evaluation('evaluation_failure') end - - def llm_messages(content) - [ - { role: 'system', content: 'You are a security analysis system that evaluates content for potential threats and scams.' }, - { role: 'user', content: Internal::AccountAnalysis::PromptsService.threat_analyser(content.to_s[0...10_000]) } - ] - end end diff --git a/enterprise/app/services/llm/base_ai_service.rb b/enterprise/app/services/llm/base_ai_service.rb index 504685e3a..a5a91cf24 100644 --- a/enterprise/app/services/llm/base_ai_service.rb +++ b/enterprise/app/services/llm/base_ai_service.rb @@ -14,8 +14,6 @@ class Llm::BaseAiService setup_temperature end - # Returns a configured RubyLLM chat instance. - # Subclasses can override model/temperature via instance variables or pass them explicitly. def chat(model: @model, temperature: @temperature) RubyLLM.chat(model: model).with_temperature(temperature) end diff --git a/enterprise/app/services/llm/legacy_base_open_ai_service.rb b/enterprise/app/services/llm/legacy_base_open_ai_service.rb index ede9f0d8f..f431830db 100644 --- a/enterprise/app/services/llm/legacy_base_open_ai_service.rb +++ b/enterprise/app/services/llm/legacy_base_open_ai_service.rb @@ -1,8 +1,11 @@ # frozen_string_literal: true # DEPRECATED: This class uses the legacy OpenAI Ruby gem directly. -# New features should use Llm::BaseAiService with RubyLLM instead. -# This class will be removed once all services are migrated to RubyLLM. +# Only used for PDF/file operations that require OpenAI's files API: +# - Captain::Llm::PdfProcessingService (files.upload for assistants) +# - Captain::Llm::PaginatedFaqGeneratorService (uses file_id from uploaded files) +# +# For all other LLM operations, use Llm::BaseAiService with RubyLLM instead. class Llm::LegacyBaseOpenAiService DEFAULT_MODEL = 'gpt-4o-mini' diff --git a/enterprise/app/services/messages/audio_transcription_service.rb b/enterprise/app/services/messages/audio_transcription_service.rb index d3cc91376..1676dd862 100644 --- a/enterprise/app/services/messages/audio_transcription_service.rb +++ b/enterprise/app/services/messages/audio_transcription_service.rb @@ -1,4 +1,8 @@ -class Messages::AudioTranscriptionService < Llm::LegacyBaseOpenAiService +class Messages::AudioTranscriptionService< Llm::LegacyBaseOpenAiService + include Integrations::LlmInstrumentation + + WHISPER_MODEL = 'whisper-1'.freeze + attr_reader :attachment, :message, :account def initialize(attachment) @@ -46,7 +50,7 @@ class Messages::AudioTranscriptionService < Llm::LegacyBaseOpenAiService temp_file_path = fetch_audio_file - response_text = nil + transcribed_text = nil File.open(temp_file_path, 'rb') do |file| response = @client.audio.transcribe( @@ -56,14 +60,23 @@ class Messages::AudioTranscriptionService < Llm::LegacyBaseOpenAiService temperature: 0.4 } ) - - response_text = response['text'] + transcribed_text = response['text'] end FileUtils.rm_f(temp_file_path) - update_transcription(response_text) - response_text + update_transcription(transcribed_text) + transcribed_text + end + + def instrumentation_params(file_path) + { + span_name: 'llm.messages.audio_transcription', + model: WHISPER_MODEL, + account_id: account&.id, + feature_name: 'audio_transcription', + file_path: file_path + } end def update_transcription(transcribed_text) diff --git a/enterprise/lib/captain/agent.rb b/enterprise/lib/captain/agent.rb deleted file mode 100644 index f0b511115..000000000 --- a/enterprise/lib/captain/agent.rb +++ /dev/null @@ -1,137 +0,0 @@ -require 'openai' -class Captain::Agent - attr_reader :name, :tools, :prompt, :persona, :goal, :secrets - - def initialize(name:, config:) - @name = name - @prompt = construct_prompt(config) - @tools = prepare_tools(config[:tools] || []) - @messages = config[:messages] || [] - @max_iterations = config[:max_iterations] || 10 - @llm = Captain::LlmService.new(api_key: config[:secrets][:OPENAI_API_KEY]) - @logger = Rails.logger - - @logger.info(@prompt) - end - - def execute(input, context) - setup_messages(input, context) - result = {} - @max_iterations.times do |iteration| - push_to_messages(role: 'system', content: 'Provide a final answer') if iteration == @max_iterations - 1 - - result = @llm.call(@messages, functions) - handle_llm_result(result) - - break if result[:stop] - end - - result[:output] - end - - def register_tool(tool) - @tools << tool - end - - private - - def setup_messages(input, context) - if @messages.empty? - push_to_messages({ role: 'system', content: @prompt }) - push_to_messages({ role: 'assistant', content: context }) if context.present? - end - push_to_messages({ role: 'user', content: input }) - end - - def handle_llm_result(result) - if result[:tool_call] - tool_result = execute_tool(result[:tool_call]) - push_to_messages({ role: 'assistant', content: tool_result }) - else - push_to_messages({ role: 'assistant', content: result[:output] }) - end - result[:output] - end - - def execute_tool(tool_call) - function_name = tool_call['function']['name'] - arguments = JSON.parse(tool_call['function']['arguments']) - - tool = @tools.find { |t| t.name == function_name } - tool.execute(arguments, {}) - rescue StandardError => e - "Tool execution failed: #{e.message}" - end - - def construct_prompt(config) - return config[:prompt] if config[:prompt] - - <<~PROMPT - Persona: #{config[:persona]} - Objective: #{config[:goal]} - - Guidelines: - - Persistently work towards achieving the stated objective without deviation. - - Use only the provided tools to complete the task. Avoid inventing or assuming function names. - - Set `'stop': true` once the objective is fully achieved. - - DO NOT return tool usage as the final result. - - If sufficient information is available to deliver result, compile and present it to the user. - - Always return a final result and ENSURE the final result is formatted in Markdown. - - Output Structure: - - 1. **Tool Usage:** - - If a relevant function is identified, call it directly without unnecessary explanations. - - 2. **Final Answer:** - When ready to provide a complete response, follow this JSON format: - - ```json - { - "thought_process": "Explain the reasoning and steps taken to arrive at the final result.", - "result": "Provide the complete response in clear, structured text.", - "stop": true - } - PROMPT - end - - def prepare_tools(tools = []) - tools.map do |_, tool| - Captain::Tool.new( - name: tool['name'], - config: { - description: tool['description'], - properties: tool['properties'], - secrets: tool['secrets'], - implementation: tool['implementation'] - } - ) - end - end - - def functions - @tools.map do |tool| - properties = {} - tool.properties.each do |property_name, property_details| - properties[property_name] = { - type: property_details[:type], - description: property_details[:description] - } - end - required = tool.properties.select { |_, details| details[:required] == true }.keys - { - type: 'function', - function: { - name: tool.name, - description: tool.description, - parameters: { type: 'object', properties: properties, required: required } - } - } - end - end - - def push_to_messages(message) - @logger.info("\n\n\nMessage: #{message}\n\n\n") - @messages << message - end -end diff --git a/enterprise/lib/captain/llm_service.rb b/enterprise/lib/captain/llm_service.rb deleted file mode 100644 index f0faa1002..000000000 --- a/enterprise/lib/captain/llm_service.rb +++ /dev/null @@ -1,64 +0,0 @@ -require 'openai' - -class Captain::LlmService - def initialize(config) - @client = OpenAI::Client.new( - access_token: config[:api_key], - log_errors: Rails.env.development? - ) - @logger = Rails.logger - end - - def call(messages, functions = []) - openai_params = { - model: 'gpt-4o', - response_format: { type: 'json_object' }, - messages: messages - } - openai_params[:tools] = functions if functions.any? - - response = @client.chat(parameters: openai_params) - handle_response(response) - rescue StandardError => e - handle_error(e) - end - - private - - def handle_response(response) - if response['choices'][0]['message']['tool_calls'] - handle_tool_calls(response) - else - handle_direct_response(response) - end - end - - def handle_tool_calls(response) - tool_call = response['choices'][0]['message']['tool_calls'][0] - { - tool_call: tool_call, - output: nil, - stop: false - } - end - - def handle_direct_response(response) - content = response.dig('choices', 0, 'message', 'content').strip - parsed = JSON.parse(content) - - { - output: parsed['result'] || parsed['thought_process'], - stop: parsed['stop'] || false - } - rescue JSON::ParserError => e - handle_error(e, content) - end - - def handle_error(error, content = nil) - @logger.error("LLM call failed: #{error.message}") - @logger.error(error.backtrace.join("\n")) - @logger.error("Content: #{content}") if content - - { output: 'Error occurred, retrying', stop: false } - end -end diff --git a/enterprise/lib/captain/tool.rb b/enterprise/lib/captain/tool.rb deleted file mode 100644 index b3ed6f69f..000000000 --- a/enterprise/lib/captain/tool.rb +++ /dev/null @@ -1,66 +0,0 @@ -class Captain::Tool - class InvalidImplementationError < StandardError; end - class InvalidSecretsError < StandardError; end - class ExecutionError < StandardError; end - - REQUIRED_PROPERTIES = %w[name description properties secrets].freeze - - attr_reader :name, :description, :properties, :secrets, :implementation, :memory - - def initialize(name:, config:) - @name = name - @description = config[:description] - @properties = config[:properties] - @secrets = config[:secrets] || [] - @implementation = config[:implementation] - @memory = config[:memory] || {} - end - - def register_method(&block) - @implementation = block - end - - def execute(input, provided_secrets = {}) - validate_secrets!(provided_secrets) - validate_input!(input) - - raise ExecutionError, 'No implementation registered' unless @implementation - - instance_exec(input, provided_secrets, memory, &@implementation) - rescue StandardError => e - raise ExecutionError, "Execution failed: #{e.message}" - end - - private - - def validate_config!(config) - missing_keys = REQUIRED_PROPERTIES - config.keys - return if missing_keys.empty? - - raise InvalidImplementationError, - "Missing required properties: #{missing_keys.join(', ')}" - end - - def validate_secrets!(provided_secrets) - required_secrets = secrets.map!(&:to_sym) - missing_secrets = required_secrets - provided_secrets.keys - - return if missing_secrets.empty? - - raise InvalidSecretsError, "Missing required secrets: #{missing_secrets.join(', ')}" - end - - def validate_input!(input) - properties.each do |property, constraints| - validate_property!(input, property, constraints) - end - end - - def validate_property!(input, property, constraints) - value = input[property.to_sym] - - raise ArgumentError, "Missing required property: #{property}" if constraints['required'] && value.nil? - - true - end -end diff --git a/lib/integrations/llm_instrumentation.rb b/lib/integrations/llm_instrumentation.rb index aff41fbce..34281c738 100644 --- a/lib/integrations/llm_instrumentation.rb +++ b/lib/integrations/llm_instrumentation.rb @@ -7,14 +7,6 @@ module Integrations::LlmInstrumentation include Integrations::LlmInstrumentationHelpers include Integrations::LlmInstrumentationSpans - PROVIDER_PREFIXES = { - 'openai' => %w[gpt- o1 o3 o4 text-embedding- whisper- tts-], - 'anthropic' => %w[claude-], - 'google' => %w[gemini-], - 'mistral' => %w[mistral- codestral-], - 'deepseek' => %w[deepseek-] - }.freeze - def instrument_llm_call(params) return yield unless ChatwootApp.otel_enabled? @@ -66,16 +58,57 @@ module Integrations::LlmInstrumentation end end - def determine_provider(model_name) - return 'openai' if model_name.blank? + def instrument_embedding_call(params) + return yield unless ChatwootApp.otel_enabled? - model = model_name.to_s.downcase - - PROVIDER_PREFIXES.each do |provider, prefixes| - return provider if prefixes.any? { |prefix| model.start_with?(prefix) } + instrument_with_span(params[:span_name] || 'llm.embedding', params) do |span, track_result| + set_embedding_span_attributes(span, params) + result = yield + track_result.call(result) + set_embedding_result_attributes(span, result) + result end + end - 'openai' + def instrument_audio_transcription(params) + return yield unless ChatwootApp.otel_enabled? + + instrument_with_span(params[:span_name] || 'llm.audio.transcription', params) do |span, track_result| + set_audio_transcription_span_attributes(span, params) + result = yield + track_result.call(result) + set_transcription_result_attributes(span, result) + result + end + end + + def instrument_moderation_call(params) + return yield unless ChatwootApp.otel_enabled? + + instrument_with_span(params[:span_name] || 'llm.moderation', params) do |span, track_result| + set_moderation_span_attributes(span, params) + result = yield + track_result.call(result) + set_moderation_result_attributes(span, result) + result + end + end + + def instrument_with_span(span_name, params, &) + result = nil + executed = false + tracer.in_span(span_name) do |span| + track_result = lambda do |r| + executed = true + result = r + end + yield(span, track_result) + end + rescue StandardError => e + ChatwootExceptionTracker.new(e, account: resolve_account(params)).capture_exception + raise unless executed + + result end private @@ -86,36 +119,4 @@ module Integrations::LlmInstrumentation nil end - - def setup_span_attributes(span, params) - set_request_attributes(span, params) - set_prompt_messages(span, params[:messages]) - set_metadata_attributes(span, params) - end - - def record_completion(span, result) - if result.respond_to?(:content) - span.set_attribute(ATTR_GEN_AI_COMPLETION_ROLE, result.role.to_s) if result.respond_to?(:role) - span.set_attribute(ATTR_GEN_AI_COMPLETION_CONTENT, result.content.to_s) - elsif result.is_a?(Hash) - set_completion_attributes(span, result) if result.is_a?(Hash) - end - end - - def set_request_attributes(span, params) - provider = determine_provider(params[:model]) - span.set_attribute(ATTR_GEN_AI_PROVIDER, provider) - span.set_attribute(ATTR_GEN_AI_REQUEST_MODEL, params[:model]) - span.set_attribute(ATTR_GEN_AI_REQUEST_TEMPERATURE, params[:temperature]) if params[:temperature] - end - - def set_prompt_messages(span, messages) - messages.each_with_index do |msg, idx| - role = msg[:role] || msg['role'] - content = msg[:content] || msg['content'] - - span.set_attribute(format(ATTR_GEN_AI_PROMPT_ROLE, idx), role) - span.set_attribute(format(ATTR_GEN_AI_PROMPT_CONTENT, idx), content.to_s) - end - end end diff --git a/lib/integrations/llm_instrumentation_completion_helpers.rb b/lib/integrations/llm_instrumentation_completion_helpers.rb new file mode 100644 index 000000000..50e071119 --- /dev/null +++ b/lib/integrations/llm_instrumentation_completion_helpers.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Integrations::LlmInstrumentationCompletionHelpers + include Integrations::LlmInstrumentationConstants + + private + + def set_embedding_span_attributes(span, params) + span.set_attribute(ATTR_GEN_AI_PROVIDER, determine_provider(params[:model])) + span.set_attribute(ATTR_GEN_AI_REQUEST_MODEL, params[:model]) + span.set_attribute('embedding.input_length', params[:input]&.length || 0) + span.set_attribute(ATTR_LANGFUSE_OBSERVATION_INPUT, params[:input].to_s) + set_common_span_metadata(span, params) + end + + def set_audio_transcription_span_attributes(span, params) + span.set_attribute(ATTR_GEN_AI_PROVIDER, 'openai') + span.set_attribute(ATTR_GEN_AI_REQUEST_MODEL, params[:model] || 'whisper-1') + span.set_attribute('audio.duration_seconds', params[:duration]) if params[:duration] + span.set_attribute(ATTR_LANGFUSE_OBSERVATION_INPUT, params[:file_path].to_s) if params[:file_path] + set_common_span_metadata(span, params) + end + + def set_moderation_span_attributes(span, params) + span.set_attribute(ATTR_GEN_AI_PROVIDER, 'openai') + span.set_attribute(ATTR_GEN_AI_REQUEST_MODEL, params[:model] || 'text-moderation-latest') + span.set_attribute('moderation.input_length', params[:input]&.length || 0) + span.set_attribute(ATTR_LANGFUSE_OBSERVATION_INPUT, params[:input].to_s) + set_common_span_metadata(span, params) + end + + def set_common_span_metadata(span, params) + span.set_attribute(ATTR_LANGFUSE_USER_ID, params[:account_id].to_s) if params[:account_id] + span.set_attribute(ATTR_LANGFUSE_TAGS, [params[:feature_name]].to_json) if params[:feature_name] + end + + def set_embedding_result_attributes(span, result) + span.set_attribute('embedding.dimensions', result&.length || 0) if result.is_a?(Array) + span.set_attribute(ATTR_LANGFUSE_OBSERVATION_OUTPUT, "[#{result&.length || 0} dimensions]") + end + + def set_transcription_result_attributes(span, result) + transcribed_text = result.respond_to?(:text) ? result.text : result.to_s + span.set_attribute('transcription.length', transcribed_text&.length || 0) + span.set_attribute(ATTR_LANGFUSE_OBSERVATION_OUTPUT, transcribed_text.to_s) + end + + def set_moderation_result_attributes(span, result) + span.set_attribute('moderation.flagged', result.flagged?) if result.respond_to?(:flagged?) + span.set_attribute('moderation.categories', result.flagged_categories.to_json) if result.respond_to?(:flagged_categories) + output = { + flagged: result.respond_to?(:flagged?) ? result.flagged? : nil, + categories: result.respond_to?(:flagged_categories) ? result.flagged_categories : [] + } + span.set_attribute(ATTR_LANGFUSE_OBSERVATION_OUTPUT, output.to_json) + end + + def set_completion_attributes(span, result) + set_completion_message(span, result) + set_usage_metrics(span, result) + set_error_attributes(span, result) + end + + def set_completion_message(span, result) + message = result[:message] || result.dig('choices', 0, 'message', 'content') + return if message.blank? + + span.set_attribute(ATTR_GEN_AI_COMPLETION_ROLE, 'assistant') + span.set_attribute(ATTR_GEN_AI_COMPLETION_CONTENT, message) + end + + def set_usage_metrics(span, result) + usage = result[:usage] || result['usage'] + return if usage.blank? + + span.set_attribute(ATTR_GEN_AI_USAGE_INPUT_TOKENS, usage['prompt_tokens']) if usage['prompt_tokens'] + span.set_attribute(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, usage['completion_tokens']) if usage['completion_tokens'] + span.set_attribute(ATTR_GEN_AI_USAGE_TOTAL_TOKENS, usage['total_tokens']) if usage['total_tokens'] + end + + def set_error_attributes(span, result) + error = result[:error] || result['error'] + return if error.blank? + + span.set_attribute(ATTR_GEN_AI_RESPONSE_ERROR, error.to_json) + span.status = OpenTelemetry::Trace::Status.error(error.to_s.truncate(1000)) + end +end diff --git a/lib/integrations/llm_instrumentation_helpers.rb b/lib/integrations/llm_instrumentation_helpers.rb index c03e3e9c7..129092ed4 100644 --- a/lib/integrations/llm_instrumentation_helpers.rb +++ b/lib/integrations/llm_instrumentation_helpers.rb @@ -2,38 +2,52 @@ module Integrations::LlmInstrumentationHelpers include Integrations::LlmInstrumentationConstants + include Integrations::LlmInstrumentationCompletionHelpers + + def determine_provider(model_name) + return 'openai' if model_name.blank? + + model = model_name.to_s.downcase + + LlmConstants::PROVIDER_PREFIXES.each do |provider, prefixes| + return provider if prefixes.any? { |prefix| model.start_with?(prefix) } + end + + 'openai' + end private - def set_completion_attributes(span, result) - set_completion_message(span, result) - set_usage_metrics(span, result) - set_error_attributes(span, result) + def setup_span_attributes(span, params) + set_request_attributes(span, params) + set_prompt_messages(span, params[:messages]) + set_metadata_attributes(span, params) end - def set_completion_message(span, result) - message = result[:message] || result.dig('choices', 0, 'message', 'content') - return if message.blank? - - span.set_attribute(ATTR_GEN_AI_COMPLETION_ROLE, 'assistant') - span.set_attribute(ATTR_GEN_AI_COMPLETION_CONTENT, message) + def record_completion(span, result) + if result.respond_to?(:content) + span.set_attribute(ATTR_GEN_AI_COMPLETION_ROLE, result.role.to_s) if result.respond_to?(:role) + span.set_attribute(ATTR_GEN_AI_COMPLETION_CONTENT, result.content.to_s) + elsif result.is_a?(Hash) + set_completion_attributes(span, result) + end end - def set_usage_metrics(span, result) - usage = result[:usage] || result['usage'] - return if usage.blank? - - span.set_attribute(ATTR_GEN_AI_USAGE_INPUT_TOKENS, usage['prompt_tokens']) if usage['prompt_tokens'] - span.set_attribute(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, usage['completion_tokens']) if usage['completion_tokens'] - span.set_attribute(ATTR_GEN_AI_USAGE_TOTAL_TOKENS, usage['total_tokens']) if usage['total_tokens'] + def set_request_attributes(span, params) + provider = determine_provider(params[:model]) + span.set_attribute(ATTR_GEN_AI_PROVIDER, provider) + span.set_attribute(ATTR_GEN_AI_REQUEST_MODEL, params[:model]) + span.set_attribute(ATTR_GEN_AI_REQUEST_TEMPERATURE, params[:temperature]) if params[:temperature] end - def set_error_attributes(span, result) - error = result[:error] || result['error'] - return if error.blank? + def set_prompt_messages(span, messages) + messages.each_with_index do |msg, idx| + role = msg[:role] || msg['role'] + content = msg[:content] || msg['content'] - span.set_attribute(ATTR_GEN_AI_RESPONSE_ERROR, error.to_json) - span.status = OpenTelemetry::Trace::Status.error(error.to_s.truncate(1000)) + span.set_attribute(format(ATTR_GEN_AI_PROMPT_ROLE, idx), role) + span.set_attribute(format(ATTR_GEN_AI_PROMPT_CONTENT, idx), content.to_s) + end end def set_metadata_attributes(span, params) diff --git a/lib/integrations/llm_instrumentation_spans.rb b/lib/integrations/llm_instrumentation_spans.rb index 9199aaa10..824b6aa4a 100644 --- a/lib/integrations/llm_instrumentation_spans.rb +++ b/lib/integrations/llm_instrumentation_spans.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'opentelemetry_config' -require_relative 'llm_instrumentation_constants' module Integrations::LlmInstrumentationSpans include Integrations::LlmInstrumentationConstants diff --git a/lib/integrations/openai/processor_service.rb b/lib/integrations/openai/processor_service.rb index 0a0dfa8ae..2f0180701 100644 --- a/lib/integrations/openai/processor_service.rb +++ b/lib/integrations/openai/processor_service.rb @@ -77,21 +77,22 @@ class Integrations::Openai::ProcessorService < Integrations::LlmBaseService end def add_message_if_within_limit(character_count, message, messages, in_array_format) - if valid_message?(message, character_count) - add_message_to_list(message, messages, in_array_format) - character_count += message.content.length + content = message.content_for_llm + if valid_message?(content, character_count) + add_message_to_list(message, messages, in_array_format, content) + character_count += content.length [character_count, true] else [character_count, false] end end - def valid_message?(message, character_count) - message.content.present? && character_count + message.content.length <= TOKEN_LIMIT + def valid_message?(content, character_count) + content.present? && character_count + content.length <= TOKEN_LIMIT end - def add_message_to_list(message, messages, in_array_format) - formatted_message = format_message(message, in_array_format) + def add_message_to_list(message, messages, in_array_format, content) + formatted_message = format_message(message, in_array_format, content) messages.prepend(formatted_message) end @@ -99,17 +100,17 @@ class Integrations::Openai::ProcessorService < Integrations::LlmBaseService in_array_format ? [] : '' end - def format_message(message, in_array_format) - in_array_format ? format_message_in_array(message) : format_message_in_string(message) + def format_message(message, in_array_format, content) + in_array_format ? format_message_in_array(message, content) : format_message_in_string(message, content) end - def format_message_in_array(message) - { role: (message.incoming? ? 'user' : 'assistant'), content: message.content } + def format_message_in_array(message, content) + { role: (message.incoming? ? 'user' : 'assistant'), content: content } end - def format_message_in_string(message) + def format_message_in_string(message, content) sender_type = message.incoming? ? 'Customer' : 'Agent' - "#{sender_type} #{message.sender&.name} : #{message.content}\n" + "#{sender_type} #{message.sender&.name} : #{content}\n" end def summarize_body diff --git a/lib/llm_constants.rb b/lib/llm_constants.rb new file mode 100644 index 000000000..054b775a5 --- /dev/null +++ b/lib/llm_constants.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module LlmConstants + DEFAULT_MODEL = 'gpt-4.1-mini' + DEFAULT_EMBEDDING_MODEL = 'text-embedding-3-small' + PDF_PROCESSING_MODEL = 'gpt-4.1-mini' + + OPENAI_API_ENDPOINT = 'https://api.openai.com' + + PROVIDER_PREFIXES = { + 'openai' => %w[gpt- o1 o3 o4 text-embedding- whisper- tts-], + 'anthropic' => %w[claude-], + 'google' => %w[gemini-], + 'mistral' => %w[mistral- codestral-], + 'deepseek' => %w[deepseek-] + }.freeze +end diff --git a/lib/open_ai_constants.rb b/lib/open_ai_constants.rb deleted file mode 100644 index 2094567a7..000000000 --- a/lib/open_ai_constants.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -module OpenAiConstants - DEFAULT_MODEL = 'gpt-4.1-mini' - DEFAULT_ENDPOINT = 'https://api.openai.com' - DEFAULT_EMBEDDING_MODEL = 'text-embedding-3-small' - PDF_PROCESSING_MODEL = 'gpt-4.1-mini' -end diff --git a/spec/enterprise/jobs/captain/documents/response_builder_job_spec.rb b/spec/enterprise/jobs/captain/documents/response_builder_job_spec.rb index 73b67ee27..c3e5eab1c 100644 --- a/spec/enterprise/jobs/captain/documents/response_builder_job_spec.rb +++ b/spec/enterprise/jobs/captain/documents/response_builder_job_spec.rb @@ -13,7 +13,7 @@ RSpec.describe Captain::Documents::ResponseBuilderJob, type: :job do before do allow(Captain::Llm::FaqGeneratorService).to receive(:new) - .with(document.content, document.account.locale_english_name) + .with(document.content, document.account.locale_english_name, account_id: document.account_id) .and_return(faq_generator) allow(faq_generator).to receive(:generate).and_return(faqs) end @@ -52,7 +52,7 @@ RSpec.describe Captain::Documents::ResponseBuilderJob, type: :job do before do allow(Captain::Llm::FaqGeneratorService).to receive(:new) - .with(spanish_document.content, 'portuguese') + .with(spanish_document.content, 'portuguese', account_id: spanish_document.account_id) .and_return(spanish_faq_generator) allow(spanish_faq_generator).to receive(:generate).and_return(faqs) end @@ -61,7 +61,7 @@ RSpec.describe Captain::Documents::ResponseBuilderJob, type: :job do described_class.new.perform(spanish_document) expect(Captain::Llm::FaqGeneratorService).to have_received(:new) - .with(spanish_document.content, 'portuguese') + .with(spanish_document.content, 'portuguese', account_id: spanish_document.account_id) end end diff --git a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb index ee993da37..0ab7f37bf 100644 --- a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb +++ b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb @@ -4,49 +4,42 @@ RSpec.describe Captain::Llm::ConversationFaqService do let(:captain_assistant) { create(:captain_assistant) } let(:conversation) { create(:conversation, first_reply_created_at: Time.zone.now) } let(:service) { described_class.new(captain_assistant, conversation) } - let(:client) { instance_double(OpenAI::Client) } let(:embedding_service) { instance_double(Captain::Llm::EmbeddingService) } + let(:mock_chat) { instance_double(RubyLLM::Chat) } + let(:sample_faqs) do + [ + { 'question' => 'What is the purpose?', 'answer' => 'To help users.' }, + { 'question' => 'How does it work?', 'answer' => 'Through AI.' } + ] + end + let(:mock_response) do + instance_double(RubyLLM::Message, content: { faqs: sample_faqs }.to_json) + end before do - create(:installation_config) { create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key') } - allow(OpenAI::Client).to receive(:new).and_return(client) + create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key') allow(Captain::Llm::EmbeddingService).to receive(:new).and_return(embedding_service) + allow(RubyLLM).to receive(:chat).and_return(mock_chat) + allow(mock_chat).to receive(:with_temperature).and_return(mock_chat) + allow(mock_chat).to receive(:with_params).and_return(mock_chat) + allow(mock_chat).to receive(:with_instructions).and_return(mock_chat) + allow(mock_chat).to receive(:ask).and_return(mock_response) end describe '#generate_and_deduplicate' do - let(:sample_faqs) do - [ - { 'question' => 'What is the purpose?', 'answer' => 'To help users.' }, - { 'question' => 'How does it work?', 'answer' => 'Through AI.' } - ] - end - - let(:openai_response) do - { - 'choices' => [ - { - 'message' => { - 'content' => { faqs: sample_faqs }.to_json - } - } - ] - } - end - context 'when successful' do before do - allow(client).to receive(:chat).and_return(openai_response) allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3]) allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([]) end - it 'creates new FAQs' do + it 'creates new FAQs for valid conversation content' do expect do service.generate_and_deduplicate end.to change(captain_assistant.responses, :count).by(2) end - it 'saves the correct FAQ content' do + it 'saves FAQs with pending status linked to conversation' do service.generate_and_deduplicate expect( captain_assistant.responses.pluck(:question, :answer, :status, :documentable_id) @@ -63,6 +56,11 @@ RSpec.describe Captain::Llm::ConversationFaqService do it 'returns an empty array without generating FAQs' do expect(service.generate_and_deduplicate).to eq([]) end + + it 'does not call the LLM API' do + expect(RubyLLM).not_to receive(:chat) + service.generate_and_deduplicate + end end context 'when finding duplicates' do @@ -70,9 +68,6 @@ RSpec.describe Captain::Llm::ConversationFaqService do create(:captain_assistant_response, assistant: captain_assistant, question: 'Similar question', answer: 'Similar answer') end let(:similar_neighbor) do - # Using OpenStruct here to mock as the Captain:AssistantResponse does not implement - # neighbor_distance as a method or attribute rather it is returned directly - # from SQL query in neighbor gem OpenStruct.new( id: 1, question: existing_response.question, @@ -82,87 +77,78 @@ RSpec.describe Captain::Llm::ConversationFaqService do end before do - allow(client).to receive(:chat).and_return(openai_response) allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3]) allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([similar_neighbor]) end - it 'filters out duplicate FAQs' do + it 'filters out duplicate FAQs based on embedding similarity' do expect do service.generate_and_deduplicate end.not_to change(captain_assistant.responses, :count) end end - context 'when OpenAI API fails' do + context 'when LLM API fails' do before do - allow(client).to receive(:chat).and_raise(OpenAI::Error.new('API Error')) + allow(mock_chat).to receive(:ask).and_raise(RubyLLM::Error.new(nil, 'API Error')) + allow(Rails.logger).to receive(:error) end - it 'handles the error and returns empty array' do - expect(Rails.logger).to receive(:error).with('OpenAI API Error: API Error') + it 'returns empty array and logs the error' do + expect(Rails.logger).to receive(:error).with('LLM API Error: API Error') expect(service.generate_and_deduplicate).to eq([]) end end context 'when JSON parsing fails' do let(:invalid_response) do - { - 'choices' => [ - { - 'message' => { - 'content' => 'invalid json' - } - } - ] - } + instance_double(RubyLLM::Message, content: 'invalid json') end before do - allow(client).to receive(:chat).and_return(invalid_response) + allow(mock_chat).to receive(:ask).and_return(invalid_response) end - it 'handles JSON parsing errors' do + it 'handles JSON parsing errors gracefully' do expect(Rails.logger).to receive(:error).with(/Error in parsing GPT processed response:/) expect(service.generate_and_deduplicate).to eq([]) end end + + context 'when response content is nil' do + let(:nil_response) do + instance_double(RubyLLM::Message, content: nil) + end + + before do + allow(mock_chat).to receive(:ask).and_return(nil_response) + end + + it 'returns empty array' do + expect(service.generate_and_deduplicate).to eq([]) + end + end end - describe '#chat_parameters' do - it 'includes correct model and response format' do - params = service.send(:chat_parameters) - expect(params[:model]).to eq('gpt-4o-mini') - expect(params[:response_format]).to eq({ type: 'json_object' }) - end - - it 'includes system prompt and conversation content' do - allow(Captain::Llm::SystemPromptsService).to receive(:conversation_faq_generator).and_return('system prompt') - params = service.send(:chat_parameters) - - expect(params[:messages]).to include( - { role: 'system', content: 'system prompt' }, - { role: 'user', content: conversation.to_llm_text } - ) - end - + describe 'language handling' do context 'when conversation has different language' do let(:account) { create(:account, locale: 'fr') } let(:conversation) do - create(:conversation, account: account, - first_reply_created_at: Time.zone.now) + create(:conversation, account: account, first_reply_created_at: Time.zone.now) end - it 'includes system prompt with correct language' do - allow(Captain::Llm::SystemPromptsService).to receive(:conversation_faq_generator) + before do + allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3]) + allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([]) + end + + it 'uses account language for system prompt' do + expect(Captain::Llm::SystemPromptsService).to receive(:conversation_faq_generator) .with('french') - .and_return('system prompt in french') + .at_least(:once) + .and_call_original - params = service.send(:chat_parameters) - - expect(params[:messages]).to include( - { role: 'system', content: 'system prompt in french' } - ) + service.generate_and_deduplicate end end end diff --git a/spec/enterprise/services/captain/llm/faq_generator_service_spec.rb b/spec/enterprise/services/captain/llm/faq_generator_service_spec.rb index 7d799dc18..003d5b715 100644 --- a/spec/enterprise/services/captain/llm/faq_generator_service_spec.rb +++ b/spec/enterprise/services/captain/llm/faq_generator_service_spec.rb @@ -4,58 +4,40 @@ RSpec.describe Captain::Llm::FaqGeneratorService do let(:content) { 'Sample content for FAQ generation' } let(:language) { 'english' } let(:service) { described_class.new(content, language) } - let(:client) { instance_double(OpenAI::Client) } + let(:mock_chat) { instance_double(RubyLLM::Chat) } + let(:sample_faqs) do + [ + { 'question' => 'What is this service?', 'answer' => 'It generates FAQs.' }, + { 'question' => 'How does it work?', 'answer' => 'Using AI technology.' } + ] + end + let(:mock_response) do + instance_double(RubyLLM::Message, content: { faqs: sample_faqs }.to_json) + end before do create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key') - allow(OpenAI::Client).to receive(:new).and_return(client) + allow(RubyLLM).to receive(:chat).and_return(mock_chat) + allow(mock_chat).to receive(:with_temperature).and_return(mock_chat) + allow(mock_chat).to receive(:with_params).and_return(mock_chat) + allow(mock_chat).to receive(:with_instructions).and_return(mock_chat) + allow(mock_chat).to receive(:ask).and_return(mock_response) end describe '#generate' do - let(:sample_faqs) do - [ - { 'question' => 'What is this service?', 'answer' => 'It generates FAQs.' }, - { 'question' => 'How does it work?', 'answer' => 'Using AI technology.' } - ] - end - - let(:openai_response) do - { - 'choices' => [ - { - 'message' => { - 'content' => { faqs: sample_faqs }.to_json - } - } - ] - } - end - context 'when successful' do - before do - allow(client).to receive(:chat).and_return(openai_response) - allow(Captain::Llm::SystemPromptsService).to receive(:faq_generator).and_return('system prompt') - end - - it 'returns parsed FAQs' do + it 'returns parsed FAQs from the LLM response' do result = service.generate expect(result).to eq(sample_faqs) end - it 'calls OpenAI client with chat parameters' do - expect(client).to receive(:chat).with(parameters: hash_including( - model: 'gpt-4o-mini', - response_format: { type: 'json_object' }, - messages: array_including( - hash_including(role: 'system'), - hash_including(role: 'user', content: content) - ) - )) + it 'sends content to LLM with JSON response format' do + expect(mock_chat).to receive(:with_params).with(response_format: { type: 'json_object' }).and_return(mock_chat) service.generate end - it 'calls SystemPromptsService with correct language' do - expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with(language) + it 'uses SystemPromptsService with the specified language' do + expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with(language).at_least(:once).and_call_original service.generate end end @@ -63,23 +45,57 @@ RSpec.describe Captain::Llm::FaqGeneratorService do context 'with different language' do let(:language) { 'spanish' } - before do - allow(client).to receive(:chat).and_return(openai_response) - end - it 'passes the correct language to SystemPromptsService' do - expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with('spanish') + expect(Captain::Llm::SystemPromptsService).to receive(:faq_generator).with('spanish').at_least(:once).and_call_original service.generate end end - context 'when OpenAI API fails' do + context 'when LLM API fails' do before do - allow(client).to receive(:chat).and_raise(OpenAI::Error.new('API Error')) + allow(mock_chat).to receive(:ask).and_raise(RubyLLM::Error.new(nil, 'API Error')) + allow(Rails.logger).to receive(:error) end - it 'handles the error and returns empty array' do - expect(Rails.logger).to receive(:error).with('OpenAI API Error: API Error') + it 'returns empty array and logs the error' do + expect(Rails.logger).to receive(:error).with('LLM API Error: API Error') + expect(service.generate).to eq([]) + end + end + + context 'when response content is nil' do + let(:nil_response) { instance_double(RubyLLM::Message, content: nil) } + + before do + allow(mock_chat).to receive(:ask).and_return(nil_response) + end + + it 'returns empty array' do + expect(service.generate).to eq([]) + end + end + + context 'when JSON parsing fails' do + let(:invalid_response) { instance_double(RubyLLM::Message, content: 'invalid json') } + + before do + allow(mock_chat).to receive(:ask).and_return(invalid_response) + end + + it 'logs error and returns empty array' do + expect(Rails.logger).to receive(:error).with(/Error in parsing GPT processed response:/) + expect(service.generate).to eq([]) + end + end + + context 'when response is missing faqs key' do + let(:missing_key_response) { instance_double(RubyLLM::Message, content: '{"data": []}') } + + before do + allow(mock_chat).to receive(:ask).and_return(missing_key_response) + end + + it 'returns empty array via KeyError rescue' do expect(service.generate).to eq([]) end end diff --git a/spec/enterprise/services/captain/onboarding/website_analyzer_service_spec.rb b/spec/enterprise/services/captain/onboarding/website_analyzer_service_spec.rb index a2735bd69..4dec4c051 100644 --- a/spec/enterprise/services/captain/onboarding/website_analyzer_service_spec.rb +++ b/spec/enterprise/services/captain/onboarding/website_analyzer_service_spec.rb @@ -4,40 +4,38 @@ RSpec.describe Captain::Onboarding::WebsiteAnalyzerService do let(:website_url) { 'https://example.com' } let(:service) { described_class.new(website_url) } let(:mock_crawler) { instance_double(Captain::Tools::SimplePageCrawlService) } - let(:mock_client) { instance_double(OpenAI::Client) } + let(:mock_chat) { instance_double(RubyLLM::Chat) } + let(:business_info) do + { + 'business_name' => 'Example Corp', + 'suggested_assistant_name' => 'Alex from Example Corp', + 'description' => 'You specialize in helping customers with business solutions and support' + } + end + let(:mock_response) do + instance_double(RubyLLM::Message, content: business_info.to_json) + end before do create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key') allow(Captain::Tools::SimplePageCrawlService).to receive(:new).and_return(mock_crawler) - allow(service).to receive(:client).and_return(mock_client) - allow(service).to receive(:model).and_return('gpt-3.5-turbo') + allow(RubyLLM).to receive(:chat).and_return(mock_chat) + allow(mock_chat).to receive(:with_temperature).and_return(mock_chat) + allow(mock_chat).to receive(:with_params).and_return(mock_chat) + allow(mock_chat).to receive(:with_instructions).and_return(mock_chat) + allow(mock_chat).to receive(:ask).and_return(mock_response) end describe '#analyze' do - context 'when website content is available and OpenAI call is successful' do - let(:openai_response) do - { - 'choices' => [{ - 'message' => { - 'content' => { - 'business_name' => 'Example Corp', - 'suggested_assistant_name' => 'Alex from Example Corp', - 'description' => 'You specialize in helping customers with business solutions and support' - }.to_json - } - }] - } - end - + context 'when website content is available and LLM call is successful' do before do allow(mock_crawler).to receive(:body_text_content).and_return('Welcome to Example Corp') allow(mock_crawler).to receive(:page_title).and_return('Example Corp - Home') allow(mock_crawler).to receive(:meta_description).and_return('Leading provider of business solutions') allow(mock_crawler).to receive(:favicon_url).and_return('https://example.com/favicon.ico') - allow(mock_client).to receive(:chat).and_return(openai_response) end - it 'returns success' do + it 'returns successful analysis with extracted business info' do result = service.analyze expect(result[:success]).to be true @@ -49,14 +47,19 @@ RSpec.describe Captain::Onboarding::WebsiteAnalyzerService do favicon_url: 'https://example.com/favicon.ico' ) end + + it 'uses low temperature for deterministic analysis' do + expect(mock_chat).to receive(:with_temperature).with(0.1).and_return(mock_chat) + service.analyze + end end - context 'when website content is errored' do + context 'when website content fetch raises an error' do before do allow(mock_crawler).to receive(:body_text_content).and_raise(StandardError, 'Network error') end - it 'returns error' do + it 'returns error response' do result = service.analyze expect(result[:success]).to be false @@ -64,14 +67,14 @@ RSpec.describe Captain::Onboarding::WebsiteAnalyzerService do end end - context 'when website content is unavailable' do + context 'when website content is empty' do before do allow(mock_crawler).to receive(:body_text_content).and_return('') allow(mock_crawler).to receive(:page_title).and_return('') allow(mock_crawler).to receive(:meta_description).and_return('') end - it 'returns error' do + it 'returns error for unavailable content' do result = service.analyze expect(result[:success]).to be false @@ -79,21 +82,57 @@ RSpec.describe Captain::Onboarding::WebsiteAnalyzerService do end end - context 'when OpenAI error' do + context 'when LLM call fails' do before do allow(mock_crawler).to receive(:body_text_content).and_return('Welcome to Example Corp') allow(mock_crawler).to receive(:page_title).and_return('Example Corp - Home') allow(mock_crawler).to receive(:meta_description).and_return('Leading provider of business solutions') allow(mock_crawler).to receive(:favicon_url).and_return('https://example.com/favicon.ico') - allow(mock_client).to receive(:chat).and_raise(StandardError, 'API error') + allow(mock_chat).to receive(:ask).and_raise(StandardError, 'API error') end - it 'returns error' do + it 'returns error response with message' do result = service.analyze expect(result[:success]).to be false expect(result[:error]).to eq('API error') end end + + context 'when LLM returns invalid JSON' do + let(:invalid_response) { instance_double(RubyLLM::Message, content: 'not valid json') } + + before do + allow(mock_crawler).to receive(:body_text_content).and_return('Welcome to Example Corp') + allow(mock_crawler).to receive(:page_title).and_return('Example Corp - Home') + allow(mock_crawler).to receive(:meta_description).and_return('Leading provider of business solutions') + allow(mock_crawler).to receive(:favicon_url).and_return('https://example.com/favicon.ico') + allow(mock_chat).to receive(:ask).and_return(invalid_response) + end + + it 'returns error for parsing failure' do + result = service.analyze + + expect(result[:success]).to be false + expect(result[:error]).to eq('Failed to parse business information from website') + end + end + + context 'when URL normalization is needed' do + let(:website_url) { 'example.com' } + + before do + allow(mock_crawler).to receive(:body_text_content).and_return('Welcome') + allow(mock_crawler).to receive(:page_title).and_return('Example') + allow(mock_crawler).to receive(:meta_description).and_return('Description') + allow(mock_crawler).to receive(:favicon_url).and_return(nil) + end + + it 'normalizes URL by adding https prefix' do + result = service.analyze + + expect(result[:data][:website_url]).to eq('https://example.com') + end + end end end diff --git a/spec/enterprise/services/internal/account_analysis/content_evaluator_service_spec.rb b/spec/enterprise/services/internal/account_analysis/content_evaluator_service_spec.rb index d7bf26c35..f959a9afa 100644 --- a/spec/enterprise/services/internal/account_analysis/content_evaluator_service_spec.rb +++ b/spec/enterprise/services/internal/account_analysis/content_evaluator_service_spec.rb @@ -3,60 +3,103 @@ require 'rails_helper' RSpec.describe Internal::AccountAnalysis::ContentEvaluatorService do let(:service) { described_class.new } let(:content) { 'This is some test content' } + let(:mock_moderation_result) do + instance_double( + RubyLLM::Moderation, + flagged?: false, + flagged_categories: [], + category_scores: {} + ) + end before do create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key') + allow(RubyLLM).to receive(:moderate).and_return(mock_moderation_result) end describe '#evaluate' do - context 'when content is present' do - let(:llm_response) do - { - 'choices' => [ - { - 'message' => { - 'content' => { - 'threat_level' => 'low', - 'threat_summary' => 'No significant threats detected', - 'detected_threats' => ['minor_concern'], - 'illegal_activities_detected' => false, - 'recommendation' => 'approve' - }.to_json - } - } - ] - } - end - - before do - allow(service).to receive(:send_to_llm).and_return(llm_response) - allow(Rails.logger).to receive(:info) - end - - it 'returns the evaluation results' do + context 'when content is safe' do + it 'returns safe evaluation with approval recommendation' do result = service.evaluate(content) expect(result).to include( - 'threat_level' => 'low', - 'threat_summary' => 'No significant threats detected', - 'detected_threats' => ['minor_concern'], + 'threat_level' => 'safe', + 'threat_summary' => 'No threats detected', + 'detected_threats' => [], 'illegal_activities_detected' => false, 'recommendation' => 'approve' ) end it 'logs the evaluation results' do + expect(Rails.logger).to receive(:info).with('Moderation evaluation - Level: safe, Threats: ') service.evaluate(content) + end + end - expect(Rails.logger).to have_received(:info).with('LLM evaluation - Level: low, Illegal activities: false') + context 'when content is flagged' do + let(:mock_moderation_result) do + instance_double( + RubyLLM::Moderation, + flagged?: true, + flagged_categories: %w[harassment hate], + category_scores: { 'harassment' => 0.6, 'hate' => 0.3 } + ) + end + + it 'returns flagged evaluation with review recommendation' do + result = service.evaluate(content) + + expect(result).to include( + 'threat_level' => 'high', + 'threat_summary' => 'Content flagged for: harassment, hate', + 'detected_threats' => %w[harassment hate], + 'illegal_activities_detected' => false, + 'recommendation' => 'review' + ) + end + end + + context 'when content contains violence' do + let(:mock_moderation_result) do + instance_double( + RubyLLM::Moderation, + flagged?: true, + flagged_categories: ['violence'], + category_scores: { 'violence' => 0.9 } + ) + end + + it 'marks illegal activities detected for violence' do + result = service.evaluate(content) + + expect(result['illegal_activities_detected']).to be true + expect(result['threat_level']).to eq('critical') + end + end + + context 'when content contains self-harm' do + let(:mock_moderation_result) do + instance_double( + RubyLLM::Moderation, + flagged?: true, + flagged_categories: ['self-harm'], + category_scores: { 'self-harm' => 0.85 } + ) + end + + it 'marks illegal activities detected for self-harm' do + result = service.evaluate(content) + + expect(result['illegal_activities_detected']).to be true end end context 'when content is blank' do let(:blank_content) { '' } - it 'returns the default evaluation without calling the LLM' do - expect(service).not_to receive(:send_to_llm) + it 'returns default evaluation without calling moderation API' do + expect(RubyLLM).not_to receive(:moderate) result = service.evaluate(blank_content) @@ -70,34 +113,16 @@ RSpec.describe Internal::AccountAnalysis::ContentEvaluatorService do end end - context 'when LLM response is nil' do - before do - allow(service).to receive(:send_to_llm).and_return(nil) - end - - it 'returns the default evaluation' do - result = service.evaluate(content) - - expect(result).to include( - 'threat_level' => 'unknown', - 'threat_summary' => 'Failed to complete content evaluation', - 'detected_threats' => [], - 'illegal_activities_detected' => false, - 'recommendation' => 'review' - ) - end - end - context 'when error occurs during evaluation' do before do - allow(service).to receive(:send_to_llm).and_raise(StandardError.new('Test error')) - allow(Rails.logger).to receive(:error) + allow(RubyLLM).to receive(:moderate).and_raise(StandardError.new('Test error')) end - it 'logs the error and returns default evaluation with error type' do + it 'logs error and returns default evaluation with error type' do + expect(Rails.logger).to receive(:error).with('Error evaluating content: Test error') + result = service.evaluate(content) - expect(Rails.logger).to have_received(:error).with('Error evaluating content: Test error') expect(result).to include( 'threat_level' => 'unknown', 'threat_summary' => 'Failed to complete content evaluation', @@ -107,5 +132,68 @@ RSpec.describe Internal::AccountAnalysis::ContentEvaluatorService do ) end end + + context 'with threat level determination' do + it 'returns critical for scores >= 0.8' do + mock_result = instance_double( + RubyLLM::Moderation, + flagged?: true, + flagged_categories: ['harassment'], + category_scores: { 'harassment' => 0.85 } + ) + allow(RubyLLM).to receive(:moderate).and_return(mock_result) + + result = service.evaluate(content) + expect(result['threat_level']).to eq('critical') + end + + it 'returns high for scores between 0.5 and 0.8' do + mock_result = instance_double( + RubyLLM::Moderation, + flagged?: true, + flagged_categories: ['harassment'], + category_scores: { 'harassment' => 0.65 } + ) + allow(RubyLLM).to receive(:moderate).and_return(mock_result) + + result = service.evaluate(content) + expect(result['threat_level']).to eq('high') + end + + it 'returns medium for scores between 0.2 and 0.5' do + mock_result = instance_double( + RubyLLM::Moderation, + flagged?: true, + flagged_categories: ['harassment'], + category_scores: { 'harassment' => 0.35 } + ) + allow(RubyLLM).to receive(:moderate).and_return(mock_result) + + result = service.evaluate(content) + expect(result['threat_level']).to eq('medium') + end + + it 'returns low for scores below 0.2' do + mock_result = instance_double( + RubyLLM::Moderation, + flagged?: true, + flagged_categories: ['harassment'], + category_scores: { 'harassment' => 0.15 } + ) + allow(RubyLLM).to receive(:moderate).and_return(mock_result) + + result = service.evaluate(content) + expect(result['threat_level']).to eq('low') + end + end + + context 'with content truncation' do + let(:long_content) { 'a' * 15_000 } + + it 'truncates content to 10000 characters before sending to moderation' do + expect(RubyLLM).to receive(:moderate).with('a' * 10_000).and_return(mock_moderation_result) + service.evaluate(long_content) + end + end end end