fix: log only on system api key failures (#13968)

Removes sentry flooding of unnecessary rubyllm logs of wrong API key.
Logs only system api key error since it would be P0.

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

lib/captain/base_task_service.rb

@@ -1,6 +1,7 @@
 class Captain::BaseTaskService
   include Integrations::LlmInstrumentation
   include Captain::ToolInstrumentation
+  include Llm::ExceptionTrackable
 
   # gpt-4o-mini supports 128,000 tokens
   # 1 token is approx 4 characters
@@ -55,7 +56,9 @@ class Captain::BaseTaskService
   end
 
   def execute_ruby_llm_request(model:, messages:, schema: nil, tools: [])
-    Llm::Config.with_api_key(api_key, api_base: api_base) do |context|
+    credential = llm_credential
+
+    Llm::Config.with_api_key(credential[:api_key], api_base: api_base) do |context|
       chat = build_chat(context, model: model, messages: messages, schema: schema, tools: tools)
 
       conversation_messages = messages.reject { |m| m[:role] == 'system' }
@@ -65,7 +68,7 @@ class Captain::BaseTaskService
       build_ruby_llm_response(chat.ask(conversation_messages.last[:content]), messages)
     end
   rescue StandardError => e
-    ChatwootExceptionTracker.new(e, account: account).capture_exception
+    capture_llm_exception(e, credential: credential)
     { error: e.message, request_messages: messages }
   end
 
@@ -147,11 +150,24 @@ class Captain::BaseTaskService
   end
 
   def api_key_configured?
-    api_key.present?
+    llm_credential.present?
   end
 
   def api_key
-    @api_key ||= openai_hook&.settings&.dig('api_key') || system_api_key
+    llm_credential&.dig(:api_key)
+  end
+
+  def llm_credential
+    @llm_credential ||= hook_llm_credential || system_llm_credential
+  end
+
+  def hook_llm_credential
+    key = openai_hook&.settings&.dig('api_key').presence
+    { api_key: key, source: :hook } if key
+  end
+
+  def system_llm_credential
+    { api_key: system_api_key, source: :system } if system_api_key.present?
   end
 
   def openai_hook
@@ -162,6 +178,10 @@ class Captain::BaseTaskService
     @system_api_key ||= InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_API_KEY')&.value
   end
 
+  def exception_tracking_account
+    account
+  end
+
   def prompt_from_file(file_name)
     Rails.root.join('lib/integrations/openai/openai_prompts', "#{file_name}.liquid").read
   end

lib/integrations/llm_base_service.rb

@@ -1,5 +1,6 @@
 class Integrations::LlmBaseService
   include Integrations::LlmInstrumentation
+  include Llm::ExceptionTrackable
 
   # gpt-4o-mini supports 128,000 tokens
   # 1 token is approx 4 characters
@@ -100,13 +101,14 @@ class Integrations::LlmBaseService
   def execute_ruby_llm_request(parsed_body)
     messages = parsed_body['messages']
     model = parsed_body['model']
+    credential = llm_credential
 
-    Llm::Config.with_api_key(hook.settings['api_key'], api_base: api_base) do |context|
+    Llm::Config.with_api_key(credential[:api_key], api_base: api_base) do |context|
       chat = context.chat(model: model)
       setup_chat_with_messages(chat, messages)
     end
   rescue StandardError => e
-    ChatwootExceptionTracker.new(e, account: hook.account).capture_exception
+    capture_llm_exception(e, credential: credential)
     build_error_response_from_exception(e, messages)
   end
 
@@ -164,6 +166,14 @@ class Integrations::LlmBaseService
     }
   end
 
+  def llm_credential
+    @llm_credential ||= { api_key: hook.settings['api_key'], source: :hook }
+  end
+
+  def exception_tracking_account
+    hook.account
+  end
+
   def build_error_response_from_exception(error, messages)
     { error: error.message, request_messages: messages }
   end

lib/llm/exception_trackable.rb

@@ -0,0 +1,11 @@
+module Llm::ExceptionTrackable
+  private
+
+  def capture_llm_exception(error, credential:)
+    if credential && credential[:source] == :system
+      ChatwootExceptionTracker.new(error, account: exception_tracking_account).capture_exception
+    else
+      Rails.logger.error("[LLM] account=#{exception_tracking_account&.id} #{error.class}: #{error.message}")
+    end
+  end
+end