feat(csat): Add WhatsApp utility template analyzer with rewrite guidance (#13575)

CSAT templates for WhatsApp are submitted as Utility, but Meta may reclassify them as Marketing based on content, which can significantly increase messaging costs. This PR introduces a Captain-powered CSAT template analyzer for WhatsApp/Twilio WhatsApp that predicts utility fit, explains likely risks, and suggests safer rewrites before submission. The flow is manual (button-triggered), Captain-gated, and applies rewrites only on explicit user action. It also updates UX copy to clearly set expectations: the system submits as Utility, Meta makes the final categorization decision. Fixes https://linear.app/chatwoot/issue/CW-6424/ai-powered-whatsapp-template-classifier-for-csat-submissions https://github.com/user-attachments/assets/8fd1d6db-2f91-447c-9771-3de271b16fd9
2026-02-24 15:11:04 +04:00
parent 2b85275e26
commit 6be95e79f8
16 changed files with 711 additions and 6 deletions
--- a/app/controllers/api/v1/accounts/inbox_csat_templates_controller.rb
+++ b/app/controllers/api/v1/accounts/inbox_csat_templates_controller.rb
@@ -1,6 +1,7 @@
 class Api::V1::Accounts::InboxCsatTemplatesController < Api::V1::Accounts::BaseController
  before_action :fetch_inbox
  before_action :validate_whatsapp_channel
+  before_action :validate_captain_enabled, only: [:analyze]

  def show
    service = CsatTemplateManagementService.new(@inbox)
@@ -24,6 +25,23 @@ class Api::V1::Accounts::InboxCsatTemplatesController < Api::V1::Accounts::BaseC
    render json: { error: 'Template parameters are required' }, status: :unprocessable_entity
  end

+  def analyze
+    template_params = extract_template_params
+    return render_missing_message_error if template_params[:message].blank?
+
+    result = CsatTemplateUtilityAnalysisService.new(
+      account: Current.account,
+      inbox: @inbox,
+      message: template_params[:message],
+      button_text: template_params[:button_text],
+      language: template_params[:language]
+    ).perform
+
+    render json: result
+  rescue ActionController::ParameterMissing
+    render json: { error: 'Template parameters are required' }, status: :unprocessable_entity
+  end
+
  private

  def fetch_inbox
@@ -46,6 +64,12 @@ class Api::V1::Accounts::InboxCsatTemplatesController < Api::V1::Accounts::BaseC
    render json: { error: 'Message is required' }, status: :unprocessable_entity
  end

+  def validate_captain_enabled
+    return if Current.account.feature_enabled?('captain_integration')
+
+    render json: { error: 'Captain is required for template analysis' }, status: :forbidden
+  end
+
  def render_template_creation_result(result)
    if result[:success]
      render_successful_template_creation(result)
--- a/app/javascript/dashboard/api/inboxes.js
+++ b/app/javascript/dashboard/api/inboxes.js
@@ -42,6 +42,12 @@ class Inboxes extends CacheEnabledApiClient {
  getCSATTemplateStatus(inboxId) {
    return axios.get(`${this.url}/${inboxId}/csat_template`);
  }
+
+  analyzeCSATTemplateUtility(inboxId, template) {
+    return axios.post(`${this.url}/${inboxId}/csat_template/analyze`, {
+      template,
+    });
+  }
 }

 export default new Inboxes();
--- a/app/javascript/dashboard/i18n/locale/en/inboxMgmt.json
+++ b/app/javascript/dashboard/i18n/locale/en/inboxMgmt.json
@@ -890,6 +890,20 @@
        "CONFIRM": "Create new template",
        "CANCEL": "Go back"
      },
+      "UTILITY_ANALYZER": {
+        "ACTION": "Check utility fit",
+        "HELPER_NOTE": "Check this message before submission to improve Utility fit. The system creates a dedicated CSAT template with buttons for reporting and submits it as Utility; Meta may still reclassify it as Marketing based on content.",
+        "RESULT_LABEL": "Meta category prediction",
+        "GUIDANCE_NOTE": "This is a guidance check, not a guarantee of Meta approval.",
+        "SUGGESTION_LABEL": "Suggested utility-safe rewrite",
+        "APPLY": "Use this rewrite",
+        "ERROR_MESSAGE": "Couldn't analyze the message. Please try again.",
+        "CLASSIFICATION": {
+          "LIKELY_UTILITY": "Likely Utility",
+          "LIKELY_MARKETING": "Likely Marketing",
+          "UNCLEAR": "Needs clarification"
+        }
+      },
      "SURVEY_RULE": {
        "LABEL": "Survey rule",
        "DESCRIPTION_PREFIX": "Send the survey if the conversation",
@@ -901,7 +915,7 @@
        "SELECT_PLACEHOLDER": "select labels"
      },
      "NOTE": "Note: CSAT surveys are sent only once per conversation",
-      "WHATSAPP_NOTE": "Note: We will create a template and send it for WhatsApp approval. After being approved, surveys will be sent only once per conversation as per the survey rule.",
+      "WHATSAPP_NOTE": "Note: When you save, the system creates a dedicated CSAT template in WhatsApp (used to capture rating and feedback in reports) and submits it as Utility for approval. Meta may still classify it as Marketing based on content. After approval, surveys are sent only once per conversation as per the survey rule.",
      "API": {
        "SUCCESS_MESSAGE": "CSAT settings updated successfully",
        "ERROR_MESSAGE": "We couldn't update CSAT settings. Please try again later."
--- a/app/javascript/dashboard/routes/dashboard/settings/inbox/settingsPage/CustomerSatisfactionPage.vue
+++ b/app/javascript/dashboard/routes/dashboard/settings/inbox/settingsPage/CustomerSatisfactionPage.vue
@@ -4,6 +4,7 @@ import { useI18n } from 'vue-i18n';
 import { useAlert } from 'dashboard/composables';
 import { useStore, useMapGetter } from 'dashboard/composables/store';
 import { useInbox } from 'dashboard/composables/useInbox';
+import { useCaptain } from 'dashboard/composables/useCaptain';
 import { CSAT_DISPLAY_TYPES } from 'shared/constants/messages';

 import Icon from 'dashboard/components-next/icon/Icon.vue';
@@ -26,6 +27,7 @@ const props = defineProps({
 const { t } = useI18n();
 const store = useStore();
 const labels = useMapGetter('labels/getLabels');
+const { captainEnabled } = useCaptain();

 const { isAWhatsAppChannel, isATwilioWhatsAppChannel } = useInbox(
  props.inbox?.id
@@ -37,6 +39,8 @@ const isAnyWhatsAppChannel = computed(
 );

 const isUpdating = ref(false);
+const utilityAnalysisLoading = ref(false);
+const utilityAnalysisResult = ref(null);
 const selectedLabelValues = ref([]);
 const currentLabel = ref('');

@@ -46,7 +50,7 @@ const state = reactive({
  message: '',
  templateButtonText: 'Please rate us',
  surveyRuleOperator: 'contains',
-  templateLanguage: '',
+  templateLanguage: 'en',
 });

 const templateStatus = ref(null);
@@ -89,6 +93,9 @@ const messagePreviewData = computed(() => ({
 const shouldShowTemplateStatus = computed(
  () => templateStatus.value && !templateLoading.value
 );
+const showUtilityAnalyzer = computed(
+  () => isAnyWhatsAppChannel.value && captainEnabled.value
+);

 const templateApprovalStatus = computed(() => {
  const statusMap = {
@@ -218,6 +225,85 @@ const updateDisplayType = type => {
  state.displayType = type;
 };

+const resetUtilityAnalysis = () => {
+  utilityAnalysisResult.value = null;
+};
+
+const analyzeTemplateUtility = async () => {
+  if (!showUtilityAnalyzer.value || !state.message?.trim()) return;
+
+  utilityAnalysisLoading.value = true;
+  resetUtilityAnalysis();
+
+  try {
+    const response = await store.dispatch(
+      'inboxes/analyzeCSATTemplateUtility',
+      {
+        inboxId: props.inbox.id,
+        template: {
+          message: state.message,
+          button_text: state.templateButtonText,
+          language: state.templateLanguage,
+        },
+      }
+    );
+    utilityAnalysisResult.value = response;
+  } catch (error) {
+    const errorMessage =
+      error.response?.data?.error ||
+      t('INBOX_MGMT.CSAT.UTILITY_ANALYZER.ERROR_MESSAGE');
+    useAlert(errorMessage);
+  } finally {
+    utilityAnalysisLoading.value = false;
+  }
+};
+
+const applyUtilitySuggestion = () => {
+  const suggestion = utilityAnalysisResult.value?.optimized_message;
+  if (!suggestion) return;
+
+  state.message = suggestion;
+  resetUtilityAnalysis();
+};
+
+watch(
+  () => [state.message, state.templateButtonText, state.templateLanguage],
+  (newValues, oldValues) => {
+    if (!oldValues || !utilityAnalysisResult.value) {
+      return;
+    }
+
+    const changed = newValues.some(
+      (value, index) => value !== oldValues[index]
+    );
+    if (changed) {
+      resetUtilityAnalysis();
+    }
+  }
+);
+
+const getUtilityClassificationLabel = classification => {
+  if (classification === 'LIKELY_UTILITY') {
+    return t('INBOX_MGMT.CSAT.UTILITY_ANALYZER.CLASSIFICATION.LIKELY_UTILITY');
+  }
+  if (classification === 'LIKELY_MARKETING') {
+    return t(
+      'INBOX_MGMT.CSAT.UTILITY_ANALYZER.CLASSIFICATION.LIKELY_MARKETING'
+    );
+  }
+  return t('INBOX_MGMT.CSAT.UTILITY_ANALYZER.CLASSIFICATION.UNCLEAR');
+};
+
+const getUtilityClassificationClass = classification => {
+  if (classification === 'LIKELY_UTILITY') {
+    return 'bg-n-teal-3 text-n-teal-11';
+  }
+  if (classification === 'LIKELY_MARKETING') {
+    return 'bg-n-ruby-3 text-n-ruby-11';
+  }
+  return 'bg-n-amber-3 text-n-amber-11';
+};
+
 const updateSurveyRuleOperator = operator => {
  state.surveyRuleOperator = operator;
 };
@@ -450,6 +536,70 @@ const handleConfirmTemplateUpdate = async () => {
                    class="w-full"
                  />
                </WithLabel>
+                <div v-if="showUtilityAnalyzer" class="flex flex-col gap-2">
+                  <NextButton
+                    sm
+                    slate
+                    :label="$t('INBOX_MGMT.CSAT.UTILITY_ANALYZER.ACTION')"
+                    :is-loading="utilityAnalysisLoading"
+                    :disabled="!state.message?.trim()"
+                    @click="analyzeTemplateUtility"
+                  />
+                  <p class="text-xs text-n-slate-11">
+                    {{ $t('INBOX_MGMT.CSAT.UTILITY_ANALYZER.HELPER_NOTE') }}
+                  </p>
+                </div>
+
+                <div
+                  v-if="utilityAnalysisResult"
+                  class="flex flex-col gap-3 p-3 rounded-xl outline outline-1 outline-n-weak bg-n-alpha-1"
+                >
+                  <div class="flex gap-2 items-center">
+                    <span class="text-sm font-medium text-n-slate-12">
+                      {{ $t('INBOX_MGMT.CSAT.UTILITY_ANALYZER.RESULT_LABEL') }}
+                    </span>
+                    <span
+                      class="px-2 py-0.5 text-xs font-medium rounded-full"
+                      :class="
+                        getUtilityClassificationClass(
+                          utilityAnalysisResult.classification
+                        )
+                      "
+                    >
+                      {{
+                        getUtilityClassificationLabel(
+                          utilityAnalysisResult.classification
+                        )
+                      }}
+                    </span>
+                  </div>
+                  <p class="text-xs text-n-slate-11">
+                    {{ $t('INBOX_MGMT.CSAT.UTILITY_ANALYZER.GUIDANCE_NOTE') }}
+                  </p>
+                  <div
+                    v-if="
+                      utilityAnalysisResult.optimized_message &&
+                      utilityAnalysisResult.classification !== 'LIKELY_UTILITY'
+                    "
+                    class="flex flex-col gap-2"
+                  >
+                    <p class="text-xs font-medium text-n-slate-12">
+                      {{
+                        $t('INBOX_MGMT.CSAT.UTILITY_ANALYZER.SUGGESTION_LABEL')
+                      }}
+                    </p>
+                    <p class="text-sm text-n-slate-12">
+                      {{ utilityAnalysisResult.optimized_message }}
+                    </p>
+                    <NextButton
+                      sm
+                      faded
+                      slate
+                      :label="$t('INBOX_MGMT.CSAT.UTILITY_ANALYZER.APPLY')"
+                      @click="applyUtilitySuggestion"
+                    />
+                  </div>
+                </div>
                <Input
                  v-model="state.templateButtonText"
                  :label="$t('INBOX_MGMT.CSAT.BUTTON_TEXT.LABEL')"
--- a/app/javascript/dashboard/store/modules/inboxes.js
+++ b/app/javascript/dashboard/store/modules/inboxes.js
@@ -360,6 +360,13 @@ export const actions = {
    const response = await InboxesAPI.getCSATTemplateStatus(inboxId);
    return response.data;
  },
+  analyzeCSATTemplateUtility: async (_, { inboxId, template }) => {
+    const response = await InboxesAPI.analyzeCSATTemplateUtility(
+      inboxId,
+      template
+    );
+    return response.data;
+  },
 };

 export const mutations = {
--- a/app/services/csat_template_management_service.rb
+++ b/app/services/csat_template_management_service.rb
@@ -110,6 +110,7 @@ class CsatTemplateManagementService

    template_service = Twilio::CsatTemplateService.new(@inbox.channel)
    status_result = template_service.get_template_status(content_sid)
+    return { template_exists: false, error: 'Template not found' } unless status_result.is_a?(Hash)

    if status_result[:success]
      {
@@ -130,6 +131,7 @@ class CsatTemplateManagementService
  def get_whatsapp_template_status(template)
    template_name = template['name'] || CsatTemplateNameService.csat_template_name(@inbox.id)
    status_result = Whatsapp::CsatTemplateService.new(@inbox.channel).get_template_status(template_name)
+    return { template_exists: false, error: 'Template not found' } unless status_result.is_a?(Hash)

    if status_result[:success]
      {
--- a/app/services/csat_template_utility_analysis_service.rb
+++ b/app/services/csat_template_utility_analysis_service.rb
@@ -0,0 +1,86 @@
+class CsatTemplateUtilityAnalysisService
+  include CsatTemplateUtilityRubric
+
+  pattr_initialize [:account!, :inbox!, :message!, { button_text: nil, language: 'en' }]
+
+  def perform
+    baseline = rule_based_result
+    return baseline if baseline[:classification] == 'LIKELY_MARKETING'
+
+    llm_result = llm_result_or_nil(baseline)
+    llm_result || baseline
+  end
+
+  private
+
+  def llm_result_or_nil(baseline)
+    llm_output = Captain::CsatUtilityAnalysisService.new(
+      account: account,
+      message: message,
+      button_text: button_text,
+      language: language,
+      baseline: baseline
+    ).perform
+
+    return nil if llm_output[:error]
+
+    normalize_llm_result(llm_output, baseline: baseline)
+  rescue StandardError => e
+    Rails.logger.error("CSAT utility LLM analysis failed for inbox #{inbox.id}: #{e.message}")
+    nil
+  end
+
+  def normalize_llm_result(result, baseline:)
+    classification = normalized_classification(result[:classification], baseline: baseline)
+    optimized_message = result[:optimized_message].presence || baseline[:optimized_message]
+    optimized_message = baseline[:optimized_message] if baseline[:classification] == 'LIKELY_MARKETING'
+
+    {
+      classification: classification,
+      optimized_message: optimized_message
+    }
+  end
+
+  def normalized_classification(value, baseline:)
+    raw = value.to_s
+    return 'LIKELY_MARKETING' if baseline[:classification] == 'LIKELY_MARKETING'
+
+    raw
+  end
+
+  def rule_based_result
+    text = sanitized_message
+    marketing_hits_count = MARKETING_PATTERNS.count { |pattern| pattern.match?(text) }
+    utility_hits_count = UTILITY_PATTERNS.count { |pattern| pattern.match?(text) }
+    criteria = evaluate_criteria(text: text, marketing_hits_count: marketing_hits_count)
+    classification = classify(criteria: criteria, utility_hits_count: utility_hits_count)
+    build_rule_payload(
+      classification: classification
+    )
+  end
+
+  def build_rule_payload(payload)
+    {
+      classification: payload[:classification],
+      optimized_message: optimized_message_for(payload[:classification])
+    }
+  end
+
+  def sanitized_message
+    message.to_s.squish
+  end
+
+  def classify(criteria:, utility_hits_count:)
+    return 'LIKELY_MARKETING' unless criteria[:marketing_prohibition]
+    return 'LIKELY_MARKETING' unless criteria[:prohibited_content]
+    return 'LIKELY_UTILITY' if criteria.values.all? && utility_hits_count >= 2
+
+    'UNCLEAR'
+  end
+
+  def optimized_message_for(classification)
+    return sanitized_message if classification == 'LIKELY_UTILITY'
+
+    build_input_aware_utility_message
+  end
+end
--- a/app/services/csat_template_utility_rubric.rb
+++ b/app/services/csat_template_utility_rubric.rb
@@ -0,0 +1,125 @@
+# rubocop:disable Metrics/ModuleLength
+module CsatTemplateUtilityRubric
+  LANGUAGE_FALLBACKS = {
+    'en' => {
+      support_request: 'support request',
+      support_ticket: 'support ticket',
+      support_conversation: 'support conversation',
+      status_closed: 'closed',
+      status_resolved: 'resolved',
+      status_completed: 'completed',
+      line_status: 'Your %<subject>s has been %<status>s.',
+      line_help: 'If you still need help, simply reply to this message.',
+      line_rate: 'To rate this support interaction, please use the button below.'
+    }
+  }.freeze
+
+  MARKETING_PATTERNS = [
+    /\b(discounts?|offers?|promos?|promotions?|deals?|sales?|buy|shop|subscribe)\b/i,
+    /\b(limited\s*time|don't\s*miss|exclusive|special\s*offer)\b/i,
+    /\b(click\s*(here|below)\s*to\s*(buy|get|shop))\b/i,
+    /\b(new\s*(plans?|products?|services?))\b/i
+  ].freeze
+
+  TRANSACTION_TRIGGER_PATTERNS = [
+    /\b(closed|closing|resolved|completed)\b/i,
+    /\b(ticket|request|case|conversation|support)\b/i
+  ].freeze
+
+  TRANSACTIONAL_CONTENT_PATTERNS = [
+    /\b(ticket|request|case|conversation)\b/i,
+    /\b(reply\s+to\s+this\s+message|if\s+you\s+still\s+need\s+help)\b/i,
+    /\b(rate|califica|calificar)\b/i
+  ].freeze
+
+  PROHIBITED_CONTENT_PATTERNS = [
+    /\b(contest|sweepstake|lottery|quiz)\b/i,
+    /\b(password|otp|pin|cvv|credit\s*card)\b/i,
+    /\b(weapon|drugs|gambling)\b/i
+  ].freeze
+
+  STATUS_PATTERNS = {
+    'closed' => /\b(closed|closing)\b/i,
+    'resolved' => /\b(resolved|resolve[sd]?)\b/i,
+    'completed' => /\b(completed|complete[sd]?)\b/i
+  }.freeze
+
+  SUBJECT_PATTERNS = {
+    'support ticket' => /\b(ticket)\b/i,
+    'support conversation' => /\b(conversation|chat)\b/i,
+    'support request' => /\b(request|case|support)\b/i
+  }.freeze
+
+  UTILITY_PATTERNS = [
+    /\b(support|ticket|request|conversation|case)\b/i,
+    /\b(closed|resolved|completed)\b/i,
+    /\b(reply\s+to\s+this\s+message\b)/i,
+    /\b(if\s+you\s+still\s+need\s+help)\b/i,
+    /\b(rate\s+this\s+(support|interaction|conversation))\b/i
+  ].freeze
+
+  private
+
+  def build_input_aware_utility_message
+    text = translation_pack
+    subject = detected_subject
+    status = detected_status
+    intro = extracted_intro_sentence
+
+    parts = []
+    parts << intro if intro.present?
+    parts << format(text[:line_status], subject: subject, status: status)
+    parts << text[:line_help]
+    parts << text[:line_rate]
+    parts.join(' ')
+  end
+
+  def detected_status
+    matched = STATUS_PATTERNS.find { |_key, pattern| pattern.match?(sanitized_message) }
+    status_key = matched&.first || 'closed'
+    translation_pack[:"status_#{status_key}"]
+  end
+
+  def detected_subject
+    matched = SUBJECT_PATTERNS.find { |_key, pattern| pattern.match?(sanitized_message) }
+    subject_key = matched&.first&.tr(' ', '_') || 'support_request'
+    translation_pack[subject_key.to_sym]
+  end
+
+  def extracted_intro_sentence
+    first_sentence = sanitized_message.split(/(?<=[.!?])\s+/).first.to_s
+    return nil if first_sentence.blank?
+    return nil if MARKETING_PATTERNS.any? { |pattern| pattern.match?(first_sentence) }
+    return nil unless first_sentence.match?(/\b(thanks|thank you|hello|hi)\b/i)
+
+    normalized = first_sentence.gsub(/\s+/, ' ').strip
+    normalized.ends_with?('.', '!', '?') ? normalized : "#{normalized}."
+  end
+
+  def translation_pack
+    LANGUAGE_FALLBACKS.fetch(primary_language_code, LANGUAGE_FALLBACKS['en'])
+  end
+
+  def primary_language_code
+    language.to_s.downcase.split(/[-_]/).first
+  end
+
+  def evaluate_criteria(text:, marketing_hits_count:)
+    {
+      trigger: TRANSACTION_TRIGGER_PATTERNS.any? { |pattern| pattern.match?(text) },
+      transactional_content: TRANSACTIONAL_CONTENT_PATTERNS.count { |pattern| pattern.match?(text) } >= 2,
+      marketing_prohibition: marketing_hits_count.zero?,
+      prohibited_content: PROHIBITED_CONTENT_PATTERNS.none? { |pattern| pattern.match?(text) },
+      clarity_and_utility: clear_utility_intent?(text)
+    }
+  end
+
+  def clear_utility_intent?(text)
+    has_support_context = text.match?(/\b(support|ticket|request|case|conversation)\b/i)
+    has_actionable_next_step = text.match?(/\b(reply\s+to\s+this\s+message)\b/i) ||
+                               text.match?(/\b(if\s+you\s+still\s+need\s+help)\b/i) ||
+                               text.match?(/\b(rate\s+this\s+(support|interaction|conversation))\b/i)
+    has_support_context && has_actionable_next_step
+  end
+end
+# rubocop:enable Metrics/ModuleLength
--- a/app/services/whatsapp/csat_template_service.rb
+++ b/app/services/whatsapp/csat_template_service.rb
@@ -2,7 +2,7 @@ class Whatsapp::CsatTemplateService
  DEFAULT_BUTTON_TEXT = 'Please rate us'.freeze
  DEFAULT_LANGUAGE = 'en'.freeze
  WHATSAPP_API_VERSION = 'v14.0'.freeze
-  TEMPLATE_CATEGORY = 'MARKETING'.freeze
+  TEMPLATE_CATEGORY = 'UTILITY'.freeze
  TEMPLATE_STATUS_PENDING = 'PENDING'.freeze

  def initialize(whatsapp_channel)