feat: Interface to validate response_source (#8894)

- This PR adds a UI to validate the response source quality quickly. It also helps to test with sample questions and update responses in the database when missing.

Co-authored-by: Pranav Raj S <pranav@chatwoot.com>

app/controllers/super_admin/application_controller.rb

@@ -5,6 +5,10 @@
 # If you want to add pagination or other controller-level concerns,
 # you're free to overwrite the RESTful controller actions.
 class SuperAdmin::ApplicationController < Administrate::ApplicationController
+  include ActionView::Helpers::TagHelper
+  include ActionView::Context
+
+  helper_method :render_vue_component
   # authenticiation done via devise : SuperAdmin Model
   before_action :authenticate_super_admin!
 
@@ -23,6 +27,17 @@ class SuperAdmin::ApplicationController < Administrate::ApplicationController
 
   private
 
+  def render_vue_component(component_name, props = {})
+    html_options = {
+      id: 'app',
+      data: {
+        component_name: component_name,
+        props: props.to_json
+      }
+    }
+    content_tag(:div, '', html_options)
+  end
+
   def invalid_action_perfomed
     # rubocop:disable Rails/I18nLocaleTexts
     flash[:error] = 'Invalid action performed'

app/javascript/packs/superadmin_pages.js

@@ -1 +1,28 @@
 import 'chart.js';
+import Vue from 'vue';
+import VueDOMPurifyHTML from 'vue-dompurify-html';
+Vue.use(VueDOMPurifyHTML);
+
+const PlaygroundIndex = () =>
+  import('../superadmin_pages/views/playground/Index.vue');
+
+const ComponentMapping = {
+  PlaygroundIndex: PlaygroundIndex,
+};
+
+const renderComponent = (componentName, props) => {
+  Vue.component(componentName, ComponentMapping[componentName]);
+  new Vue({
+    data: { props: props },
+    template: `<${componentName} :component-data="props"/>`,
+  }).$mount('#app');
+};
+
+document.addEventListener('DOMContentLoaded', () => {
+  const element = document.getElementById('app');
+  if (element) {
+    const componentName = element.dataset.componentName;
+    const props = JSON.parse(element.dataset.props);
+    renderComponent(componentName, props);
+  }
+});

app/javascript/superadmin_pages/components/playground/BotMessage.vue

@@ -0,0 +1,17 @@
+<script setup>
+defineProps({
+  message: {
+    type: String,
+    required: true,
+  },
+});
+</script>
+
+<template>
+  <div class="w-full mb-4 flex items-center justify-start">
+    <div
+      v-dompurify-html="message"
+      class="px-4 py-3 bg-white max-w-4xl text-slate-700 leading-6 text-sm rounded-md inline-block border border-slate-100"
+    />
+  </div>
+</template>

app/javascript/superadmin_pages/components/playground/Header.vue

@@ -0,0 +1,40 @@
+<script setup>
+defineProps({
+  responseSourcePath: {
+    type: String,
+    required: true,
+  },
+  responseSourceName: {
+    type: String,
+    required: true,
+  },
+});
+</script>
+
+<template>
+  <header
+    class="flex items-center px-8 py-4 bg-white border-b border-slate-100"
+    role="banner"
+  >
+    <a :href="responseSourcePath" class="text-woot-500 hover:underline mr-4">
+      Back
+    </a>
+    <div
+      class="border border-solid border-slate-100 text-slate-700 mr-4 p-2 rounded-full"
+    >
+      <svg width="24" height="24"><use xlink:href="#icon-mist-fill" /></svg>
+    </div>
+    <div class="flex flex-col h-14 justify-center">
+      <h1 id="page-title" class="text-base font-medium text-slate-900">
+        Robin AI playground
+      </h1>
+      <p class="text-sm text-slate-600">
+        Chat with the source
+        <span class="font-medium">
+          {{ responseSourceName }}
+        </span>
+        and evaluate it’s efficiency.
+      </p>
+    </div>
+  </header>
+</template>

app/javascript/superadmin_pages/components/playground/TypingIndicator.vue

@@ -0,0 +1,13 @@
+<script setup>
+import TypingIndicator from './assets/typing.gif';
+</script>
+
+<template>
+  <div class="w-full mb-4 flex items-center justify-start">
+    <div
+      class="px-2 py-2 bg-white max-w-4xl text-slate-700 leading-6 text-sm rounded-md inline-block border border-slate-100"
+    >
+      <img :src="TypingIndicator" alt="TypingIndicator" class="h-4" />
+    </div>
+  </div>
+</template>

app/javascript/superadmin_pages/components/playground/UserMessage.vue

@@ -0,0 +1,17 @@
+<script setup>
+defineProps({
+  message: {
+    type: String,
+    required: true,
+  },
+});
+</script>
+
+<template>
+  <div class="w-full mb-4 flex items-center justify-end">
+    <div
+      v-dompurify-html="message"
+      class="px-4 py-3 bg-woot-400 text-white text-sm rounded-md inline-block"
+    />
+  </div>
+</template>

app/javascript/superadmin_pages/views/playground/Index.vue

@@ -0,0 +1,139 @@
+<template>
+  <section class="h-full w-full flex flex-col bg-slate-25">
+    <PlaygroundHeader
+      :response-source-name="componentData.responseSourceName"
+      :response-source-path="componentData.responseSourcePath"
+    />
+    <div ref="chatContainer" class="flex-1 overflow-auto px-8 py-4">
+      <div
+        v-for="message in messages"
+        :id="`message-${message.id}`"
+        :key="message.id"
+      >
+        <UserMessage
+          v-if="message.type === 'User'"
+          :message="formatMessage(message.content)"
+        />
+        <BotMessage v-else :message="formatMessage(message.content)" />
+      </div>
+      <TypingIndicator v-if="isWaiting" />
+    </div>
+    <div class="w-full px-8 py-6">
+      <textarea
+        ref="messageInput"
+        v-model="messageContent"
+        :rows="4"
+        class="resize-none block p-2.5 w-full text-sm text-gray-900 bg-gray-50 rounded-lg border !outline-2 border-slate-100 focus:ring-woot-500 focus:border-woot-500 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-woot-500 dark:focus:border-woot-500"
+        placeholder="Type a message... [CMD/CTRL + Enter to send]"
+        autofocus
+        autocomplete="off"
+      />
+    </div>
+  </section>
+</template>
+
+<script>
+import { buildHotKeys } from 'shared/helpers/KeyboardHelpers';
+import messageFormatterMixin from 'shared/mixins/messageFormatterMixin';
+import Header from '../../components/playground/Header.vue';
+import UserMessage from '../../components/playground/UserMessage.vue';
+import BotMessage from '../../components/playground/BotMessage.vue';
+import TypingIndicator from '../../components/playground/TypingIndicator.vue';
+
+export default {
+  components: {
+    PlaygroundHeader: Header,
+    UserMessage,
+    BotMessage,
+    TypingIndicator,
+  },
+  mixins: [messageFormatterMixin],
+  props: {
+    componentData: {
+      type: Object,
+      default: () => ({}),
+    },
+  },
+  data() {
+    return { messages: [], messageContent: '', isWaiting: false };
+  },
+  computed: {
+    previousMessages() {
+      return this.messages.map(message => ({
+        type: message.type,
+        message: message.content,
+      }));
+    },
+  },
+  mounted() {
+    this.focusInput();
+    document.addEventListener('keydown', this.handleKeyEvents);
+  },
+  beforeDestroy() {
+    document.removeEventListener('keydown', this.handleKeyEvents);
+  },
+  methods: {
+    handleKeyEvents(e) {
+      const keyCode = buildHotKeys(e);
+      if (['meta+enter', 'ctrl+enter'].includes(keyCode)) {
+        this.onMessageSend();
+      }
+    },
+    focusInput() {
+      this.$refs.messageInput.focus();
+    },
+    onMessageSend() {
+      this.addMessageToData('User', this.messageContent);
+      this.sendMessageToServer(this.messageContent);
+    },
+    scrollToLastMessage() {
+      this.$nextTick(() => {
+        const messageId = this.messages[this.messages.length - 1].id;
+        const messageElement = document.getElementById(`message-${messageId}`);
+        messageElement.scrollIntoView({ behavior: 'smooth' });
+      });
+    },
+    addMessageToData(type, content) {
+      this.messages.push({ id: this.messages.length, type, content });
+      this.scrollToLastMessage();
+    },
+    async sendMessageToServer(messageContent) {
+      this.messageContent = '';
+      this.isWaiting = true;
+      const csrfToken = document
+        .querySelector('meta[name="csrf-token"]')
+        .getAttribute('content');
+
+      try {
+        const response = await fetch(window.location.href, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'X-CSRF-Token': csrfToken,
+          },
+          body: JSON.stringify({
+            message: messageContent,
+            previous_messages: this.previousMessages,
+          }),
+          credentials: 'include',
+        });
+
+        if (!response.ok) {
+          throw new Error(`HTTP error! status: ${response.status}`);
+        }
+
+        const { message } = await response.json();
+        this.addMessageToData('Bot', message);
+      } catch (error) {
+        this.addMessageToData(
+          'bot',
+          'Error: Could not retrieve response. Please check the console for more details.'
+        );
+      } finally {
+        this.isWaiting = false;
+        this.focusInput();
+      }
+    },
+  },
+};
+</script>

app/views/super_admin/accounts/show.html.erb

@@ -25,7 +25,7 @@ as well as a link to its edit page.
 
   <div>
     <%= link_to(
-      t("administrate.actions.edit_resource", name: page.page_title),
+      "Edit",
       [:edit, namespace, page.resource],
       class: "button",
     ) if accessible_action?(page.resource, :edit) %>

app/views/super_admin/application/show.html.erb

@@ -0,0 +1,65 @@
+<%#
+# Show
+
+This view is the template for the show page.
+It renders the attributes of a resource,
+as well as a link to its edit page.
+
+## Local variables:
+
+- `page`:
+  An instance of [Administrate::Page::Show][1].
+  Contains methods for accessing the resource to be displayed on the page,
+  as well as helpers for describing how each attribute of the resource
+  should be displayed.
+
+[1]: http://www.rubydoc.info/gems/administrate/Administrate/Page/Show
+%>
+
+<% content_for(:title) { t("administrate.actions.show_resource", name: page.page_title) } %>
+
+<header class="main-content__header">
+  <h1 class="main-content__page-title">
+    <%= content_for(:title) %>
+  </h1>
+
+  <div>
+    <%= link_to(
+      "Edit",
+      [:edit, namespace, page.resource],
+      class: "button",
+    ) if accessible_action?(page.resource, :edit) %>
+
+    <%= link_to(
+      t("administrate.actions.destroy"),
+      [namespace, page.resource],
+      class: "button button--danger",
+      method: :delete,
+      data: { confirm: t("administrate.actions.confirm") }
+    ) if accessible_action?(page.resource, :destroy) %>
+  </div>
+</header>
+
+<section class="main-content__body">
+  <dl>
+    <% page.attributes.each do |title, attributes| %>
+      <fieldset class="<%= "field-unit--nested" if title.present? %>">
+        <% if title.present? %>
+          <legend><%= t "helpers.label.#{page.resource_name}.#{title}", default: title %></legend>
+        <% end %>
+
+        <% attributes.each do |attribute| %>
+          <dt class="attribute-label" id="<%= attribute.name %>">
+          <%= t(
+            "helpers.label.#{resource_name}.#{attribute.name}",
+            default: page.resource.class.human_attribute_name(attribute.name),
+          ) %>
+          </dt>
+
+          <dd class="attribute-data attribute-data--<%=attribute.html_class%>"
+              ><%= render_field attribute, page: page %></dd>
+        <% end %>
+      </fieldset>
+    <% end %>
+  </dl>
+</section>

app/views/super_admin/users/show.html.erb

@@ -25,10 +25,10 @@ as well as a link to its edit page.
 
   <div>
     <%= link_to(
-      t("administrate.actions.edit_resource", name: page.page_title),
+            t("administrate.actions.edit"),
-      [:edit, namespace, page.resource],
+            [:edit, namespace,  page.resource.becomes(User)],
-      class: "button",
+            class: "button",
-    ) if accessible_action?(:edit, page.resource) %>
+          ) if authorized_action? page.resource, :edit %>
   </div>
 </header>
 

config/routes.rb

@@ -453,7 +453,10 @@ Rails.application.routes.draw do
       end
 
       resources :access_tokens, only: [:index, :show]
-      resources :response_sources, only: [:index, :show, :new, :create, :edit, :update, :destroy]
+      resources :response_sources, only: [:index, :show, :new, :create, :edit, :update, :destroy] do
+        get :chat, on: :member
+        post :chat, on: :member, action: :process_chat
+      end
       resources :response_documents, only: [:index, :show, :new, :create, :edit, :update, :destroy]
       resources :responses, only: [:index, :show, :new, :create, :edit, :update, :destroy]
       resources :installation_configs, only: [:index, :new, :create, :show, :edit, :update]

enterprise/app/controllers/super_admin/enterprise_base_controller.rb

@@ -0,0 +1,8 @@
+class SuperAdmin::EnterpriseBaseController < SuperAdmin::ApplicationController
+  before_action :prepend_view_paths
+
+  # Prepend the view path to the enterprise/app/views won't be available by default
+  def prepend_view_paths
+    prepend_view_path 'enterprise/app/views/'
+  end
+end

enterprise/app/controllers/super_admin/response_documents_controller.rb

@@ -1,4 +1,4 @@
-class SuperAdmin::ResponseDocumentsController < SuperAdmin::ApplicationController
+class SuperAdmin::ResponseDocumentsController < SuperAdmin::EnterpriseBaseController
   # Overwrite any of the RESTful controller actions to implement custom behavior
   # For example, you may want to send an email after a foo is updated.
   #

enterprise/app/controllers/super_admin/response_sources_controller.rb

@@ -1,4 +1,4 @@
-class SuperAdmin::ResponseSourcesController < SuperAdmin::ApplicationController
+class SuperAdmin::ResponseSourcesController < SuperAdmin::EnterpriseBaseController
   # Overwrite any of the RESTful controller actions to implement custom behavior
   # For example, you may want to send an email after a foo is updated.
   #
@@ -41,4 +41,36 @@ class SuperAdmin::ResponseSourcesController < SuperAdmin::ApplicationController
 
   # See https://administrate-demo.herokuapp.com/customizing_controller_actions
   # for more information
+
+  before_action :set_response_source, only: %i[chat process_chat]
+
+  def chat; end
+
+  def process_chat
+    previous_messages = []
+    get_previous_messages(previous_messages)
+    robin_response = ChatGpt.new(
+      Enterprise::MessageTemplates::ResponseBotService.response_sections(params[:message], @response_source)
+    ).generate_response(
+      params[:message], previous_messages
+    )
+    message_content = robin_response['response']
+    if robin_response['context_ids'].present?
+      message_content += Enterprise::MessageTemplates::ResponseBotService.generate_sources_section(robin_response['context_ids'])
+    end
+    render json: { message: message_content }
+  end
+
+  private
+
+  def get_previous_messages(previous_messages)
+    params[:previous_messages].each do |message|
+      role = message['type'] == 'user' ? 'user' : 'system'
+      previous_messages << { content: message['message'], role: role }
+    end
+  end
+
+  def set_response_source
+    @response_source = requested_resource
+  end
 end

enterprise/app/controllers/super_admin/responses_controller.rb

@@ -1,4 +1,4 @@
-class SuperAdmin::ResponsesController < SuperAdmin::ApplicationController
+class SuperAdmin::ResponsesController < SuperAdmin::EnterpriseBaseController
   # Overwrite any of the RESTful controller actions to implement custom behavior
   # For example, you may want to send an email after a foo is updated.
   #

enterprise/app/jobs/response_builder_job.rb

@@ -17,6 +17,7 @@ class ResponseBuilderJob < ApplicationJob
   def prepare_data(response_document)
     {
       model: 'gpt-3.5-turbo',
+      response_format: { type: 'json_object' },
       messages: [
         {
           role: 'system',
@@ -32,16 +33,15 @@ class ResponseBuilderJob < ApplicationJob
 
   def system_message_content
     <<~SYSTEM_MESSAGE_CONTENT
-       You are a content writer looking to convert user content into short FAQs which can be added to your website's helper centre.
+      You are a content writer looking to convert user content into short FAQs which can be added to your website's helper centre.
-       Format the webpage content provided in the message to FAQ format like the following example.#{' '}
+      Format the webpage content provided in the message to FAQ format mentioned below in the json
-       Ensure that you only generate faqs from the information provider in the message.#{' '}
+      Ensure that you only generate faqs from the information provider in the message.
-       Ensure that output is always valid json.#{'  '}
+      Ensure that output is always valid json.
-       If no match is available, return an empty JSON.
+      If no match is available, return an empty JSON.
-       ```
+
-      [ { "question": "What is the pricing?",
+      ```json
-         "answer" : " There are different pricing tiers available."
+      {faqs: [{question: '', answer: ''}]
-       }]
+      ```
-       ```
     SYSTEM_MESSAGE_CONTENT
   end
 
@@ -67,7 +67,7 @@ class ResponseBuilderJob < ApplicationJob
 
     return if content.nil?
 
-    faqs = JSON.parse(content.strip)
+    faqs = JSON.parse(content.strip).fetch('faqs', [])
 
     faqs.each do |faq|
       response_document.responses.create!(

enterprise/app/models/response_source.rb

@@ -25,4 +25,9 @@ class ResponseSource < ApplicationRecord
   has_many :responses, dependent: :destroy_async
 
   accepts_nested_attributes_for :response_documents
+
+  def get_responses(query)
+    embedding = Openai::EmbeddingsService.new.get_embedding(query)
+    responses.active.nearest_neighbors(:embedding, embedding, distance: 'cosine').first(5)
+  end
 end

enterprise/app/services/enterprise/message_templates/response_bot_service.rb

@@ -1,6 +1,39 @@
 class Enterprise::MessageTemplates::ResponseBotService
   pattr_initialize [:conversation!]
 
+  def self.generate_sources_section(article_ids)
+    sources_content = ''
+
+    articles_hash = get_article_hash(article_ids.uniq)
+
+    articles_hash.first(3).each do |article_hash|
+      sources_content += " - [#{article_hash[:response].question}](#{article_hash[:response_document].document_link}) \n"
+    end
+    sources_content = "\n \n \n **Sources**  \n#{sources_content}" if sources_content.present?
+    sources_content
+  end
+
+  def self.get_article_hash(article_ids)
+    seen_documents = Set.new
+    article_ids.uniq.filter_map do |article_id|
+      response = Response.find(article_id)
+      response_document = response.response_document
+      next if response_document.blank? || seen_documents.include?(response_document)
+
+      seen_documents << response_document
+      { response: response, response_document: response_document }
+    end
+  end
+
+  def self.response_sections(content, response_source)
+    sections = ''
+
+    response_source.get_responses(content).each do |response|
+      sections += "{context_id: #{response.id}, context: #{response.question} ? #{response.answer}},"
+    end
+    sections
+  end
+
   def perform
     ActiveRecord::Base.transaction do
       @response = get_response(conversation.messages.incoming.last.content)
@@ -12,15 +45,6 @@ class Enterprise::MessageTemplates::ResponseBotService
     true
   end
 
-  def response_sections(content)
-    sections = ''
-
-    inbox.get_responses(content).each do |response|
-      sections += "{context_id: #{response.id}, context: #{response.question} ? #{response.answer}},"
-    end
-    sections
-  end
-
   private
 
   delegate :contact, :account, :inbox, to: :conversation
@@ -28,7 +52,7 @@ class Enterprise::MessageTemplates::ResponseBotService
   def get_response(content)
     previous_messages = []
     get_previous_messages(previous_messages)
-    ChatGpt.new(response_sections(content)).generate_response('', previous_messages)
+    ChatGpt.new(self.class.response_sections(content, inbox)).generate_response('', previous_messages)
   end
 
   def get_previous_messages(previous_messages)
@@ -63,24 +87,11 @@ class Enterprise::MessageTemplates::ResponseBotService
 
   def create_messages
     message_content = @response['response']
-    message_content += generate_sources_section if @response['context_ids'].present?
+    message_content += self.class.generate_sources_section(@response['context_ids']) if @response['context_ids'].present?
 
     create_outgoing_message(message_content)
   end
 
-  def generate_sources_section
-    article_ids = @response['context_ids']
-    sources_content = ''
-
-    articles_hash = get_article_hash(article_ids.uniq)
-
-    articles_hash.first(3).each do |article_hash|
-      sources_content += " - [#{article_hash[:response].question}](#{article_hash[:response_document].document_link}) \n"
-    end
-    sources_content = "\n \n \n **Sources**  \n#{sources_content}" if sources_content.present?
-    sources_content
-  end
-
   def create_outgoing_message(message_content)
     conversation.messages.create!(
       {
@@ -91,16 +102,4 @@ class Enterprise::MessageTemplates::ResponseBotService
       }
     )
   end
-
-  def get_article_hash(article_ids)
-    seen_documents = Set.new
-    article_ids.uniq.filter_map do |article_id|
-      response = Response.find(article_id)
-      response_document = response.response_document
-      next if response_document.blank? || seen_documents.include?(response_document)
-
-      seen_documents << response_document
-      { response: response, response_document: response_document }
-    end
-  end
 end

enterprise/app/views/super_admin/response_sources/chat.html.erb

@@ -0,0 +1,5 @@
+<% content_for :title, "Robin AI playground: #{@response_source.name}" %>
+<%= render_vue_component('PlaygroundIndex', {
+  responseSourceName: @response_source.name,
+  responseSourcePath: super_admin_response_source_path(@response_source)
+}) %>

enterprise/app/views/super_admin/response_sources/show.html.erb

@@ -0,0 +1,71 @@
+<%#
+# Show
+
+This view is the template for the show page.
+It renders the attributes of a resource,
+as well as a link to its edit page.
+
+## Local variables:
+
+- `page`:
+  An instance of [Administrate::Page::Show][1].
+  Contains methods for accessing the resource to be displayed on the page,
+  as well as helpers for describing how each attribute of the resource
+  should be displayed.
+
+[1]: http://www.rubydoc.info/gems/administrate/Administrate/Page/Show
+%>
+
+<% content_for(:title) { t("administrate.actions.show_resource", name: page.page_title) } %>
+
+<header class="main-content__header">
+  <h1 class="main-content__page-title">
+    <%= content_for(:title) %>
+  </h1>
+
+  <div>
+    <%= link_to(
+      "Chat",
+      [:chat, namespace, page.resource],
+      class: "button"
+    ) %>
+
+    <%= link_to(
+      "Edit",
+      [:edit, namespace, page.resource],
+      class: "button",
+    ) if accessible_action?(page.resource, :edit) %>
+
+    <%= link_to(
+      t("administrate.actions.destroy"),
+      [namespace, page.resource],
+      class: "button button--danger",
+      method: :delete,
+      data: { confirm: t("administrate.actions.confirm") }
+    ) if accessible_action?(page.resource, :destroy) %>
+  </div>
+</header>
+
+<section class="main-content__body">
+  <dl>
+    <% page.attributes.each do |title, attributes| %>
+      <fieldset class="<%= "field-unit--nested" if title.present? %>">
+        <% if title.present? %>
+          <legend><%= t "helpers.label.#{page.resource_name}.#{title}", default: title %></legend>
+        <% end %>
+
+        <% attributes.each do |attribute| %>
+          <dt class="attribute-label" id="<%= attribute.name %>">
+          <%= t(
+            "helpers.label.#{resource_name}.#{attribute.name}",
+            default: page.resource.class.human_attribute_name(attribute.name),
+          ) %>
+          </dt>
+
+          <dd class="attribute-data attribute-data--<%=attribute.html_class%>"
+              ><%= render_field attribute, page: page %></dd>
+        <% end %>
+      </fieldset>
+    <% end %>
+  </dl>
+</section>

enterprise/lib/chat_gpt.rb

@@ -4,7 +4,7 @@ class ChatGpt
   end
 
   def initialize(context_sections = '')
-    @model = 'gpt-4-1106-preview'
+    @model = 'gpt-4-0125-preview'
     @messages = [system_message(context_sections)]
   end
 

tailwind.config.js

@@ -16,6 +16,7 @@ const defaultTheme = require('tailwindcss/defaultTheme');
 module.exports = {
   darkMode: 'class',
   content: [
+    './enterprise/app/views/**/*.html.erb',
     './app/javascript/widget/**/*.vue',
     './app/javascript/v3/**/*.vue',
     './app/javascript/dashboard/**/*.vue',