feat: IndexedDB based caching for labels, inboxes and teams [CW-50] (#6710)

* feat: allow caching of labels in the account scope

* feat: send cache keys in account json response

* feat: kickstart web worker

* feat: setup basic architecture for workers

* feat: install idb

* feat: add datamanger

* fix: typos

* refactor: rename method

* feat: make init db a manual step

* refactor: separate accountIdFromRoute

* feat: cache enabled API client

* feat: enable caching for inboxes and labels

* feat: enable cache for team

* feat: manage exceptions for team

* feat: add team to data manager

* feat: add a generic listener

* refactor: send only cache keys

* refactor: separate validate method

* feat: add listeners

* feat: add event for revalidate

* feat: add cache keys endpoint

* refactor: fetch cache keys instead of full account data

* fix: key pattern

* feat: don't fetch account for cache_keys

* fix: cache key base class

* refactor: cache keys helper

* feat: add helper

* fix: cache-key update logic

* feat: delete indexeddb on logout

* feat: remove worker.js

* refactor: move data-manager

* refactor: name of file

* feat: add test for DataManager

* refactor: add fake idb to jest setup

* test: cache keys helper

* test: cache keys helper

* test: cache_keys in accounts controller

* refactor: remove cache_keys context

* feat: add policy for cache-keys

app/javascript/dashboard/api/ApiClient.js

@@ -13,6 +13,19 @@ class ApiClient {
     return `${this.baseUrl()}/${this.resource}`;
   }
 
+  // eslint-disable-next-line class-methods-use-this
+  get accountIdFromRoute() {
+    const isInsideAccountScopedURLs = window.location.pathname.includes(
+      '/app/accounts'
+    );
+
+    if (isInsideAccountScopedURLs) {
+      return window.location.pathname.split('/')[3];
+    }
+
+    return '';
+  }
+
   baseUrl() {
     let url = this.apiVersion;
 
@@ -20,15 +33,8 @@ class ApiClient {
       url = `/enterprise${url}`;
     }
 
-    if (this.options.accountScoped) {
-      const isInsideAccountScopedURLs = window.location.pathname.includes(
-        '/app/accounts'
-      );
-
-      if (isInsideAccountScopedURLs) {
-        const accountId = window.location.pathname.split('/')[3];
-        url = `${url}/accounts/${accountId}`;
-      }
+    if (this.options.accountScoped && this.accountIdFromRoute) {
+      url = `${url}/accounts/${this.accountIdFromRoute}`;
     }
 
     return url;

app/javascript/dashboard/api/CacheEnabledApiClient.js

@@ -0,0 +1,82 @@
+/* global axios */
+import { DataManager } from '../helper/CacheHelper/DataManager';
+import ApiClient from './ApiClient';
+
+class CacheEnabledApiClient extends ApiClient {
+  constructor(resource, options = {}) {
+    super(resource, options);
+    this.dataManager = new DataManager(this.accountIdFromRoute);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  get cacheModelName() {
+    throw new Error('cacheModelName is not defined');
+  }
+
+  get(cache = false) {
+    if (cache) {
+      return this.getFromCache();
+    }
+
+    return axios.get(this.url);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  extractDataFromResponse(response) {
+    return response.data.payload;
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  marshallData(dataToParse) {
+    return { data: { payload: dataToParse } };
+  }
+
+  async getFromCache() {
+    await this.dataManager.initDb();
+
+    const { data } = await axios.get(
+      `/api/v1/accounts/${this.accountIdFromRoute}/cache_keys`
+    );
+    const cacheKeyFromApi = data.cache_keys[this.cacheModelName];
+    const isCacheValid = await this.validateCacheKey(cacheKeyFromApi);
+
+    let localData = [];
+    if (isCacheValid) {
+      localData = await this.dataManager.get({
+        modelName: this.cacheModelName,
+      });
+    }
+
+    if (localData.length === 0) {
+      return this.refetchAndCommit(cacheKeyFromApi);
+    }
+
+    return this.marshallData(localData);
+  }
+
+  async refetchAndCommit(newKey = null) {
+    await this.dataManager.initDb();
+    const response = await axios.get(this.url);
+    this.dataManager.replace({
+      modelName: this.cacheModelName,
+      data: this.extractDataFromResponse(response),
+    });
+
+    await this.dataManager.setCacheKeys({
+      [this.cacheModelName]: newKey,
+    });
+
+    return response;
+  }
+
+  async validateCacheKey(cacheKeyFromApi) {
+    if (!this.dataManager.db) {
+      await this.dataManager.initDb();
+    }
+
+    const cachekey = await this.dataManager.getCacheKey(this.cacheModelName);
+    return cacheKeyFromApi === cachekey;
+  }
+}
+
+export default CacheEnabledApiClient;

app/javascript/dashboard/api/auth.js

@@ -2,7 +2,11 @@
 
 import Cookies from 'js-cookie';
 import endPoints from './endPoints';
-import { setAuthCredentials, clearCookiesOnLogout } from '../store/utils/api';
+import {
+  setAuthCredentials,
+  clearCookiesOnLogout,
+  deleteIndexedDBOnLogout,
+} from '../store/utils/api';
 
 export default {
   login(creds) {
@@ -50,6 +54,7 @@ export default {
       axios
         .delete(urlData.url)
         .then(response => {
+          deleteIndexedDBOnLogout();
           clearCookiesOnLogout();
           resolve(response);
         })

app/javascript/dashboard/api/inboxes.js

@@ -1,11 +1,16 @@
 /* global axios */
-import ApiClient from './ApiClient';
+import CacheEnabledApiClient from './CacheEnabledApiClient';
 
-class Inboxes extends ApiClient {
+class Inboxes extends CacheEnabledApiClient {
   constructor() {
     super('inboxes', { accountScoped: true });
   }
 
+  // eslint-disable-next-line class-methods-use-this
+  get cacheModelName() {
+    return 'inbox';
+  }
+
   getCampaigns(inboxId) {
     return axios.get(`${this.url}/${inboxId}/campaigns`);
   }

app/javascript/dashboard/api/labels.js

@@ -1,9 +1,14 @@
-import ApiClient from './ApiClient';
+import CacheEnabledApiClient from './CacheEnabledApiClient';
 
-class LabelsAPI extends ApiClient {
+class LabelsAPI extends CacheEnabledApiClient {
   constructor() {
     super('labels', { accountScoped: true });
   }
+
+  // eslint-disable-next-line class-methods-use-this
+  get cacheModelName() {
+    return 'label';
+  }
 }
 
 export default new LabelsAPI();

app/javascript/dashboard/api/teams.js

@@ -1,11 +1,27 @@
 /* global axios */
-import ApiClient from './ApiClient';
+// import ApiClient from './ApiClient';
+import CacheEnabledApiClient from './CacheEnabledApiClient';
 
-export class TeamsAPI extends ApiClient {
+export class TeamsAPI extends CacheEnabledApiClient {
   constructor() {
     super('teams', { accountScoped: true });
   }
 
+  // eslint-disable-next-line class-methods-use-this
+  get cacheModelName() {
+    return 'team';
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  extractDataFromResponse(response) {
+    return response.data;
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  marshallData(dataToParse) {
+    return { data: dataToParse };
+  }
+
   getAgents({ teamId }) {
     return axios.get(`${this.url}/${teamId}/team_members`);
   }

app/javascript/dashboard/helper/CacheHelper/DataManager.js

@@ -0,0 +1,70 @@
+import { openDB } from 'idb';
+import { DATA_VERSION } from './version';
+
+export class DataManager {
+  constructor(accountId) {
+    this.modelsToSync = ['inbox', 'label', 'team'];
+    this.accountId = accountId;
+    this.db = null;
+  }
+
+  async initDb() {
+    if (this.db) return this.db;
+    this.db = await openDB(`cw-store-${this.accountId}`, DATA_VERSION, {
+      upgrade(db) {
+        db.createObjectStore('cache-keys');
+        db.createObjectStore('inbox', { keyPath: 'id' });
+        db.createObjectStore('label', { keyPath: 'id' });
+        db.createObjectStore('team', { keyPath: 'id' });
+      },
+    });
+
+    return this.db;
+  }
+
+  validateModel(name) {
+    if (!name) throw new Error('Model name is not defined');
+    if (!this.modelsToSync.includes(name)) {
+      throw new Error(`Model ${name} is not defined`);
+    }
+    return true;
+  }
+
+  async replace({ modelName, data }) {
+    this.validateModel(modelName);
+
+    this.db.clear(modelName);
+    return this.push({ modelName, data });
+  }
+
+  async push({ modelName, data }) {
+    this.validateModel(modelName);
+
+    if (Array.isArray(data)) {
+      const tx = this.db.transaction(modelName, 'readwrite');
+      data.forEach(item => {
+        tx.store.add(item);
+      });
+      await tx.done;
+    } else {
+      await this.db.add(modelName, data);
+    }
+  }
+
+  async get({ modelName }) {
+    this.validateModel(modelName);
+    return this.db.getAll(modelName);
+  }
+
+  async setCacheKeys(cacheKeys) {
+    Object.keys(cacheKeys).forEach(async modelName => {
+      this.db.put('cache-keys', cacheKeys[modelName], modelName);
+    });
+  }
+
+  async getCacheKey(modelName) {
+    this.validateModel(modelName);
+
+    return this.db.get('cache-keys', modelName);
+  }
+}

app/javascript/dashboard/helper/CacheHelper/version.js

@@ -0,0 +1,3 @@
+// Monday, 13 March 2023
+// Change this version if you want to invalidate old data
+export const DATA_VERSION = '1678706392';

app/javascript/dashboard/helper/actionCable.js

@@ -26,6 +26,7 @@ class ActionCableConnector extends BaseActionCableConnector {
       'first.reply.created': this.onFirstReplyCreated,
       'conversation.read': this.onConversationRead,
       'conversation.updated': this.onConversationUpdated,
+      'account.cache_invalidated': this.onCacheInvalidate,
     };
   }
 
@@ -156,6 +157,13 @@ class ActionCableConnector extends BaseActionCableConnector {
   onFirstReplyCreated = () => {
     bus.$emit('fetch_overview_reports');
   };
+
+  onCacheInvalidate = data => {
+    const keys = data.cache_keys;
+    this.app.$store.dispatch('labels/revalidate', { newKey: keys.label });
+    this.app.$store.dispatch('inboxes/revalidate', { newKey: keys.inbox });
+    this.app.$store.dispatch('teams/revalidate', { newKey: keys.team });
+  };
 }
 
 export default {

app/javascript/dashboard/helper/specs/CacheHelper/DataManger.spec.js

@@ -0,0 +1,114 @@
+import { DataManager } from '../../CacheHelper/DataManager';
+
+describe('DataManager', () => {
+  const accountId = 'test-account';
+  let dataManager;
+
+  beforeAll(async () => {
+    dataManager = new DataManager(accountId);
+    await dataManager.initDb();
+  });
+
+  afterEach(async () => {
+    const tx = dataManager.db.transaction(
+      dataManager.modelsToSync,
+      'readwrite'
+    );
+    dataManager.modelsToSync.forEach(modelName => {
+      tx.objectStore(modelName).clear();
+    });
+    await tx.done;
+  });
+
+  describe('initDb', () => {
+    it('should initialize the database', async () => {
+      expect(dataManager.db).not.toBeNull();
+    });
+
+    it('should return the same instance of the database', async () => {
+      const db1 = await dataManager.initDb();
+      const db2 = await dataManager.initDb();
+      expect(db1).toBe(db2);
+    });
+  });
+
+  describe('validateModel', () => {
+    it('should throw an error for empty input', async () => {
+      expect(() => {
+        dataManager.validateModel();
+      }).toThrow();
+    });
+
+    it('should throw an error for invalid model', async () => {
+      expect(() => {
+        dataManager.validateModel('invalid-model');
+      }).toThrow();
+    });
+
+    it('should not throw an error for valid model', async () => {
+      expect(dataManager.validateModel('label')).toBeTruthy();
+    });
+  });
+
+  describe('replace', () => {
+    it('should replace existing data in the specified model', async () => {
+      const inboxData = [
+        { id: 1, name: 'inbox-1' },
+        { id: 2, name: 'inbox-2' },
+      ];
+      const newData = [
+        { id: 3, name: 'inbox-3' },
+        { id: 4, name: 'inbox-4' },
+      ];
+
+      await dataManager.push({ modelName: 'inbox', data: inboxData });
+      await dataManager.replace({ modelName: 'inbox', data: newData });
+      const result = await dataManager.get({ modelName: 'inbox' });
+      expect(result).toEqual(newData);
+    });
+  });
+
+  describe('push', () => {
+    it('should add data to the specified model', async () => {
+      const inboxData = { id: 1, name: 'inbox-1' };
+
+      await dataManager.push({ modelName: 'inbox', data: inboxData });
+      const result = await dataManager.get({ modelName: 'inbox' });
+      expect(result).toEqual([inboxData]);
+    });
+
+    it('should add multiple items to the specified model if an array of data is provided', async () => {
+      const inboxData = [
+        { id: 1, name: 'inbox-1' },
+        { id: 2, name: 'inbox-2' },
+      ];
+
+      await dataManager.push({ modelName: 'inbox', data: inboxData });
+      const result = await dataManager.get({ modelName: 'inbox' });
+      expect(result).toEqual(inboxData);
+    });
+  });
+
+  describe('get', () => {
+    it('should return all data in the specified model', async () => {
+      const inboxData = [
+        { id: 1, name: 'inbox-1' },
+        { id: 2, name: 'inbox-2' },
+      ];
+
+      await dataManager.push({ modelName: 'inbox', data: inboxData });
+      const result = await dataManager.get({ modelName: 'inbox' });
+      expect(result).toEqual(inboxData);
+    });
+  });
+
+  describe('setCacheKeys', () => {
+    it('should add cache keys for each model', async () => {
+      const cacheKeys = { inbox: 'cache-key-1', label: 'cache-key-2' };
+
+      await dataManager.setCacheKeys(cacheKeys);
+      const result = await dataManager.getCacheKey('inbox');
+      expect(result).toEqual(cacheKeys.inbox);
+    });
+  });
+});

app/javascript/dashboard/store/modules/inboxes.js

@@ -126,10 +126,21 @@ const sendAnalyticsEvent = channelType => {
 };
 
 export const actions = {
+  revalidate: async ({ commit }, { newKey }) => {
+    try {
+      const isExistingKeyValid = await InboxesAPI.validateCacheKey(newKey);
+      if (!isExistingKeyValid) {
+        const response = await InboxesAPI.refetchAndCommit(newKey);
+        commit(types.default.SET_INBOXES, response.data.payload);
+      }
+    } catch (error) {
+      // Ignore error
+    }
+  },
   get: async ({ commit }) => {
     commit(types.default.SET_INBOXES_UI_FLAG, { isFetching: true });
     try {
-      const response = await InboxesAPI.get();
+      const response = await InboxesAPI.get(true);
       commit(types.default.SET_INBOXES_UI_FLAG, { isFetching: false });
       commit(types.default.SET_INBOXES, response.data.payload);
     } catch (error) {

app/javascript/dashboard/store/modules/labels.js

@@ -29,10 +29,22 @@ export const getters = {
 };
 
 export const actions = {
+  revalidate: async function revalidate({ commit }, { newKey }) {
+    try {
+      const isExistingKeyValid = await LabelsAPI.validateCacheKey(newKey);
+      if (!isExistingKeyValid) {
+        const response = await LabelsAPI.refetchAndCommit(newKey);
+        commit(types.SET_LABELS, response.data.payload);
+      }
+    } catch (error) {
+      // Ignore error
+    }
+  },
+
   get: async function getLabels({ commit }) {
     commit(types.SET_LABEL_UI_FLAG, { isFetching: true });
     try {
-      const response = await LabelsAPI.get();
+      const response = await LabelsAPI.get(true);
       commit(types.SET_LABELS, response.data.payload);
     } catch (error) {
       // Ignore error

app/javascript/dashboard/store/modules/teams/actions.js

@@ -22,10 +22,21 @@ export const actions = {
       commit(SET_TEAM_UI_FLAG, { isCreating: false });
     }
   },
+  revalidate: async ({ commit }, { newKey }) => {
+    try {
+      const isExistingKeyValid = await TeamsAPI.validateCacheKey(newKey);
+      if (!isExistingKeyValid) {
+        const response = await TeamsAPI.refetchAndCommit(newKey);
+        commit(SET_TEAMS, response.data);
+      }
+    } catch (error) {
+      // Ignore error
+    }
+  },
   get: async ({ commit }) => {
     commit(SET_TEAM_UI_FLAG, { isFetching: true });
     try {
-      const { data } = await TeamsAPI.get();
+      const { data } = await TeamsAPI.get(true);
       commit(CLEAR_TEAMS);
       commit(SET_TEAMS, data);
     } catch (error) {

app/javascript/dashboard/store/utils/api.js

@@ -42,6 +42,13 @@ export const clearLocalStorageOnLogout = () => {
   LocalStorage.remove(LOCAL_STORAGE_KEYS.DRAFT_MESSAGES);
 };
 
+export const deleteIndexedDBOnLogout = async () => {
+  const dbs = await window.indexedDB.databases();
+  dbs.forEach(db => {
+    window.indexedDB.deleteDatabase(db.name);
+  });
+};
+
 export const clearCookiesOnLogout = () => {
   window.bus.$emit(CHATWOOT_RESET);
   window.bus.$emit(ANALYTICS_RESET);