feat(Ai-assistant): auto detect spoken language (#4505)

Co-authored-by: mickelr <[email protected]>

Author: junhao3268

packages/@webex/internal-plugin-voicea/README.md

@@ -66,10 +66,16 @@ await webex.internal.voicea.toggleManualCaption(false);
 Set Spoken Language
 
 - Host can set the spoken language of the meeting
+- initial language: DEFAULT
+- auto detect spoken language: AUTO
+- host change language by manual and stop auto detect: MANUAL
 - Triggers voicea:spokenLanguageSet
 
 ```js
-webex.internal.voicea.setSpokenLanguage('en');
+webex.internal.voicea.setSpokenLanguage('en', 'DEFAULT');
+webex.internal.voicea.setSpokenLanguage('en', 'AUTO');
+webex.internal.voicea.setSpokenLanguage('en', 'MANUAL');
+
 ```
 
 Set Caption Language

packages/@webex/internal-plugin-voicea/src/constants.ts

@@ -37,6 +37,7 @@ export const TRANSCRIPTION_TYPE = {
   MANUAL_CAPTION_INTERIM_RESULTS: 'manual_caption_interim_results',
   MANUAL_CAPTION_INTERIM_RESULT: 'manual_caption_interim_result',
   MANUAL_CAPTION_FINAL_RESULT: 'manual_caption_final_result',
+  LANGUAGE_DETECTED: 'language_detected',
 };
 
 export const VOICEA = 'voicea';
@@ -58,3 +59,9 @@ export const TOGGLE_MANUAL_CAPTION_STATUS = {
   IDLE: 'idle',
   SENDING: 'sending',
 };
+
+export const LANGUAGE_ASSIGNMENT = {
+  AUTO: 'AUTO',
+  MANUAL: 'MANUAL',
+  DEFAULT: 'DEFAULT',
+};

packages/@webex/internal-plugin-voicea/src/voicea.ts

@@ -10,6 +10,7 @@ import {
   TURN_ON_CAPTION_STATUS,
   TOGGLE_MANUAL_CAPTION_STATUS,
   DEFAULT_SPOKEN_LANGUAGE,
+  LANGUAGE_ASSIGNMENT,
 } from './constants';
 // eslint-disable-next-line no-unused-vars
 import {
@@ -197,6 +198,11 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel {
         });
         break;
 
+      case TRANSCRIPTION_TYPE.LANGUAGE_DETECTED:
+        // @ts-ignore
+        this.setSpokenLanguage(voiceaPayload.language, LANGUAGE_ASSIGNMENT.AUTO);
+        break;
+
       default:
         break;
     }
@@ -268,9 +274,13 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel {
   /**
    * Set Spoken Language for the meeting
    * @param {string} languageCode
+   * @param {"DEFAULT" | "AUTO" | "MANUAL"} languageAssignment
    * @returns {Promise}
    */
-  public setSpokenLanguage = (languageCode: string): Promise<void> =>
+  public setSpokenLanguage = (
+    languageCode: string,
+    languageAssignment = LANGUAGE_ASSIGNMENT.DEFAULT
+  ): Promise<void> =>
     // @ts-ignore
     this.request({
       method: 'PUT',
@@ -279,6 +289,7 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel {
       body: {
         transcribe: {
           spokenLanguage: languageCode,
+          languageAssignment,
         },
       },
     }).then(() => {

packages/@webex/internal-plugin-voicea/src/voicea.types.ts

@@ -70,6 +70,7 @@ interface TranscriptionResponse {
   csis: number[];
   data: string;
   command_response: string;
+  language: string;
 }
 /**
  * Type for CaptionLanguageResponse

packages/@webex/internal-plugin-voicea/test/unit/spec/voicea.js

@@ -291,6 +291,7 @@ describe('plugin-voicea', () => {
     describe('#setSpokenLanguage', () => {
       it('sets spoken language', async () => {
         const languageCode = 'en';
+        let languageAssignment = 'DEFAULT';
         const triggerSpy = sinon.spy();
 
         voiceaService.on(EVENT_TRIGGERS.SPOKEN_LANGUAGE_UPDATE, triggerSpy);
@@ -306,7 +307,8 @@ describe('plugin-voicea', () => {
             url: `${locusUrl}/controls/`,
             body: {
               transcribe: {
-                spokenLanguage: languageCode
+                spokenLanguage: languageCode,
+                languageAssignment,
               }
             },
           })
@@ -880,6 +882,28 @@ describe('plugin-voicea', () => {
           timestamp: '11:00',
         });
       });
+
+      it('processes a language detected', async () => {
+        voiceaService.on(EVENT_TRIGGERS.SPOKEN_LANGUAGE_UPDATE, triggerSpy);
+
+        const voiceaPayload = {
+          id: '9bc51440-1a22-7c81-6add-4b6ff7b59f7c',
+          meeting: 'fd5bd0fc-06fb-4fd1-982b-554c4368f101',
+          type: 'language_detected',
+          language: 'pl'
+        };
+
+        // eslint-disable-next-line no-underscore-dangle
+        await voiceaService.webex.internal.llm._emit('event:relay.event', {
+          headers: {from: 'ws'},
+          data: {relayType: 'voicea.transcription', voiceaPayload},
+        });
+
+        assert.calledOnceWithExactly(functionSpy, voiceaPayload);
+
+        assert.calledOnceWithExactly(triggerSpy, {languageCode: 'pl'});
+      });
+
     });
 
     describe('#processManualTranscription', () => {

packages/@webex/plugin-meetings/src/constants.ts

@@ -966,6 +966,7 @@ export const DISPLAY_HINTS = {
   MANUAL_CAPTION_STOP: 'MANUAL_CAPTION_STOP',
   MANUAL_CAPTION_STATUS_ACTIVE: 'MANUAL_CAPTION_STATUS_ACTIVE',
   DISPLAY_REAL_TIME_TRANSLATION: 'DISPLAY_REAL_TIME_TRANSLATION',
+  SPOKEN_LANGUAGE_AUTO_DETECTION_ENABLED: 'SPOKEN_LANGUAGE_AUTO_DETECTION_ENABLED',
   ENABLE_CAPTION_PANEL: 'ENABLE_CAPTION_PANEL',
   DISPLAY_NON_ENGLISH_ASR: 'DISPLAY_NON_ENGLISH_ASR',
   TRANSCRIPTION_CONTROL_START: 'TRANSCRIPTION_CONTROL_START',

packages/@webex/plugin-meetings/src/meeting/in-meeting-actions.ts

@@ -44,6 +44,7 @@ interface IInMeetingActions {
 
   isManualCaptionActive?: boolean;
   isSaveTranscriptsEnabled?: boolean;
+  isSpokenLanguageAutoDetectionEnabled?: boolean;
   isWebexAssistantActive?: boolean;
   canViewCaptionPanel?: boolean;
   isRealTimeTranslationEnabled?: boolean;
@@ -187,6 +188,8 @@ export default class InMeetingActions implements IInMeetingActions {
 
   isSaveTranscriptsEnabled = null;
 
+  isSpokenLanguageAutoDetectionEnabled = null;
+
   isWebexAssistantActive = null;
 
   canViewCaptionPanel = null;
@@ -363,6 +366,7 @@ export default class InMeetingActions implements IInMeetingActions {
     canStopManualCaption: this.canStopManualCaption,
     isManualCaptionActive: this.isManualCaptionActive,
     isSaveTranscriptsEnabled: this.isSaveTranscriptsEnabled,
+    isSpokenLanguageAutoDetectionEnabled: this.isSpokenLanguageAutoDetectionEnabled,
     isWebexAssistantActive: this.isWebexAssistantActive,
     canViewCaptionPanel: this.canViewCaptionPanel,
     isRealTimeTranslationEnabled: this.isRealTimeTranslationEnabled,

packages/@webex/plugin-meetings/src/meeting/index.ts

@@ -4226,6 +4226,9 @@ export default class Meeting extends StatelessWebexPlugin {
           isLocalRecordingPaused: MeetingUtil.isLocalRecordingPaused(this.userDisplayHints),
           isManualCaptionActive: MeetingUtil.isManualCaptionActive(this.userDisplayHints),
           isSaveTranscriptsEnabled: MeetingUtil.isSaveTranscriptsEnabled(this.userDisplayHints),
+          isSpokenLanguageAutoDetectionEnabled: MeetingUtil.isSpokenLanguageAutoDetectionEnabled(
+            this.userDisplayHints
+          ),
           isWebexAssistantActive: MeetingUtil.isWebexAssistantActive(this.userDisplayHints),
           canViewCaptionPanel: MeetingUtil.canViewCaptionPanel(this.userDisplayHints),
           isRealTimeTranslationEnabled: MeetingUtil.isRealTimeTranslationEnabled(

packages/@webex/plugin-meetings/src/meeting/util.ts

@@ -638,6 +638,9 @@ const MeetingUtil = {
   isManualCaptionActive: (displayHints) =>
     displayHints.includes(DISPLAY_HINTS.MANUAL_CAPTION_STATUS_ACTIVE),
 
+  isSpokenLanguageAutoDetectionEnabled: (displayHints) =>
+    displayHints.includes(DISPLAY_HINTS.SPOKEN_LANGUAGE_AUTO_DETECTION_ENABLED),
+
   isWebexAssistantActive: (displayHints) =>
     displayHints.includes(DISPLAY_HINTS.WEBEX_ASSISTANT_STATUS_ACTIVE),
 

packages/@webex/plugin-meetings/test/unit/spec/meeting/in-meeting-actions.ts

@@ -38,6 +38,7 @@ describe('plugin-meetings', () => {
         isManualCaptionActive: null,
         isPremiseRecordingEnabled: null,
         isSaveTranscriptsEnabled: null,
+        isSpokenLanguageAutoDetectionEnabled: null,
         isWebexAssistantActive: null,
         canViewCaptionPanel: null,
         isRealTimeTranslationEnabled: null,
@@ -151,6 +152,7 @@ describe('plugin-meetings', () => {
       'isManualCaptionActive',
       'isPremiseRecordingEnabled',
       'isSaveTranscriptsEnabled',
+      'isSpokenLanguageAutoDetectionEnabled',
       'isWebexAssistantActive',
       'canViewCaptionPanel',
       'isRealTimeTranslationEnabled',