From 5cd5b24de3f50254fa3b191d1e4f26ccf9e67a6d Mon Sep 17 00:00:00 2001 From: Rushikesh Gavali Date: Thu, 26 Dec 2024 18:33:17 +0530 Subject: [PATCH 01/10] feat: Added continuous listening functionality which is controlled by prop From 11f66fb2bd9ef858535e2bee7a7a58eeee463a25 Mon Sep 17 00:00:00 2001 From: Rushikesh Gavali Date: Fri, 27 Dec 2024 12:25:16 +0530 Subject: [PATCH 02/10] Added the continuous listening feature --- packages/api/src/hooks/Composer.tsx | 6 +++++- packages/api/src/hooks/index.ts | 4 +++- packages/api/src/hooks/useContinuousListening.ts | 5 +++++ packages/component/src/Dictation.js | 11 ++++++++--- .../component/src/SendBox/MicrophoneButton.tsx | 12 +++++++++++- .../core/src/actions/setContinuousListening.ts | 10 ++++++++++ packages/core/src/createReducer.ts | 4 +++- packages/core/src/index.ts | 2 ++ .../core/src/reducers/continuousListening.ts | 16 ++++++++++++++++ .../src/sagas/stopDictateOnCardActionSaga.js | 8 ++++++-- .../core/src/selectors/continuousListening.ts | 3 +++ packages/core/src/types/internal/ReduxState.ts | 1 + 12 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 packages/api/src/hooks/useContinuousListening.ts create mode 100644 packages/core/src/actions/setContinuousListening.ts create mode 100644 packages/core/src/reducers/continuousListening.ts create mode 100644 packages/core/src/selectors/continuousListening.ts diff --git a/packages/api/src/hooks/Composer.tsx b/packages/api/src/hooks/Composer.tsx index 6ae1f9340d..75992f2a38 100644 --- a/packages/api/src/hooks/Composer.tsx +++ b/packages/api/src/hooks/Composer.tsx @@ -12,6 +12,7 @@ import { sendMessage, sendMessageBack, sendPostBack, + setContinuousListening, setDictateInterims, setDictateState, setLanguage, @@ -263,6 +264,7 @@ type ComposerCoreProps = Readonly<{ uiState?: 'blueprint' | 'disabled' | undefined; userID?: string; username?: string; + enableContinuousListening?: boolean; }>; const ComposerCore = ({ @@ -277,6 +279,7 @@ const ComposerCore = ({ directLine, disabled, downscaleImageToDataURL, + enableContinuousListening, grammars, groupActivitiesMiddleware, internalErrorBoxClass, @@ -311,7 +314,8 @@ const ComposerCore = ({ useEffect(() => { dispatch(setLanguage(locale)); - }, [dispatch, locale]); + dispatch(setContinuousListening(enableContinuousListening)); + }, [dispatch, locale, enableContinuousListening]); useEffect(() => { dispatch(setSendTypingIndicator(!!sendTypingIndicator)); diff --git a/packages/api/src/hooks/index.ts b/packages/api/src/hooks/index.ts index 2768ef5b4b..0170c1c58a 100644 --- a/packages/api/src/hooks/index.ts +++ b/packages/api/src/hooks/index.ts @@ -70,6 +70,7 @@ import useUIState from './useUIState'; import useUserID from './useUserID'; import useUsername from './useUsername'; import useVoiceSelector from './useVoiceSelector'; +import useContinuousListening from './useContinuousListening'; export { useActiveTyping, @@ -143,5 +144,6 @@ export { useUIState, useUserID, useUsername, - useVoiceSelector + useVoiceSelector, + useContinuousListening }; diff --git a/packages/api/src/hooks/useContinuousListening.ts b/packages/api/src/hooks/useContinuousListening.ts new file mode 100644 index 0000000000..6574e2fa21 --- /dev/null +++ b/packages/api/src/hooks/useContinuousListening.ts @@ -0,0 +1,5 @@ +import { useSelector } from './internal/WebChatReduxContext'; + +export default function useContinuousListening(): boolean { + return useSelector(({ continuousListening }) => continuousListening); +} diff --git a/packages/component/src/Dictation.js b/packages/component/src/Dictation.js index 3934350a91..c234066c0e 100644 --- a/packages/component/src/Dictation.js +++ b/packages/component/src/Dictation.js @@ -21,7 +21,8 @@ const { useShouldSpeakIncomingActivity, useStopDictate, useSubmitSendBox, - useUIState + useUIState, + useContinuousListening } = hooks; const { @@ -44,6 +45,7 @@ const Dictation = ({ onError }) => { const setDictateState = useSetDictateState(); const stopDictate = useStopDictate(); const submitSendBox = useSubmitSendBox(); + const continuousListening = useContinuousListening(); const numSpeakingActivities = useMemo( () => activities.filter(({ channelData: { speak } = {} }) => speak).length, @@ -54,8 +56,10 @@ const Dictation = ({ onError }) => { ({ result: { confidence, transcript } = {} }) => { if (dictateState === DICTATING || dictateState === STARTING) { setDictateInterims([]); - setDictateState(IDLE); - stopDictate(); + if (!continuousListening) { + setDictateState(IDLE); + stopDictate(); + } if (transcript) { setSendBox(transcript); @@ -65,6 +69,7 @@ const Dictation = ({ onError }) => { } }, [ + continuousListening, dictateState, setDictateInterims, setDictateState, diff --git a/packages/component/src/SendBox/MicrophoneButton.tsx b/packages/component/src/SendBox/MicrophoneButton.tsx index 45a42e4a5c..ab99dcbfb7 100644 --- a/packages/component/src/SendBox/MicrophoneButton.tsx +++ b/packages/component/src/SendBox/MicrophoneButton.tsx @@ -2,6 +2,8 @@ /* eslint react/forbid-dom-props: "off" */ import { hooks } from 'botframework-webchat-api'; +import { useSetDictateState } from 'botframework-webchat-api/internal'; + import { Constants } from 'botframework-webchat-core'; import classNames from 'classnames'; import memoize from 'memoize-one'; @@ -25,7 +27,8 @@ const { useShouldSpeakIncomingActivity, useStartDictate, useStopDictate, - useUIState + useUIState, + useContinuousListening } = hooks; const ROOT_STYLE = { @@ -53,6 +56,8 @@ function useMicrophoneButtonClick(): () => void { const [webSpeechPonyfill] = useWebSpeechPonyfill(); const startDictate = useStartDictate(); const stopDictate = useStopDictate(); + const setDictateState = useSetDictateState(); + const continuousListening = useContinuousListening(); const { speechSynthesis, SpeechSynthesisUtterance } = webSpeechPonyfill || {}; @@ -75,6 +80,9 @@ function useMicrophoneButtonClick(): () => void { } else if (dictateState === DictateState.DICTATING) { stopDictate(); setSendBox(dictateInterims.join(' ')); + if (continuousListening) { + setDictateState(DictateState.IDLE); + } } else { setShouldSpeakIncomingActivity(false); startDictate(); @@ -86,6 +94,8 @@ function useMicrophoneButtonClick(): () => void { dictateState, primeSpeechSynthesis, setSendBox, + setDictateState, + continuousListening, setShouldSpeakIncomingActivity, speechSynthesis, SpeechSynthesisUtterance, diff --git a/packages/core/src/actions/setContinuousListening.ts b/packages/core/src/actions/setContinuousListening.ts new file mode 100644 index 0000000000..c4358092fc --- /dev/null +++ b/packages/core/src/actions/setContinuousListening.ts @@ -0,0 +1,10 @@ +const SET_CONTINUOUS_LISTENING = 'WEB_CHAT/SET_CONTINUOUS_LISTENING'; + +export default function setContinuousListening(continuousListening) { + return { + type: SET_CONTINUOUS_LISTENING, + payload: { continuousListening } + }; +} + +export { SET_CONTINUOUS_LISTENING }; diff --git a/packages/core/src/createReducer.ts b/packages/core/src/createReducer.ts index 3e9c9894b1..5172aa52d7 100644 --- a/packages/core/src/createReducer.ts +++ b/packages/core/src/createReducer.ts @@ -1,6 +1,7 @@ import { combineReducers } from 'redux'; import connectivityStatus from './reducers/connectivityStatus'; +import continuousListening from './reducers/continuousListening'; import createActivitiesReducer from './reducers/createActivitiesReducer'; import createInternalReducer from './reducers/createInternalReducer'; import createNotificationsReducer from './reducers/createNotificationsReducer'; @@ -38,6 +39,7 @@ export default function createReducer(ponyfill: GlobalScopePonyfill) { shouldSpeakIncomingActivity, suggestedActions, suggestedActionsOriginActivity, - typing: createTypingReducer(ponyfill) + typing: createTypingReducer(ponyfill), + continuousListening }); } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2eb76a2da6..4ca835537e 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -12,6 +12,7 @@ import sendFiles from './actions/sendFiles'; import sendMessage from './actions/sendMessage'; import sendMessageBack from './actions/sendMessageBack'; import sendPostBack from './actions/sendPostBack'; +import setContinuousListening from './actions/setContinuousListening' import setDictateInterims from './actions/setDictateInterims'; import setDictateState from './actions/setDictateState'; import setLanguage from './actions/setLanguage'; @@ -107,6 +108,7 @@ export { sendMessage, sendMessageBack, sendPostBack, + setContinuousListening, setDictateInterims, setDictateState, setLanguage, diff --git a/packages/core/src/reducers/continuousListening.ts b/packages/core/src/reducers/continuousListening.ts new file mode 100644 index 0000000000..d6f13c1284 --- /dev/null +++ b/packages/core/src/reducers/continuousListening.ts @@ -0,0 +1,16 @@ +import { SET_CONTINUOUS_LISTENING } from '../actions/setContinuousListening'; + +const DEFAULT_STATE = false; + +export default function continuousListening(state = DEFAULT_STATE, { payload, type }) { + switch (type) { + case SET_CONTINUOUS_LISTENING: + state = payload.continuousListening; + break; + + default: + break; + } + + return state; +} diff --git a/packages/core/src/sagas/stopDictateOnCardActionSaga.js b/packages/core/src/sagas/stopDictateOnCardActionSaga.js index 4981c00f14..8df0e77e55 100644 --- a/packages/core/src/sagas/stopDictateOnCardActionSaga.js +++ b/packages/core/src/sagas/stopDictateOnCardActionSaga.js @@ -1,8 +1,9 @@ -import { put, takeEvery } from 'redux-saga/effects'; +import { put, takeEvery, select } from 'redux-saga/effects'; import { POST_ACTIVITY_PENDING } from '../actions/postActivity'; import stopDictate from '../actions/stopDictate'; import whileConnected from './effects/whileConnected'; +import continuousListeningSelector from '../selectors/continuousListening'; function* stopDictateOnCardAction() { // TODO: [P2] We should stop speech input when the user click on anything on a card, including open URL which doesn't generate postActivity @@ -14,7 +15,10 @@ function* stopDictateOnCardAction() { // In the future, if we have an action for card input, we should use that instead ({ payload, type }) => type === POST_ACTIVITY_PENDING && payload.activity.type === 'message', function* putStopDictate() { - yield put(stopDictate()); + const continuousListening = yield select(continuousListeningSelector); + if (!continuousListening) { + yield put(stopDictate()); + } } ); } diff --git a/packages/core/src/selectors/continuousListening.ts b/packages/core/src/selectors/continuousListening.ts new file mode 100644 index 0000000000..408fc2519a --- /dev/null +++ b/packages/core/src/selectors/continuousListening.ts @@ -0,0 +1,3 @@ +import type { ReduxState } from '../types/internal/ReduxState'; + +export default ({ continuousListening }: ReduxState): boolean => continuousListening; diff --git a/packages/core/src/types/internal/ReduxState.ts b/packages/core/src/types/internal/ReduxState.ts index a3c00c16d5..d6377c8fbd 100644 --- a/packages/core/src/types/internal/ReduxState.ts +++ b/packages/core/src/types/internal/ReduxState.ts @@ -14,6 +14,7 @@ type ReduxState = { sendTimeout: number; sendTypingIndicator: boolean; shouldSpeakIncomingActivity: boolean; + continuousListening: boolean; }; export type { ReduxState }; From bb942402f81d42824fac253afd8794bb3fd33d3d Mon Sep 17 00:00:00 2001 From: Rushikesh Gavali Date: Fri, 27 Dec 2024 21:24:51 +0530 Subject: [PATCH 03/10] Added the fallback value as false for the enableContinuous listening state --- packages/api/src/hooks/Composer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/hooks/Composer.tsx b/packages/api/src/hooks/Composer.tsx index 75992f2a38..14bb55c07f 100644 --- a/packages/api/src/hooks/Composer.tsx +++ b/packages/api/src/hooks/Composer.tsx @@ -314,7 +314,7 @@ const ComposerCore = ({ useEffect(() => { dispatch(setLanguage(locale)); - dispatch(setContinuousListening(enableContinuousListening)); + dispatch(setContinuousListening(enableContinuousListening ?? false)); }, [dispatch, locale, enableContinuousListening]); useEffect(() => { From 206489a0e7543e30b598c835a2e05bf13ca817e8 Mon Sep 17 00:00:00 2001 From: Rushikesh Gavali Date: Mon, 30 Dec 2024 19:45:54 +0530 Subject: [PATCH 04/10] Added the test case for continuous listening feature --- __tests__/html/continuouseListening.html | 102 +++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 __tests__/html/continuouseListening.html diff --git a/__tests__/html/continuouseListening.html b/__tests__/html/continuouseListening.html new file mode 100644 index 0000000000..806af2f7f5 --- /dev/null +++ b/__tests__/html/continuouseListening.html @@ -0,0 +1,102 @@ + + + + + + + + + +
+ + + From 3373e2f8cc3987bd08b61df5202cad79ce7fbce5 Mon Sep 17 00:00:00 2001 From: Rushikesh Gavali Date: Mon, 30 Dec 2024 19:57:17 +0530 Subject: [PATCH 05/10] Updated Changelog.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5541621d68..fc992a8990 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,7 @@ Notes: web developers are advised to use [`~` (tilde range)](https://github.com/ - Copy button is added to fenced code blocks (`
`)
    -  Configure HTML sanitizer via `request.allowedTags`
 -  Added support for math blocks using `$$` delimiter alongside existing `\[...\]` and `\(...\)` notations, in PR [#5381](https://github.com/microsoft/BotFramework-WebChat/pull/5381), by [@OEvgeny](https://github.com/OEvgeny)
+- Introduced continuous listening capability, in PR [#5397](https://github.com/microsoft/BotFramework-WebChat/pull/5397), by [@RushikeshGavali](https://github.com/RushikeshGavali)
 
 ### Changed
 

From 21594d0605bec477e5e565c4d80643ca9ba7d690 Mon Sep 17 00:00:00 2001
From: Rushikesh Gavali 
Date: Fri, 3 Jan 2025 12:47:20 +0530
Subject: [PATCH 06/10] Moved test file to html2

---
 .../continuousListening/continuousListening.html}               | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename __tests__/{html/continuouseListening.html => html2/continuousListening/continuousListening.html} (95%)

diff --git a/__tests__/html/continuouseListening.html b/__tests__/html2/continuousListening/continuousListening.html
similarity index 95%
rename from __tests__/html/continuouseListening.html
rename to __tests__/html2/continuousListening/continuousListening.html
index 806af2f7f5..4de55348b4 100644
--- a/__tests__/html/continuouseListening.html
+++ b/__tests__/html2/continuousListening/continuousListening.html
@@ -95,7 +95,7 @@
         // THEN: The bot should respond to both messages.
         await pageConditions.numActivitiesShown(4);
 
-        await host.snapshot();
+        await host.snapshot('local');
       });
     
   

From 487b94809fa5fb3825d0257d47c9220f59427cd0 Mon Sep 17 00:00:00 2001
From: Rushikesh Gavali 
Date: Mon, 6 Jan 2025 14:17:38 +0530
Subject: [PATCH 07/10] Fixed build failure

---
 __tests__/html2/continuousListening/continuousListening.html | 2 +-
 packages/core/src/index.ts                                   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/__tests__/html2/continuousListening/continuousListening.html b/__tests__/html2/continuousListening/continuousListening.html
index 4de55348b4..6c6c8197e0 100644
--- a/__tests__/html2/continuousListening/continuousListening.html
+++ b/__tests__/html2/continuousListening/continuousListening.html
@@ -1,4 +1,4 @@
-
+
 
   
     
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 4ca835537e..a38b8e10dc 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -12,7 +12,7 @@ import sendFiles from './actions/sendFiles';
 import sendMessage from './actions/sendMessage';
 import sendMessageBack from './actions/sendMessageBack';
 import sendPostBack from './actions/sendPostBack';
-import setContinuousListening from './actions/setContinuousListening'
+import setContinuousListening from './actions/setContinuousListening';
 import setDictateInterims from './actions/setDictateInterims';
 import setDictateState from './actions/setDictateState';
 import setLanguage from './actions/setLanguage';

From 758577b73abdcf4dfc69eabccda4f45d01a0bd39 Mon Sep 17 00:00:00 2001
From: Rushikesh Gavali 
Date: Thu, 9 Jan 2025 00:02:47 +0530
Subject: [PATCH 08/10] Moved the test file to html folder back

---
 .../{html2/continuousListening => html}/continuousListening.html  | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename __tests__/{html2/continuousListening => html}/continuousListening.html (100%)

diff --git a/__tests__/html2/continuousListening/continuousListening.html b/__tests__/html/continuousListening.html
similarity index 100%
rename from __tests__/html2/continuousListening/continuousListening.html
rename to __tests__/html/continuousListening.html

From bfa53e4a630f84500954d568dafabd1795c4448a Mon Sep 17 00:00:00 2001
From: Rushikesh Gavali 
Date: Wed, 22 Jan 2025 15:25:29 +0530
Subject: [PATCH 09/10] Passed the continuous props to DictateComposed and
 updated the condition of the started props to support the continuos listening

---
 packages/component/src/Dictation.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/component/src/Dictation.js b/packages/component/src/Dictation.js
index c234066c0e..955b3c978b 100644
--- a/packages/component/src/Dictation.js
+++ b/packages/component/src/Dictation.js
@@ -112,6 +112,7 @@ const Dictation = ({ onError }) => {
 
   return (
      {
       speechGrammarList={SpeechGrammarList}
       speechRecognition={SpeechRecognition}
       started={
-        uiState !== 'disabled' && (dictateState === STARTING || dictateState === DICTATING) && !numSpeakingActivities
+        uiState !== 'disabled' &&
+        (dictateState === STARTING || dictateState === DICTATING) &&
+        (continuousListening || !numSpeakingActivities)
       }
     />
   );

From 1448766583a7933a0fc86ecd4372e8c1fe312cd3 Mon Sep 17 00:00:00 2001
From: Rushikesh Gavali 
Date: Sat, 1 Feb 2025 00:10:21 +0530
Subject: [PATCH 10/10] Updated the react-dictate-button package version

---
 packages/component/package.json     | 2 +-
 packages/component/src/Dictation.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/component/package.json b/packages/component/package.json
index 1edef023d6..23c4579585 100644
--- a/packages/component/package.json
+++ b/packages/component/package.json
@@ -151,7 +151,7 @@
     "prop-types": "15.8.1",
     "punycode": "2.3.1",
     "react-chain-of-responsibility": "0.2.0-main.3cb47ab",
-    "react-dictate-button": "2.0.1",
+    "react-dictate-button": "3.0.0",
     "react-film": "3.1.1-main.f623bf6",
     "react-redux": "7.2.9",
     "react-say": "2.1.0",
diff --git a/packages/component/src/Dictation.js b/packages/component/src/Dictation.js
index 955b3c978b..afeeadfc2d 100644
--- a/packages/component/src/Dictation.js
+++ b/packages/component/src/Dictation.js
@@ -112,7 +112,7 @@ const Dictation = ({ onError }) => {
 
   return (