diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c635abdff..6050192aca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ Notes: web developers are advised to use [`~` (tilde range)](https://github.com/ - New style option supports two values: `'activity-actions'` and `'activity-status'` (default) - When set to `'activity-actions'`, feedback buttons are displayed in the activity actions toolbar - When set to `'activity-status'`, feedback buttons appear in the activity status area (default behavior) +- Introduced continuous listening capability, in PR [#5397](https://github.com/microsoft/BotFramework-WebChat/pull/5397), by [@RushikeshGavali](https://github.com/RushikeshGavali) ### Changed diff --git a/__tests__/html/continuousListening.html b/__tests__/html/continuousListening.html new file mode 100644 index 0000000000..6c6c8197e0 --- /dev/null +++ b/__tests__/html/continuousListening.html @@ -0,0 +1,102 @@ + + + + + + + + + +
+ + + diff --git a/packages/api/src/hooks/Composer.tsx b/packages/api/src/hooks/Composer.tsx index 6ae1f9340d..14bb55c07f 100644 --- a/packages/api/src/hooks/Composer.tsx +++ b/packages/api/src/hooks/Composer.tsx @@ -12,6 +12,7 @@ import { sendMessage, sendMessageBack, sendPostBack, + setContinuousListening, setDictateInterims, setDictateState, setLanguage, @@ -263,6 +264,7 @@ type ComposerCoreProps = Readonly<{ uiState?: 'blueprint' | 'disabled' | undefined; userID?: string; username?: string; + enableContinuousListening?: boolean; }>; const ComposerCore = ({ @@ -277,6 +279,7 @@ const ComposerCore = ({ directLine, disabled, downscaleImageToDataURL, + enableContinuousListening, grammars, groupActivitiesMiddleware, internalErrorBoxClass, @@ -311,7 +314,8 @@ const ComposerCore = ({ useEffect(() => { dispatch(setLanguage(locale)); - }, [dispatch, locale]); + dispatch(setContinuousListening(enableContinuousListening ?? false)); + }, [dispatch, locale, enableContinuousListening]); useEffect(() => { dispatch(setSendTypingIndicator(!!sendTypingIndicator)); diff --git a/packages/api/src/hooks/index.ts b/packages/api/src/hooks/index.ts index 2768ef5b4b..0170c1c58a 100644 --- a/packages/api/src/hooks/index.ts +++ b/packages/api/src/hooks/index.ts @@ -70,6 +70,7 @@ import useUIState from './useUIState'; import useUserID from './useUserID'; import useUsername from './useUsername'; import useVoiceSelector from './useVoiceSelector'; +import useContinuousListening from './useContinuousListening'; export { useActiveTyping, @@ -143,5 +144,6 @@ export { useUIState, useUserID, useUsername, - useVoiceSelector + useVoiceSelector, + useContinuousListening }; diff --git a/packages/api/src/hooks/useContinuousListening.ts b/packages/api/src/hooks/useContinuousListening.ts new file mode 100644 index 0000000000..6574e2fa21 --- /dev/null +++ b/packages/api/src/hooks/useContinuousListening.ts @@ -0,0 +1,5 @@ +import { useSelector } from './internal/WebChatReduxContext'; + +export default function useContinuousListening(): boolean { + return useSelector(({ continuousListening }) => continuousListening); +} diff --git a/packages/component/src/Dictation.js b/packages/component/src/Dictation.js index 3934350a91..955b3c978b 100644 --- a/packages/component/src/Dictation.js +++ b/packages/component/src/Dictation.js @@ -21,7 +21,8 @@ const { useShouldSpeakIncomingActivity, useStopDictate, useSubmitSendBox, - useUIState + useUIState, + useContinuousListening } = hooks; const { @@ -44,6 +45,7 @@ const Dictation = ({ onError }) => { const setDictateState = useSetDictateState(); const stopDictate = useStopDictate(); const submitSendBox = useSubmitSendBox(); + const continuousListening = useContinuousListening(); const numSpeakingActivities = useMemo( () => activities.filter(({ channelData: { speak } = {} }) => speak).length, @@ -54,8 +56,10 @@ const Dictation = ({ onError }) => { ({ result: { confidence, transcript } = {} }) => { if (dictateState === DICTATING || dictateState === STARTING) { setDictateInterims([]); - setDictateState(IDLE); - stopDictate(); + if (!continuousListening) { + setDictateState(IDLE); + stopDictate(); + } if (transcript) { setSendBox(transcript); @@ -65,6 +69,7 @@ const Dictation = ({ onError }) => { } }, [ + continuousListening, dictateState, setDictateInterims, setDictateState, @@ -107,6 +112,7 @@ const Dictation = ({ onError }) => { return ( { speechGrammarList={SpeechGrammarList} speechRecognition={SpeechRecognition} started={ - uiState !== 'disabled' && (dictateState === STARTING || dictateState === DICTATING) && !numSpeakingActivities + uiState !== 'disabled' && + (dictateState === STARTING || dictateState === DICTATING) && + (continuousListening || !numSpeakingActivities) } /> ); diff --git a/packages/component/src/SendBox/MicrophoneButton.tsx b/packages/component/src/SendBox/MicrophoneButton.tsx index 45a42e4a5c..ab99dcbfb7 100644 --- a/packages/component/src/SendBox/MicrophoneButton.tsx +++ b/packages/component/src/SendBox/MicrophoneButton.tsx @@ -2,6 +2,8 @@ /* eslint react/forbid-dom-props: "off" */ import { hooks } from 'botframework-webchat-api'; +import { useSetDictateState } from 'botframework-webchat-api/internal'; + import { Constants } from 'botframework-webchat-core'; import classNames from 'classnames'; import memoize from 'memoize-one'; @@ -25,7 +27,8 @@ const { useShouldSpeakIncomingActivity, useStartDictate, useStopDictate, - useUIState + useUIState, + useContinuousListening } = hooks; const ROOT_STYLE = { @@ -53,6 +56,8 @@ function useMicrophoneButtonClick(): () => void { const [webSpeechPonyfill] = useWebSpeechPonyfill(); const startDictate = useStartDictate(); const stopDictate = useStopDictate(); + const setDictateState = useSetDictateState(); + const continuousListening = useContinuousListening(); const { speechSynthesis, SpeechSynthesisUtterance } = webSpeechPonyfill || {}; @@ -75,6 +80,9 @@ function useMicrophoneButtonClick(): () => void { } else if (dictateState === DictateState.DICTATING) { stopDictate(); setSendBox(dictateInterims.join(' ')); + if (continuousListening) { + setDictateState(DictateState.IDLE); + } } else { setShouldSpeakIncomingActivity(false); startDictate(); @@ -86,6 +94,8 @@ function useMicrophoneButtonClick(): () => void { dictateState, primeSpeechSynthesis, setSendBox, + setDictateState, + continuousListening, setShouldSpeakIncomingActivity, speechSynthesis, SpeechSynthesisUtterance, diff --git a/packages/core/src/actions/setContinuousListening.ts b/packages/core/src/actions/setContinuousListening.ts new file mode 100644 index 0000000000..c4358092fc --- /dev/null +++ b/packages/core/src/actions/setContinuousListening.ts @@ -0,0 +1,10 @@ +const SET_CONTINUOUS_LISTENING = 'WEB_CHAT/SET_CONTINUOUS_LISTENING'; + +export default function setContinuousListening(continuousListening) { + return { + type: SET_CONTINUOUS_LISTENING, + payload: { continuousListening } + }; +} + +export { SET_CONTINUOUS_LISTENING }; diff --git a/packages/core/src/createReducer.ts b/packages/core/src/createReducer.ts index 3e9c9894b1..5172aa52d7 100644 --- a/packages/core/src/createReducer.ts +++ b/packages/core/src/createReducer.ts @@ -1,6 +1,7 @@ import { combineReducers } from 'redux'; import connectivityStatus from './reducers/connectivityStatus'; +import continuousListening from './reducers/continuousListening'; import createActivitiesReducer from './reducers/createActivitiesReducer'; import createInternalReducer from './reducers/createInternalReducer'; import createNotificationsReducer from './reducers/createNotificationsReducer'; @@ -38,6 +39,7 @@ export default function createReducer(ponyfill: GlobalScopePonyfill) { shouldSpeakIncomingActivity, suggestedActions, suggestedActionsOriginActivity, - typing: createTypingReducer(ponyfill) + typing: createTypingReducer(ponyfill), + continuousListening }); } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2eb76a2da6..a38b8e10dc 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -12,6 +12,7 @@ import sendFiles from './actions/sendFiles'; import sendMessage from './actions/sendMessage'; import sendMessageBack from './actions/sendMessageBack'; import sendPostBack from './actions/sendPostBack'; +import setContinuousListening from './actions/setContinuousListening'; import setDictateInterims from './actions/setDictateInterims'; import setDictateState from './actions/setDictateState'; import setLanguage from './actions/setLanguage'; @@ -107,6 +108,7 @@ export { sendMessage, sendMessageBack, sendPostBack, + setContinuousListening, setDictateInterims, setDictateState, setLanguage, diff --git a/packages/core/src/reducers/continuousListening.ts b/packages/core/src/reducers/continuousListening.ts new file mode 100644 index 0000000000..d6f13c1284 --- /dev/null +++ b/packages/core/src/reducers/continuousListening.ts @@ -0,0 +1,16 @@ +import { SET_CONTINUOUS_LISTENING } from '../actions/setContinuousListening'; + +const DEFAULT_STATE = false; + +export default function continuousListening(state = DEFAULT_STATE, { payload, type }) { + switch (type) { + case SET_CONTINUOUS_LISTENING: + state = payload.continuousListening; + break; + + default: + break; + } + + return state; +} diff --git a/packages/core/src/sagas/stopDictateOnCardActionSaga.js b/packages/core/src/sagas/stopDictateOnCardActionSaga.js index 4981c00f14..8df0e77e55 100644 --- a/packages/core/src/sagas/stopDictateOnCardActionSaga.js +++ b/packages/core/src/sagas/stopDictateOnCardActionSaga.js @@ -1,8 +1,9 @@ -import { put, takeEvery } from 'redux-saga/effects'; +import { put, takeEvery, select } from 'redux-saga/effects'; import { POST_ACTIVITY_PENDING } from '../actions/postActivity'; import stopDictate from '../actions/stopDictate'; import whileConnected from './effects/whileConnected'; +import continuousListeningSelector from '../selectors/continuousListening'; function* stopDictateOnCardAction() { // TODO: [P2] We should stop speech input when the user click on anything on a card, including open URL which doesn't generate postActivity @@ -14,7 +15,10 @@ function* stopDictateOnCardAction() { // In the future, if we have an action for card input, we should use that instead ({ payload, type }) => type === POST_ACTIVITY_PENDING && payload.activity.type === 'message', function* putStopDictate() { - yield put(stopDictate()); + const continuousListening = yield select(continuousListeningSelector); + if (!continuousListening) { + yield put(stopDictate()); + } } ); } diff --git a/packages/core/src/selectors/continuousListening.ts b/packages/core/src/selectors/continuousListening.ts new file mode 100644 index 0000000000..408fc2519a --- /dev/null +++ b/packages/core/src/selectors/continuousListening.ts @@ -0,0 +1,3 @@ +import type { ReduxState } from '../types/internal/ReduxState'; + +export default ({ continuousListening }: ReduxState): boolean => continuousListening; diff --git a/packages/core/src/types/internal/ReduxState.ts b/packages/core/src/types/internal/ReduxState.ts index a3c00c16d5..d6377c8fbd 100644 --- a/packages/core/src/types/internal/ReduxState.ts +++ b/packages/core/src/types/internal/ReduxState.ts @@ -14,6 +14,7 @@ type ReduxState = { sendTimeout: number; sendTypingIndicator: boolean; shouldSpeakIncomingActivity: boolean; + continuousListening: boolean; }; export type { ReduxState };