Skip to content

Commit 2b9d816

Browse files
carocao-msftgithub-actions[bot]
carocao-msft
and
github-actions[bot]
authored
ACS Captions (#4229)
* captions fix * Change files * Duplicate change files for beta release * acs captions * fix error * fix cc error * linter * ccc * Fixed comment * Change files * Duplicate change files for beta release * acs captions * fix error * fix cc error * linter * ccc * Fixed comment * Change files * Duplicate change files for beta release * carocao/acsCaptions * CaptionsInfo * CaptionsInfo * Build error fix * Update packages/react-composites CallWithChatComposite browser test snapshots * Update packages/react-composites CallWithChatComposite browser test snapshots * Update packages/react-composites CallComposite browser test snapshots * Update packages/react-composites CallComposite browser test snapshots * fix snapshot issue * Update packages/react-composites CallComposite browser test snapshots * Update packages/react-composites CallComposite browser test snapshots --------- Signed-off-by: carocao-msft <[email protected]> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
1 parent 468a66b commit 2b9d816

File tree

77 files changed

+277
-56
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+277
-56
lines changed

Diff for: ‎change-beta/@azure-communication-react-53ae44f2-943a-4a1c-b337-0702c7c979a5.json

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"type": "prerelease",
3+
"area": "feature",
4+
"workstream": "ACS Captions",
5+
"comment": "Implement ACS Captions",
6+
"packageName": "@azure/communication-react",
7+
"email": "[email protected]",
8+
"dependentChangeType": "patch"
9+
}

Diff for: ‎change/@azure-communication-react-53ae44f2-943a-4a1c-b337-0702c7c979a5.json

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"type": "prerelease",
3+
"area": "feature",
4+
"workstream": "ACS Captions",
5+
"comment": "Implement ACS Captions",
6+
"packageName": "@azure/communication-react",
7+
"email": "[email protected]",
8+
"dependentChangeType": "patch"
9+
}

Diff for: ‎common/config/babel/features.js

+6-2
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,9 @@ module.exports = {
8787
// Feature for spotlight
8888
'spotlight',
8989
// Feature for tracking beta start call identifier
90-
'start-call-beta'
90+
'start-call-beta',
91+
// Close captions feature for ACS calls
92+
"acs-close-captions"
9193
],
9294
// A list of in progress beta feature.
9395
// These features are still beta feature but "in progress"
@@ -114,7 +116,9 @@ module.exports = {
114116
// Feature for Rich Text Editor (RTE) support
115117
'rich-text-editor',
116118
// Feature for spotlight
117-
'spotlight'
119+
'spotlight',
120+
// Close captions feature for ACS calls
121+
"acs-close-captions",
118122
],
119123
// A list of stabilized features.
120124
// These features can be listed in the conditional compilation directives without

Diff for: ‎packages/calling-stateful-client/src/CallContext.ts

+16-2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import { RaisedHand } from '@azure/communication-calling';
1515
import { CapabilitiesChangeInfo, ParticipantCapabilities } from '@azure/communication-calling';
1616
/* @conditional-compile-remove(close-captions) */
1717
import { TeamsCaptionsInfo } from '@azure/communication-calling';
18+
/* @conditional-compile-remove(acs-close-captions) */
19+
import { CaptionsInfo as AcsCaptionsInfo } from '@azure/communication-calling';
1820
/* @conditional-compile-remove(unsupported-browser) */
1921
import { EnvironmentInfo } from '@azure/communication-calling';
2022
/* @conditional-compile-remove(rooms) */ /* @conditional-compile-remove(capabilities) */
@@ -52,6 +54,8 @@ import { CallIdHistory } from './CallIdHistory';
5254
/* @conditional-compile-remove(video-background-effects) */
5355
import { LocalVideoStreamVideoEffectsState } from './CallClientState';
5456
/* @conditional-compile-remove(close-captions) */
57+
import { convertFromTeamsSDKToCaptionInfoState } from './Converter';
58+
/* @conditional-compile-remove(acs-close-captions) */
5559
import { convertFromSDKToCaptionInfoState } from './Converter';
5660
/* @conditional-compile-remove(raise-hand) */
5761
import { convertFromSDKToRaisedHandState } from './Converter';
@@ -962,7 +966,7 @@ export class CallContext {
962966
}
963967
}
964968
/* @conditional-compile-remove(close-captions) */
965-
public addCaption(callId: string, caption: TeamsCaptionsInfo): void {
969+
public addTeamsCaption(callId: string, caption: TeamsCaptionsInfo): void {
966970
this.modifyState((draft: CallClientState) => {
967971
const call = draft.calls[this._callIdHistory.latestCallId(callId)];
968972
if (call) {
@@ -972,12 +976,22 @@ export class CallContext {
972976
currentCaptionLanguage === '' ||
973977
currentCaptionLanguage === undefined
974978
) {
975-
this.processNewCaption(call.captionsFeature.captions, convertFromSDKToCaptionInfoState(caption));
979+
this.processNewCaption(call.captionsFeature.captions, convertFromTeamsSDKToCaptionInfoState(caption));
976980
}
977981
}
978982
});
979983
}
980984

985+
/* @conditional-compile-remove(acs-close-captions) */
986+
public addCaption(callId: string, caption: AcsCaptionsInfo): void {
987+
this.modifyState((draft: CallClientState) => {
988+
const call = draft.calls[this._callIdHistory.latestCallId(callId)];
989+
if (call) {
990+
this.processNewCaption(call.captionsFeature.captions, convertFromSDKToCaptionInfoState(caption));
991+
}
992+
});
993+
}
994+
981995
/* @conditional-compile-remove(close-captions) */
982996
public clearCaptions(callId: string): void {
983997
this.modifyState((draft: CallClientState) => {

Diff for: ‎packages/calling-stateful-client/src/CallDeclarativeCommon.ts

+55-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import { CallContext } from './CallContext';
55
import { CallCommon } from './BetaToStableTypes';
66
/* @conditional-compile-remove(close-captions) */ /* @conditional-compile-remove(call-transfer) */
77
import { Features } from '@azure/communication-calling';
8+
/* @conditional-compile-remove(acs-close-captions) */
9+
import { Captions } from '@azure/communication-calling';
810
/* @conditional-compile-remove(close-captions) */
911
import { TeamsCaptions } from '@azure/communication-calling';
1012
/* @conditional-compile-remove(call-transfer) */
@@ -83,7 +85,13 @@ export abstract class ProxyCallCommon implements ProxyHandler<CallCommon> {
8385
/* @conditional-compile-remove(close-captions) */
8486
if (args[0] === Features.Captions) {
8587
const captionsFeature = target.feature(Features.Captions).captions;
86-
const proxyFeature = new ProxyTeamsCaptions(this._context, target);
88+
let proxyFeature;
89+
/* @conditional-compile-remove(acs-close-captions) */
90+
if (captionsFeature.kind === 'Captions') {
91+
proxyFeature = new ProxyCaptions(this._context, target);
92+
return { captions: new Proxy(captionsFeature, proxyFeature) };
93+
}
94+
proxyFeature = new ProxyTeamsCaptions(this._context, target);
8795
return { captions: new Proxy(captionsFeature, proxyFeature) };
8896
}
8997
/* @conditional-compile-remove(call-transfer) */
@@ -162,6 +170,52 @@ class ProxyTeamsCaptions implements ProxyHandler<TeamsCaptions> {
162170
}
163171
}
164172

173+
/* @conditional-compile-remove(acs-close-captions) */
174+
/**
175+
* @private
176+
*/
177+
class ProxyCaptions implements ProxyHandler<Captions> {
178+
private _context: CallContext;
179+
private _call: CallCommon;
180+
181+
constructor(context: CallContext, call: CallCommon) {
182+
this._context = context;
183+
this._call = call;
184+
}
185+
186+
public get<P extends keyof Captions>(target: Captions, prop: P): any {
187+
switch (prop) {
188+
case 'startCaptions':
189+
return this._context.withAsyncErrorTeedToState(async (...args: Parameters<TeamsCaptions['startCaptions']>) => {
190+
this._context.setStartCaptionsInProgress(this._call.id, true);
191+
const ret = await target.startCaptions(...args);
192+
this._context.setSelectedSpokenLanguage(this._call.id, args[0]?.spokenLanguage ?? 'en-us');
193+
return ret;
194+
}, 'Call.feature');
195+
break;
196+
case 'stopCaptions':
197+
return this._context.withAsyncErrorTeedToState(async (...args: Parameters<TeamsCaptions['stopCaptions']>) => {
198+
const ret = await target.stopCaptions(...args);
199+
this._context.setIsCaptionActive(this._call.id, false);
200+
this._context.setStartCaptionsInProgress(this._call.id, false);
201+
this._context.clearCaptions(this._call.id);
202+
return ret;
203+
}, 'Call.feature');
204+
case 'setSpokenLanguage':
205+
return this._context.withAsyncErrorTeedToState(
206+
async (...args: Parameters<TeamsCaptions['setSpokenLanguage']>) => {
207+
const ret = await target.setSpokenLanguage(...args);
208+
this._context.setSelectedSpokenLanguage(this._call.id, args[0]);
209+
return ret;
210+
},
211+
'Call.feature'
212+
);
213+
default:
214+
return Reflect.get(target, prop);
215+
}
216+
}
217+
}
218+
165219
/* @conditional-compile-remove(spotlight) */
166220
/**
167221
* @private

Diff for: ‎packages/calling-stateful-client/src/CallSubscriber.ts

+25-6
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,16 @@
22
// Licensed under the MIT License.
33

44
import { Features, LocalVideoStream, RemoteParticipant } from '@azure/communication-calling';
5+
/* @conditional-compile-remove(acs-close-captions) */
6+
import { Captions } from '@azure/communication-calling';
57
/* @conditional-compile-remove(close-captions) */
68
import { TeamsCaptions } from '@azure/communication-calling';
79
import { toFlatCommunicationIdentifier } from '@internal/acs-ui-common';
810
import { CallCommon } from './BetaToStableTypes';
911
import { CallContext } from './CallContext';
1012
import { CallIdRef } from './CallIdRef';
1113
/* @conditional-compile-remove(close-captions) */
12-
import { CaptionsSubscriber } from './CaptionsSubscriber';
14+
import { TeamsCaptionsSubscriber } from './CaptionsSubscriber';
1315
import {
1416
convertSdkLocalStreamToDeclarativeLocalStream,
1517
convertSdkParticipantToDeclarativeParticipant
@@ -34,6 +36,8 @@ import { CapabilitiesSubscriber } from './CapabilitiesSubscriber';
3436
import { ReactionSubscriber } from './ReactionSubscriber';
3537
/* @conditional-compile-remove(spotlight) */
3638
import { SpotlightSubscriber } from './SpotlightSubscriber';
39+
/* @conditional-compile-remove(acs-close-captions) */
40+
import { CaptionsSubscriber } from './CaptionsSubscriber';
3741

3842
/**
3943
* Keeps track of the listeners assigned to a particular call because when we get an event from SDK, it doesn't tell us
@@ -55,7 +59,9 @@ export class CallSubscriber {
5559
/* @conditional-compile-remove(optimal-video-count) */
5660
private _optimalVideoCountSubscriber: OptimalVideoCountSubscriber;
5761
/* @conditional-compile-remove(close-captions) */
58-
private _captionsSubscriber?: CaptionsSubscriber;
62+
private _TeamsCaptionsSubscriber?: TeamsCaptionsSubscriber;
63+
/* @conditional-compile-remove(acs-close-captions) */
64+
private _CaptionsSubscriber?: CaptionsSubscriber;
5965
/* @conditional-compile-remove(raise-hand) */
6066
private _raiseHandSubscriber?: RaiseHandSubscriber;
6167
/* @conditional-compile-remove(reaction) */
@@ -208,7 +214,9 @@ export class CallSubscriber {
208214
/* @conditional-compile-remove(ppt-live) */
209215
this._pptLiveSubscriber.unsubscribe();
210216
/* @conditional-compile-remove(close-captions) */
211-
this._captionsSubscriber?.unsubscribe();
217+
this._TeamsCaptionsSubscriber?.unsubscribe();
218+
/* @conditional-compile-remove(acs-close-captions) */
219+
this._CaptionsSubscriber?.unsubscribe();
212220
/* @conditional-compile-remove(raise-hand) */
213221
this._raiseHandSubscriber?.unsubscribe();
214222
/* @conditional-compile-remove(capabilities) */
@@ -244,15 +252,26 @@ export class CallSubscriber {
244252
/* @conditional-compile-remove(close-captions) */
245253
private initCaptionSubscriber = (): void => {
246254
// subscribe to captions here so that we don't call captions when call is not initialized
247-
if (this._call.state === 'Connected' && !this._captionsSubscriber) {
255+
if (
256+
this._call.state === 'Connected' &&
257+
!this._TeamsCaptionsSubscriber &&
258+
/* @conditional-compile-remove(acs-close-captions) */ !this._CaptionsSubscriber
259+
) {
248260
if (this._call.feature(Features.Captions).captions.kind === 'TeamsCaptions') {
249-
this._captionsSubscriber = new CaptionsSubscriber(
261+
this._TeamsCaptionsSubscriber = new TeamsCaptionsSubscriber(
250262
this._callIdRef,
251263
this._context,
252264
this._call.feature(Features.Captions).captions as TeamsCaptions
253265
);
254-
this._call.off('stateChanged', this.initCaptionSubscriber);
266+
} else {
267+
/* @conditional-compile-remove(acs-close-captions) */
268+
this._CaptionsSubscriber = new CaptionsSubscriber(
269+
this._callIdRef,
270+
this._context,
271+
this._call.feature(Features.Captions).captions as Captions
272+
);
255273
}
274+
this._call.off('stateChanged', this.initCaptionSubscriber);
256275
}
257276
};
258277

Diff for: ‎packages/calling-stateful-client/src/CaptionsSubscriber.ts

+49-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import {
88
TeamsCaptionsHandler,
99
TeamsCaptionsInfo
1010
} from '@azure/communication-calling';
11+
/* @conditional-compile-remove(acs-close-captions) */
12+
import { Captions, CaptionsHandler, CaptionsInfo } from '@azure/communication-calling';
1113
/* @conditional-compile-remove(close-captions) */
1214
import { CallContext } from './CallContext';
1315
/* @conditional-compile-remove(close-captions) */
@@ -17,7 +19,7 @@ import { CallIdRef } from './CallIdRef';
1719
/**
1820
* @private
1921
*/
20-
export class CaptionsSubscriber {
22+
export class TeamsCaptionsSubscriber {
2123
private _callIdRef: CallIdRef;
2224
private _context: CallContext;
2325
private _captions: TeamsCaptions;
@@ -53,7 +55,7 @@ export class CaptionsSubscriber {
5355
};
5456

5557
private onCaptionsReceived: TeamsCaptionsHandler = (caption: TeamsCaptionsInfo): void => {
56-
this._context.addCaption(this._callIdRef.callId, caption);
58+
this._context.addTeamsCaption(this._callIdRef.callId, caption);
5759
};
5860

5961
private isCaptionsActiveChanged: PropertyChangedEvent = (): void => {
@@ -67,5 +69,50 @@ export class CaptionsSubscriber {
6769
};
6870
}
6971

72+
/* @conditional-compile-remove(acs-close-captions) */
73+
/**
74+
* @private
75+
*/
76+
export class CaptionsSubscriber {
77+
private _callIdRef: CallIdRef;
78+
private _context: CallContext;
79+
private _captions: Captions;
80+
81+
constructor(callIdRef: CallIdRef, context: CallContext, captions: Captions) {
82+
this._callIdRef = callIdRef;
83+
this._context = context;
84+
this._captions = captions;
85+
if (this._captions.isCaptionsFeatureActive) {
86+
this._context.setIsCaptionActive(this._callIdRef.callId, this._captions.isCaptionsFeatureActive);
87+
}
88+
this._context.setAvailableSpokenLanguages(this._callIdRef.callId, this._captions.supportedSpokenLanguages);
89+
this._context.setSelectedSpokenLanguage(this._callIdRef.callId, this._captions.activeSpokenLanguage);
90+
this.subscribe();
91+
}
92+
93+
private subscribe = (): void => {
94+
this._captions.on('CaptionsActiveChanged', this.isCaptionsActiveChanged);
95+
this._captions.on('SpokenLanguageChanged', this.isSpokenLanguageChanged);
96+
this._captions.on('CaptionsReceived', this.onCaptionsReceived);
97+
};
98+
99+
public unsubscribe = (): void => {
100+
this._captions.off('CaptionsActiveChanged', this.isCaptionsActiveChanged);
101+
this._captions.off('SpokenLanguageChanged', this.isSpokenLanguageChanged);
102+
this._captions.off('CaptionsReceived', this.onCaptionsReceived);
103+
};
104+
105+
private onCaptionsReceived: CaptionsHandler = (caption: CaptionsInfo): void => {
106+
this._context.addCaption(this._callIdRef.callId, caption);
107+
};
108+
109+
private isCaptionsActiveChanged: PropertyChangedEvent = (): void => {
110+
this._context.setIsCaptionActive(this._callIdRef.callId, this._captions.isCaptionsFeatureActive);
111+
};
112+
private isSpokenLanguageChanged: PropertyChangedEvent = (): void => {
113+
this._context.setSelectedSpokenLanguage(this._callIdRef.callId, this._captions.activeSpokenLanguage);
114+
};
115+
}
116+
70117
// This is a placeholder to bypass CC of "close-captions", remove when move the feature to stable
71118
export {};

Diff for: ‎packages/calling-stateful-client/src/Converter.ts

+14-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import {
99
} from '@azure/communication-calling';
1010
/* @conditional-compile-remove(close-captions) */
1111
import { TeamsCaptionsInfo } from '@azure/communication-calling';
12+
/* @conditional-compile-remove(acs-close-captions) */
13+
import { CaptionsInfo as AcsCaptionsInfo } from '@azure/communication-calling';
1214
/* @conditional-compile-remove(teams-identity-support) */
1315
import { CallKind } from '@azure/communication-calling';
1416
import { toFlatCommunicationIdentifier } from '@internal/acs-ui-common';
@@ -198,12 +200,23 @@ export function convertFromSDKToDeclarativeVideoStreamRendererView(
198200
/**
199201
* @private
200202
*/
201-
export function convertFromSDKToCaptionInfoState(caption: TeamsCaptionsInfo): CaptionsInfo {
203+
export function convertFromTeamsSDKToCaptionInfoState(caption: TeamsCaptionsInfo): CaptionsInfo {
202204
return {
203205
...caption
204206
};
205207
}
206208

209+
/* @conditional-compile-remove(acs-close-captions) */
210+
/**
211+
* @private
212+
*/
213+
export function convertFromSDKToCaptionInfoState(caption: AcsCaptionsInfo): CaptionsInfo {
214+
return {
215+
captionText: caption.spokenText,
216+
...caption
217+
};
218+
}
219+
207220
/* @conditional-compile-remove(video-background-effects) */
208221
/** @private */
209222
export function convertFromSDKToDeclarativeVideoStreamVideoEffects(

Diff for: ‎packages/react-components/src/components/CaptionsSettingsModal.tsx

+5-2
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,9 @@ export const _CaptionsSettingsModal = (props: _CaptionsSettingsModalProps): JSX.
133133
const captionLanguageCode = selectedCaptionLanguage.key;
134134
if (isCaptionsFeatureActive) {
135135
onSetSpokenLanguage(spokenLanguageCode);
136-
onSetCaptionLanguage(captionLanguageCode);
136+
if (changeCaptionLanguage) {
137+
onSetCaptionLanguage(captionLanguageCode);
138+
}
137139
} else {
138140
await onStartCaptions({ spokenLanguage: spokenLanguageCode });
139141
}
@@ -145,7 +147,8 @@ export const _CaptionsSettingsModal = (props: _CaptionsSettingsModalProps): JSX.
145147
onSetCaptionLanguage,
146148
onStartCaptions,
147149
selectedSpokenLanguage.key,
148-
selectedCaptionLanguage.key
150+
selectedCaptionLanguage.key,
151+
changeCaptionLanguage
149152
]);
150153

151154
const spokenLanguageDropdownOptions: IDropdownOption[] = useMemo(() => {

0 commit comments

Comments
 (0)