Skip to content

Commit d35dffd

Browse files
lukasIOlukasIO
authored
Add useVoiceAssistant (#917)
1 parent d2b518c commit d35dffd

16 files changed

+181
-45
lines changed

‎.changeset/dull-pots-applaud.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
"@livekit/components-core": patch
3+
"@livekit/components-react": patch
4+
"eslint-config-lk-custom": patch
5+
---
6+
7+
Add useVoiceAssistant

‎packages/core/etc/components-core.api.md

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import type { LocalParticipant } from 'livekit-client';
1515
import { LocalVideoTrack } from 'livekit-client';
1616
import loglevel from 'loglevel';
1717
import { Observable } from 'rxjs';
18-
import type { Participant } from 'livekit-client';
18+
import { Participant } from 'livekit-client';
1919
import { ParticipantEvent } from 'livekit-client';
2020
import type { ParticipantEventCallbacks } from 'livekit-client/dist/src/room/participant/Participant';
2121
import type { ParticipantKind } from 'livekit-client';
@@ -174,7 +174,7 @@ export const defaultUserChoices: LocalUserChoices;
174174
export function didActiveSegmentsChange<T extends TranscriptionSegment>(prevActive: T[], newActive: T[]): boolean;
175175

176176
// @public (undocumented)
177-
export function encryptionStatusObservable(room: Room, participant: Participant): Observable<boolean>;
177+
export function encryptionStatusObservable(room: Room, participant: Participant | undefined): Observable<boolean>;
178178

179179
// @public (undocumented)
180180
export function getActiveTranscriptionSegments(segments: ReceivedTranscriptionSegment[], syncTimes: {
@@ -312,9 +312,12 @@ export function observeTrackEvents(track: TrackPublication, ...events: TrackEven
312312
export function participantAttributesObserver(participant: Participant): Observable<{
313313
changed: Readonly<Record<string, string>>;
314314
attributes: Readonly<Record<string, string>>;
315-
} | {
316-
changed: Readonly<Record<string, string>>;
317-
attributes: Readonly<Record<string, string>>;
315+
}>;
316+
317+
// @public (undocumented)
318+
export function participantAttributesObserver(participant: undefined): Observable<{
319+
changed: undefined;
320+
attributes: undefined;
318321
}>;
319322

320323
// Warning: (ae-incompatible-release-tags) The symbol "participantByIdentifierObserver" is marked as @public, but its signature references "ParticipantIdentifier" which is marked as @beta
@@ -347,15 +350,15 @@ export type ParticipantIdentifier = RequireAtLeastOne<{
347350
}, 'identity' | 'kind'>;
348351

349352
// @public (undocumented)
350-
export function participantInfoObserver(participant: Participant): Observable<{
353+
export function participantInfoObserver(participant?: Participant): Observable<{
351354
name: string | undefined;
352355
identity: string;
353356
metadata: string | undefined;
354357
} | {
355358
name: string | undefined;
356359
identity: string;
357360
metadata: string | undefined;
358-
}>;
361+
}> | undefined;
359362

360363
// @public (undocumented)
361364
export interface ParticipantMedia<T extends Participant = Participant> {
@@ -572,7 +575,7 @@ export function setupParticipantName(participant: Participant): {
572575
name: string | undefined;
573576
identity: string;
574577
metadata: string | undefined;
575-
}>;
578+
}> | undefined;
576579
};
577580

578581
// @public (undocumented)

‎packages/core/src/observables/participant.ts

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import type { ParticipantPermission } from '@livekit/protocol';
2-
import type { Participant, RemoteParticipant, Room, TrackPublication } from 'livekit-client';
2+
import { Participant, RemoteParticipant, Room, TrackPublication } from 'livekit-client';
33
import { ParticipantEvent, RoomEvent, Track } from 'livekit-client';
44
import type { ParticipantEventCallbacks } from 'livekit-client/dist/src/room/participant/Participant';
55
import type { Subscriber } from 'rxjs';
@@ -85,7 +85,10 @@ export function createTrackObserver(participant: Participant, options: TrackIden
8585
);
8686
}
8787

88-
export function participantInfoObserver(participant: Participant) {
88+
export function participantInfoObserver(participant?: Participant) {
89+
if (!participant) {
90+
return undefined;
91+
}
8992
const observer = observeParticipantEvents(
9093
participant,
9194
ParticipantEvent.ParticipantMetadataChanged,
@@ -287,7 +290,18 @@ export function participantByIdentifierObserver(
287290
return observable;
288291
}
289292

290-
export function participantAttributesObserver(participant: Participant) {
293+
export function participantAttributesObserver(participant: Participant): Observable<{
294+
changed: Readonly<Record<string, string>>;
295+
attributes: Readonly<Record<string, string>>;
296+
}>;
297+
export function participantAttributesObserver(participant: undefined): Observable<{
298+
changed: undefined;
299+
attributes: undefined;
300+
}>;
301+
export function participantAttributesObserver(participant: Participant | undefined) {
302+
if (typeof participant === 'undefined') {
303+
return new Observable<{ changed: undefined; attributes: undefined }>();
304+
}
291305
return participantEventSelector(participant, ParticipantEvent.AttributesChanged).pipe(
292306
map(([changedAttributes]) => {
293307
return {

‎packages/core/src/observables/room.ts

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -248,16 +248,18 @@ export function createActiveDeviceObservable(room: Room, kind: MediaDeviceKind)
248248
);
249249
}
250250

251-
export function encryptionStatusObservable(room: Room, participant: Participant) {
251+
export function encryptionStatusObservable(room: Room, participant: Participant | undefined) {
252252
return roomEventSelector(room, RoomEvent.ParticipantEncryptionStatusChanged).pipe(
253253
filter(
254254
([, p]) =>
255-
participant.identity === p?.identity ||
256-
(!p && participant.identity === room.localParticipant.identity),
255+
participant?.identity === p?.identity ||
256+
(!p && participant?.identity === room.localParticipant.identity),
257257
),
258258
map(([encrypted]) => encrypted),
259259
startWith(
260-
participant instanceof LocalParticipant ? participant.isE2EEEnabled : participant.isEncrypted,
260+
participant instanceof LocalParticipant
261+
? participant.isE2EEEnabled
262+
: !!participant?.isEncrypted,
261263
),
262264
);
263265
}

‎packages/react/etc/components-react.api.md

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -573,7 +573,7 @@ export interface RoomAudioRendererProps {
573573
export const RoomContext: React_2.Context<Room | undefined>;
574574

575575
// @public
576-
export const RoomName: (props: RoomNameProps & React_2.RefAttributes<HTMLSpanElement>) => React_2.ReactNode;
576+
export const RoomName: React_2.FC<RoomNameProps & React_2.RefAttributes<HTMLSpanElement>>;
577577

578578
// @public (undocumented)
579579
export interface RoomNameProps extends React_2.HTMLAttributes<HTMLSpanElement> {
@@ -637,6 +637,13 @@ export interface TrackMutedIndicatorProps extends React_2.HTMLAttributes<HTMLDiv
637637
// @public
638638
export const TrackRefContext: React_2.Context<TrackReferenceOrPlaceholder | undefined>;
639639

640+
// Warning: (ae-internal-missing-underscore) The name "TrackRefContextIfNeeded" should be prefixed with an underscore because the declaration is marked as @internal
641+
//
642+
// @internal
643+
export function TrackRefContextIfNeeded(props: React_2.PropsWithChildren<{
644+
trackRef?: TrackReferenceOrPlaceholder;
645+
}>): React_2.JSX.Element;
646+
640647
// @public (undocumented)
641648
export type TrackReference = {
642649
participant: Participant;
@@ -881,7 +888,7 @@ export function useMediaDeviceSelect({ kind, room, track, requestPermissions, on
881888
devices: MediaDeviceInfo[];
882889
className: string;
883890
activeDeviceId: string;
884-
setActiveMediaDevice: (id: string, options?: SetMediaDeviceOptions) => Promise<void>;
891+
setActiveMediaDevice: (id: string, options?: SetMediaDeviceOptions | undefined) => Promise<void>;
885892
};
886893

887894
// @public (undocumented)
@@ -916,7 +923,7 @@ export function useParticipantAttribute(attributeKey: string, options?: UseParti
916923

917924
// @public (undocumented)
918925
export function useParticipantAttributes(props?: UseParticipantAttributesOptions): {
919-
attributes: Readonly<Record<string, string>>;
926+
attributes: Readonly<Record<string, string>> | undefined;
920927
};
921928

922929
// @public
@@ -930,7 +937,7 @@ export function useParticipantContext(): Participant;
930937

931938
// @public (undocumented)
932939
export function useParticipantInfo(props?: UseParticipantInfoOptions): {
933-
identity: string;
940+
identity: string | undefined;
934941
name: string | undefined;
935942
metadata: string | undefined;
936943
};
@@ -1152,7 +1159,7 @@ export type UseTracksOptions = {
11521159

11531160
// @public
11541161
export function useTrackToggle<T extends ToggleSource>({ source, onChange, initialState, captureOptions, publishOptions, onDeviceError, ...rest }: UseTrackToggleProps<T>): {
1155-
toggle: (forceState?: boolean, captureOptions?: CaptureOptionsBySource<T> | undefined) => Promise<void>;
1162+
toggle: (forceState?: boolean | undefined, captureOptions?: CaptureOptionsBySource<T> | undefined) => Promise<void>;
11561163
enabled: boolean;
11571164
pending: boolean;
11581165
track: LocalTrackPublication | undefined;
@@ -1164,7 +1171,7 @@ export interface UseTrackToggleProps<T extends ToggleSource> extends Omit<TrackT
11641171
}
11651172

11661173
// @alpha (undocumented)
1167-
export function useTrackTranscription(trackRef: TrackReferenceOrPlaceholder, options?: TrackTranscriptionOptions): {
1174+
export function useTrackTranscription(trackRef: TrackReferenceOrPlaceholder | undefined, options?: TrackTranscriptionOptions): {
11681175
segments: ReceivedTranscriptionSegment[];
11691176
};
11701177

@@ -1180,6 +1187,9 @@ export interface UseVisualStableUpdateOptions {
11801187
customSortFunction?: (trackReferences: TrackReferenceOrPlaceholder[]) => TrackReferenceOrPlaceholder[];
11811188
}
11821189

1190+
// @alpha
1191+
export function useVoiceAssistant(): VoiceAssistant;
1192+
11831193
// @public
11841194
export function VideoConference({ chatMessageFormatter, chatMessageDecoder, chatMessageEncoder, SettingsComponent, ...props }: VideoConferenceProps): React_2.JSX.Element;
11851195

@@ -1211,6 +1221,26 @@ export interface VideoTrackProps extends React_2.VideoHTMLAttributes<HTMLVideoEl
12111221
trackRef?: TrackReference;
12121222
}
12131223

1224+
// @alpha (undocumented)
1225+
export interface VoiceAssistant {
1226+
// (undocumented)
1227+
agent: RemoteParticipant | undefined;
1228+
// (undocumented)
1229+
agentAttributes: RemoteParticipant['attributes'] | undefined;
1230+
// (undocumented)
1231+
agentTranscriptions: ReceivedTranscriptionSegment[];
1232+
// (undocumented)
1233+
audioTrack: TrackReference | undefined;
1234+
// (undocumented)
1235+
state: VoiceAssistantState;
1236+
}
1237+
1238+
// @alpha (undocumented)
1239+
export const VoiceAssistantContext: React_2.Context<VoiceAssistant | undefined>;
1240+
1241+
// @alpha (undocumented)
1242+
export type VoiceAssistantState = 'disconnected' | 'connecting' | 'listening' | 'thinking' | 'speaking';
1243+
12141244
// @public (undocumented)
12151245
export type WidgetState = {
12161246
showChat: boolean;

‎packages/react/src/components/RoomName.tsx

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,8 @@ export interface RoomNameProps extends React.HTMLAttributes<HTMLSpanElement> {
1919
*
2020
* @param props - RoomNameProps
2121
*/
22-
export const RoomName: (
23-
props: RoomNameProps & React.RefAttributes<HTMLSpanElement>,
24-
) => React.ReactNode = /* @__PURE__ */ React.forwardRef<HTMLSpanElement, RoomNameProps>(
25-
function RoomName(
22+
export const RoomName: React.FC<RoomNameProps & React.RefAttributes<HTMLSpanElement>> =
23+
/* @__PURE__ */ React.forwardRef<HTMLSpanElement, RoomNameProps>(function RoomName(
2624
{ childrenPosition = 'before', children, ...htmlAttributes }: RoomNameProps,
2725
ref,
2826
) {
@@ -35,5 +33,4 @@ export const RoomName: (
3533
{childrenPosition === 'after' && children}
3634
</span>
3735
);
38-
},
39-
);
36+
});

‎packages/react/src/components/participant/ParticipantTile.tsx

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,9 @@ export function ParticipantContextIfNeeded(
5151

5252
/**
5353
* Only create a `TrackRefContext` if there is no `TrackRefContext` already.
54+
* @internal
5455
*/
55-
function TrackRefContextIfNeeded(
56+
export function TrackRefContextIfNeeded(
5657
props: React.PropsWithChildren<{
5758
trackRef?: TrackReferenceOrPlaceholder;
5859
}>,

‎packages/react/src/context/index.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,4 @@ export {
2424
} from './track-reference-context';
2525

2626
export { FeatureFlags, useFeatureContext, LKFeatureContext } from './feature-context';
27+
export { VoiceAssistantContext } from './voice-assistant-context';

‎packages/react/src/context/voice-assistant-context.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import * as React from 'react';
2+
import type { VoiceAssistant } from '../hooks/useVoiceAssistant';
3+
4+
/** @alpha */
5+
export const VoiceAssistantContext = React.createContext<VoiceAssistant | undefined>(undefined);

‎packages/react/src/hooks/index.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,5 +51,6 @@ export { UseIsEncryptedOptions, useIsEncrypted } from './useIsEncrypted';
5151
export * from './useTrackVolume';
5252
export * from './useParticipantTracks';
5353
export * from './useTrackTranscription';
54+
export * from './useVoiceAssistant';
5455
export * from './useParticipantAttributes';
5556
export * from './useIsRecording';

‎packages/react/src/hooks/useIsEncrypted.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,13 @@ export interface UseIsEncryptedOptions {
1717
*/
1818
export function useIsEncrypted(participant?: Participant, options: UseIsEncryptedOptions = {}) {
1919
const p = useEnsureParticipant(participant);
20+
2021
const room = useEnsureRoom(options.room);
2122

2223
const observer = React.useMemo(() => encryptionStatusObservable(room, p), [room, p]);
2324
const isEncrypted = useObservableState(
2425
observer,
25-
p instanceof LocalParticipant ? p.isE2EEEnabled : p.isEncrypted,
26+
p instanceof LocalParticipant ? p.isE2EEEnabled : !!p?.isEncrypted,
2627
);
2728
return isEncrypted;
2829
}

‎packages/react/src/hooks/useParticipantAttributes.ts

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { participantAttributesObserver } from '@livekit/components-core';
22
import type { Participant } from 'livekit-client';
33
import * as React from 'react';
4-
import { useEnsureParticipant } from '../context';
4+
import { useEnsureParticipant, useMaybeParticipantContext } from '../context';
55
import { useObservableState } from './internal';
66

77
/**
@@ -20,10 +20,15 @@ export interface UseParticipantAttributesOptions {
2020

2121
/** @public */
2222
export function useParticipantAttributes(props: UseParticipantAttributesOptions = {}) {
23-
const p = useEnsureParticipant(props.participant);
24-
const attributeObserver = React.useMemo(() => participantAttributesObserver(p), [p]);
23+
const participantContext = useMaybeParticipantContext();
24+
const p = props.participant ?? participantContext;
25+
const attributeObserver = React.useMemo(
26+
// weird typescript constraint
27+
() => (p ? participantAttributesObserver(p) : participantAttributesObserver(p)),
28+
[p],
29+
);
2530
const { attributes } = useObservableState(attributeObserver, {
26-
attributes: p.attributes,
31+
attributes: p?.attributes,
2732
});
2833

2934
return { attributes };
@@ -47,6 +52,9 @@ export function useParticipantAttribute(
4752
const [attribute, setAttribute] = React.useState(p.attributes[attributeKey]);
4853

4954
React.useEffect(() => {
55+
if (!p) {
56+
return;
57+
}
5058
const subscription = participantAttributesObserver(p).subscribe((val) => {
5159
if (val.changed[attributeKey] !== undefined) {
5260
setAttribute(val.changed[attributeKey]);

‎packages/react/src/hooks/useParticipantInfo.ts

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { participantInfoObserver } from '@livekit/components-core';
22
import type { Participant } from 'livekit-client';
33
import * as React from 'react';
4-
import { useEnsureParticipant } from '../context';
4+
import { useMaybeParticipantContext } from '../context';
55
import { useObservableState } from './internal';
66

77
/**
@@ -20,12 +20,15 @@ export interface UseParticipantInfoOptions {
2020

2121
/** @public */
2222
export function useParticipantInfo(props: UseParticipantInfoOptions = {}) {
23-
const p = useEnsureParticipant(props.participant);
23+
let p = useMaybeParticipantContext();
24+
if (props.participant) {
25+
p = props.participant;
26+
}
2427
const infoObserver = React.useMemo(() => participantInfoObserver(p), [p]);
2528
const { identity, name, metadata } = useObservableState(infoObserver, {
26-
name: p.name,
27-
identity: p.identity,
28-
metadata: p.metadata,
29+
name: p?.name,
30+
identity: p?.identity,
31+
metadata: p?.metadata,
2932
});
3033

3134
return { identity, name, metadata };

‎packages/react/src/hooks/useTrackSyncTime.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,13 @@ import { useObservableState } from './internal';
55
/**
66
* @internal
77
*/
8-
export function useTrackSyncTime({ publication }: TrackReferenceOrPlaceholder) {
8+
export function useTrackSyncTime(ref: TrackReferenceOrPlaceholder | undefined) {
99
const observable = React.useMemo(
10-
() => (publication?.track ? trackSyncTimeObserver(publication.track) : undefined),
11-
[publication?.track],
10+
() => (ref?.publication?.track ? trackSyncTimeObserver(ref?.publication.track) : undefined),
11+
[ref?.publication?.track],
1212
);
1313
return useObservableState(observable, {
1414
timestamp: Date.now(),
15-
rtpTimestamp: publication?.track?.rtpTimestamp,
15+
rtpTimestamp: ref?.publication?.track?.rtpTimestamp,
1616
});
1717
}

0 commit comments

Comments
 (0)