Better error handling on non-web engine

packages/client/src/app/app.module.ts

@@ -111,7 +111,7 @@ export function HttpLoaderFactory(http: HttpClient) {
       UserEffects,
     ]),
     StoreDevtoolsModule.instrument({
-      maxAge: 10,
+      maxAge: 30,
     }),
     HttpClientModule,
     TranslateModule.forRoot({

packages/client/src/app/effects/audio-stream.effects.ts

@@ -3,6 +3,7 @@ import { Actions, createEffect, ofType } from "@ngrx/effects";
 import { catchError, map, of, switchMap } from "rxjs";
 import { AudioStreamActions } from '../models/audio-stream.model';
 import { MediaService } from "../modules/media/services/media.service";
+import { RecognitionActions } from "../actions/recogntion.actions";
 
 @Injectable()
 export class AudioStreamEffects {
@@ -27,4 +28,13 @@ export class AudioStreamEffects {
     })
   ))
 
+	disconnectStreamFailure$ = createEffect(() =>
+		this.actions$.pipe(
+			ofType(RecognitionActions.disconnectFailure),
+			map((props) => {
+				this.mediaService.disconnectAllStreams();
+				return AudioStreamActions.audioStreamError({ error: props.error })
+			})
+		))
+
 }

packages/client/src/app/effects/recognition.effects.ts

@@ -118,17 +118,4 @@ export class RecognitionEffects {
     )
   )
 
-	initAzureRecognition$ = createEffect(() =>
-		this.actions$.pipe(
-			ofType(RecognitionActions.setEngine),
-			filter(({ engine }) => engine === 'azure'),
-			switchMap(() => this.recognitionService.initializeAzure()
-				.pipe(
-					map(({token, region}) => RecognitionActions.setEngineSuccess({token, region})),
-					catchError((err) => of(RecognitionActions.setEngineFailure({error: err})))
-				)
-			)
-		)
-	)
-
 }

packages/client/src/app/modules/media/components/recognized-text/recognized-text.component.html

@@ -1,5 +1,8 @@
 <div class="flex flex-col justify-end basis-full flex-grow-0 flex-shrink-0 bg-base-100 text-base-content p-6 gap-3 overflow-hidden relative">
-  <div *ngIf="error() as err" class="text-error self-stretch text-center text-md sm:text-xl">{{'ERRORS.RECOGNITION.' + err | translate}}</div>
+  <div *ngIf="error() as err" class="text-error self-stretch text-center text-md sm:text-xl">
+		<span *ngIf="provider() === 'web'; else renderPlaintextError">{{'ERRORS.RECOGNITION.' + err | translate}}</span>
+		<ng-template #renderPlaintextError>{{err}}</ng-template>
+	</div>
   <div *ngIf="!hasLiveResults()" @fadeOutOnLeave class="text-center h-auto fixed top-1/2 left-1/2 -translate-y-1/2 -translate-x-1/2">
     <div class="text-sm text-accent flex flex-row text-center" translate>{{hintText}}</div>
   </div>

packages/client/src/app/modules/media/components/recognized-text/recognized-text.component.ts

@@ -4,11 +4,12 @@ import { Store, select } from '@ngrx/store';
 import { fadeInOnEnterAnimation, fadeOutOnLeaveAnimation } from 'angular-animations';
 import { AppState } from '../../../../models/app.model';
 import { selectBroadcastPaused } from '../../../../selectors/peer.selectors';
-import { recognitionPausedSelector } from '../../../../selectors/recognition.selector';
+import { recognitionPausedSelector, selectRecognitionEngineProvider } from '../../../../selectors/recognition.selector';
 import { selectFontFamily, selectLineHeight, selectTextSize } from '../../../../selectors/settings.selector';
 import { FontFamilyClassMap, LineHeight, TextSize } from '../../../settings/models/settings.model';
 import { map } from 'rxjs';
 import { RecognitionActions } from '../../../../actions/recogntion.actions';
+import { RecognitionEngineState } from '../../../../models/recognition.model';
 
 @Component({
   selector: 'app-recognized-text',
@@ -31,6 +32,7 @@ export class RecognizedTextComponent {
   public isPaused: Signal<boolean | undefined>;
   public renderedResults: Signal<string[]>;
   public recognitionPaused: Signal<boolean | undefined>;
+	public provider: Signal<RecognitionEngineState['provider'] | undefined>;
 
   private textSize: Signal<TextSize>;
   private lineHeight: Signal<LineHeight>;
@@ -44,6 +46,7 @@ export class RecognizedTextComponent {
     this.recognitionPaused = toSignal(this.store.select(recognitionPausedSelector));
     const broadcastPaused = toSignal(this.store.select(selectBroadcastPaused));
     this.isPaused = computed(() => this.recognitionPaused() || broadcastPaused());
+		this.provider = toSignal(this.store.select(selectRecognitionEngineProvider))
 
     this.renderedResults = computed(() => {
       const textArray = this.textOutput();

packages/client/src/app/modules/media/services/azure-recognition.service.ts

@@ -8,6 +8,7 @@ import { RecognitionActions } from "../../../actions/recogntion.actions";
 import { AppState } from "../../../models/app.model";
 import { selectTranscriptionEnabled } from "../../../selectors/settings.selector";
 import { InterfaceLanguage, RecognitionDialect } from "../../settings/models/settings.model";
+import { UserActions } from "../../../actions/user.actions";
 
 @Injectable({
 	providedIn: 'root'
@@ -47,27 +48,32 @@ export class AzureRecognitionService {
 	public setLanguage(language: InterfaceLanguage | RecognitionDialect): void {
 		if (this.recognizer) {
 			if (this.recognizer.speechRecognitionLanguage !== language) {
-				this.initialize(language).pipe(take(1)).subscribe((result) => {
-					this.store.dispatch(RecognitionActions.setToken(result))
-				})
+				this.recognizer.close();
+				this.initialize(language).pipe(take(1)).subscribe()
 			}
 		}
 	}
 
 	public connectToStream(language: InterfaceLanguage | RecognitionDialect): void {
-		this.initialize(language).pipe(take(1)).subscribe((result) => {
-			this.store.dispatch(RecognitionActions.setToken(result))
-			this.isStreaming = true;
-			this.recognizer?.startContinuousRecognitionAsync(
-				() => {
-					console.log('recognizer started continuous async', this.isStreaming)
-				},
-				(err) => {
-					console.warn('start continuous error!', err)
-					this.isStreaming = false;
-					this.store.dispatch(RecognitionActions.error({error: err}))
-				}
-			)
+		this.initialize(language).pipe(take(1)).subscribe({
+			next: (result) => {
+				this.isStreaming = true;
+				this.recognizer?.startContinuousRecognitionAsync(
+					() => {
+						console.log('recognizer started continuous async', this.isStreaming)
+					},
+					(err: any) => {
+						console.warn('start continuous error!', err)
+						this.isStreaming = false;
+						this.store.dispatch(RecognitionActions.disconnectFailure({error: err.message }))
+					}
+				)
+			},
+			error: (err: any) => {
+				console.warn('start continuous error!', err)
+				this.isStreaming = false;
+				this.store.dispatch(RecognitionActions.disconnectFailure({error: err.message }))
+			}
 		})
 	}
 
@@ -77,16 +83,16 @@ export class AzureRecognitionService {
 			() => {
 				console.log('recognizer stopped continuous async')
 			},
-			(err) => {
-				this.store.dispatch(RecognitionActions.error({error: err}))
+			(err: any) => {
+				this.store.dispatch(RecognitionActions.disconnectFailure({error: err.message }))
 			}
 		)
 		this.recognizer?.close(() => {
 			console.log('recognizer closed')
 			this.recognizer = undefined;
 			},
 			(err) => {
-				this.store.dispatch(RecognitionActions.error({error: err}))
+				this.store.dispatch(RecognitionActions.disconnectFailure({error: err}))
 			}
 		)
 	}
@@ -111,14 +117,12 @@ export class AzureRecognitionService {
 		if (this.recognizer) {
 			let segmentStart: Date | undefined;
 			this.recognizer.sessionStarted = (sender: sdk.Recognizer, event: sdk.SessionEventArgs) => {
-				console.log('session started', event.sessionId);
-				this._startSession(event.sessionId, Date.now()).pipe(take(1)).subscribe();
+				console.log('session started', event.sessionId, this.isStreaming);
+				this._startSession(event.sessionId, Date.now());
 			}
 			this.recognizer.sessionStopped = (sender: sdk.Recognizer, event: sdk.SessionEventArgs) => {
 				console.log('session stopped', event.sessionId);
-				this._endSession(event.sessionId, Date.now()).pipe(take(1)).subscribe(({userId, creditBalance}) => {
-					console.log('session ended');
-				});
+				this._endSession(event.sessionId, Date.now());
 			}
 			this.recognizer.recognizing = (sender: sdk.Recognizer, event: sdk.SpeechRecognitionEventArgs) => {
 				this.liveOutput.set(event.result.text);
@@ -128,7 +132,7 @@ export class AzureRecognitionService {
 			}
 			this.recognizer.recognized = (sender: sdk.Recognizer, event: sdk.SpeechRecognitionEventArgs) => {
 				console.log('recognized', event.result.text);
-				this._updateSession(event.sessionId, Date.now()).pipe(take(1)).subscribe();
+				this._updateSession(event.sessionId, Date.now());
 				this.recognizedText.update((current: string[]) => {
 					current.push(event.result.text);
 					return current;
@@ -142,7 +146,7 @@ export class AzureRecognitionService {
 			this.recognizer.canceled = (sender: sdk.Recognizer,  event: sdk.SpeechRecognitionCanceledEventArgs) => {
 				console.log('cancelled', event);
 				this.isStreaming = false;
-				this.store.dispatch(RecognitionActions.error({ error: 'Recognition was cancelled for some reason!'}));
+				this.store.dispatch(RecognitionActions.disconnectFailure({ error: event.errorDetails }));
 			}
 		}
 	}
@@ -151,16 +155,35 @@ export class AzureRecognitionService {
 		return this.http.get<{token: string; region: string}>(`${this.azureSttEndpoint}/get-token`);
 	}
 
-	private _startSession(sessionId: string, timestamp?: number): Observable<{id: string}> {
-		return this.http.post<{id: string}>(`${this.azureSttEndpoint}/start`, { sessionId, timestamp });
+	private _startSession(sessionId: string, timestamp?: number): void {
+		this.http.post<{id: string}>(`${this.azureSttEndpoint}/start`, { sessionId, timestamp }).pipe(take(1)).subscribe({
+			error: (err: any) => {
+				this.recognizer?.stopContinuousRecognitionAsync();
+				this.store.dispatch(RecognitionActions.disconnectFailure({ error: err.message }))
+			}
+		});
 	}
 
-	private _updateSession(sessionId: string, timestamp?: number): Observable<void> {
-		return this.http.post<void>(`${this.azureSttEndpoint}/track`, { sessionId, timestamp });
+	private _updateSession(sessionId: string, timestamp?: number): void {
+		this.http.post<void>(`${this.azureSttEndpoint}/track`, { sessionId, timestamp }).pipe(take(1)).subscribe({
+			error: (err: any) => {
+				this.recognizer?.stopContinuousRecognitionAsync();
+				this.store.dispatch(RecognitionActions.disconnectFailure({ error: err.message }))
+			}
+		});
 	}
 
-	private _endSession(sessionId: string, timestamp?: number): Observable<{userId: string, creditBalance: number}> {
-		return this.http.post<{userId: string, creditBalance: number}>(`${this.azureSttEndpoint}/end`, { sessionId, timestamp });
+	private _endSession(sessionId: string, timestamp?: number) {
+		console.log('end session')
+		return this.http.post<{creditBalance: number}>(`${this.azureSttEndpoint}/end`, { sessionId, timestamp }).pipe(take(1)).subscribe({
+			next: ({ creditBalance }) => {
+				this.store.dispatch(UserActions.updateBalance({ creditBalance }));
+			},
+			error: (err: any) => {
+				this.recognizer?.stopContinuousRecognitionAsync();
+				this.store.dispatch(RecognitionActions.disconnectFailure({ error: err.message }))
+			}
+		});
 	}
 
 

packages/client/src/app/modules/media/services/media.service.ts

@@ -26,7 +26,7 @@ export class MediaService {
         this._watchStreamVolume(stream);
         return stream.id;
       }),
-      
+
     )
   }
 
@@ -43,6 +43,19 @@ export class MediaService {
     return streamId;
   }
 
+	public disconnectAllStreams(): string {
+		let rtnId = '';
+		for (const [id, stream] of this.streamsMap) {
+			stream.dispatchEvent(new Event('stop_observation'));
+      stream.getAudioTracks().forEach((track) => {
+        track.stop();
+      });
+			rtnId = id;
+		}
+		this.streamsMap.clear()
+		return rtnId;
+	}
+
   public getVolumeForStream(streamId: string): Signal<number> {
     if (!this.volumeAnalyserMap.has(streamId)) {
       throw new Error(`Stream id does not appear to have a volume analyzer`);

packages/client/src/app/modules/media/services/recognition.service.ts

@@ -107,8 +107,4 @@ export class RecognitionService {
 			return this.azureRecognition.getRecognizedText();
 		}
   }
-
-	initializeAzure(): Observable<{token: string; region: string}> {
-		return this.azureRecognition.initialize(this.activeLanguage());
-	}
 }

packages/client/src/app/modules/media/services/web-recognition.service.ts

@@ -270,7 +270,6 @@ export class WebRecognitionService {
 	}
 
 	private _handleRecognitionError(err: any, fatal = false) {
-    console.warn('recognition error', err);
     this.store.dispatch(RecognitionActions.error({ error: err.error }))
     if (fatal) {
       this.isStreaming = false;

packages/client/src/app/reducers/recognition.reducer.ts

@@ -14,6 +14,7 @@ export const defaultRecognitionState: RecognitionState = {
 export const recognitionReducers = createReducer(defaultRecognitionState,
   on(RecognitionActions.connect, (state: RecognitionState, action: { id: string}) => ({...state, id: action.id, status: RecognitionStatus.connecting})),
   on(RecognitionActions.disconnect, (state: RecognitionState, action: { id: string}) => ({...state, id: action.id, status: RecognitionStatus.disconnected, error: undefined})),
+	on(RecognitionActions.disconnectFailure, (state: RecognitionState, action: { error: string}) => ({...state, status: RecognitionStatus.disconnected, error: action.error})),
   on(RecognitionActions.pauseSuccess, (state: RecognitionState) => ({...state, status: RecognitionStatus.paused})),
   on(RecognitionActions.resumeSuccess, (state: RecognitionState) => ({...state, status: RecognitionStatus.connected, error: undefined })),
   on(RecognitionActions.error, (state: RecognitionState, action:{ error: string}) => ({...state, error: action.error })),

packages/client/src/app/selectors/recognition.selector.ts

@@ -47,4 +47,9 @@ export const selectTranscriptDbInitialized = createSelector(
 export const selectRecognitionEngine = createSelector(
 	selectRecognition,
 	(state) => state.engine
+)
+
+export const selectRecognitionEngineProvider = createSelector(
+	selectRecognitionEngine,
+	(state) => state?.provider
 )

packages/server/src/app/modules/azure-stt/azure-stt.controller.ts

@@ -45,7 +45,7 @@ export class AzureSttController {
 
 	@Post('end')
 	@UseGuards(JwtAuthGuard)
-	async endAzureSttSession(@Req() req, @Body() body: { sessionId: string, timestamp?: number }): Promise<{userId: string, creditBalance: number}> {
+	async endAzureSttSession(@Req() req, @Body() body: { sessionId: string, timestamp?: number }): Promise<{creditBalance: number}> {
 		const result = await this.azureSttService.completeExpenditure(req.user.id, body.sessionId, body.timestamp);
 		await this.userService.updateUser(result.userId, { creditBalance: result.creditBalance })
 		this._burstCacheForKey(UserController.PROFILE_CACHE_KEY, { id: result.userId });

packages/server/src/app/modules/azure-stt/services/azure-stt.service.ts

@@ -17,6 +17,7 @@ export class AzureSttService {
 	) { }
 
 	async getToken(): Promise<{token: string; region: string}> {
+		console.log('speech key', this.speechKey[0])
 		const headers = {
 			headers: {
 					'Ocp-Apim-Subscription-Key': this.speechKey,
@@ -65,7 +66,8 @@ export class AzureSttService {
 		console.log('start', spend.createdAt.getTime());
 		spend.durationMs = end - spend.createdAt.getTime()
 		console.log('duration', spend.durationMs)
-		const cost = Math.min(1, Math.ceil((spend.durationMs / 1000 / 60) * this.STT_CREDITS_PER_MINUTE ));
+
+		const cost = Math.max(1, Math.ceil((spend.durationMs / 1000 / 60) * this.STT_CREDITS_PER_MINUTE ));
 		console.log('cost', cost);
 		spend.creditsUsed = cost
 		await spend.save();