Initial expenditure tracking

packages/client/src/app/actions/user.actions.ts

@@ -9,10 +9,12 @@ export const UserActions = createActionGroup({
     'Get Profile Success': props<{profile: UserProfile}>(),
     'Get Profile Failure': props<{error: string}>(),
 
+		'Update Balance': props<{creditBalance: number}>(),
+
     'Set User ID': props<{id: string}>(),
 
     'Get Settings': emptyProps(),
-    'Get Settings Success': props<{ settings: SettingsState }>(), 
+    'Get Settings Success': props<{ settings: SettingsState }>(),
     'Get Settings Failure': props<{error: string}>(),
 
     'Get Rooms': emptyProps(),
@@ -30,7 +32,7 @@ export const UserActions = createActionGroup({
     'Load and Apply Settings': emptyProps(),
     'Load and Apply Settings Success': emptyProps(),
     'Load and Apply Settings Failure': props<{error: string}>(),
-    
+
     'Save Settings State': props<{ settings: SettingsState }>(),
     'Save Settings State Success': props<{ settings: SettingsState }>(),
     'Save Settings State Failure': props<{error: string}>(),

packages/client/src/app/modules/media/components/recognized-live-text/recognized-live-text.component.html

@@ -1,9 +1,9 @@
 <div #container class="flex max-h-full justify-end live-text-render flex-shrink flex-grow-0 text-base-content" [ngClass]="{'flex-col-reverse': textFlowDown(), 'flex-col': !textFlowDown()}">
   <ng-container *ngIf="textFlowDown(); else textFlowUp">
-    <div *ngIf="text() as txt" @slideInDownOnEnter [classList]="classList()">{{txt | proper}}</div>
+    <div *ngIf="text() as txt" @slideInDownOnEnter [classList]="classList()">{{txt}}</div>
   </ng-container>
   <ng-template #textFlowUp>
-    <div *ngIf="text() as txt"  [classList]="classList()">{{txt | proper}}</div>
+    <div *ngIf="text() as txt"  [classList]="classList()">{{txt}}</div>
   </ng-template>
 </div>
 

packages/client/src/app/modules/media/components/recognized-text/recognized-text.component.html

@@ -3,9 +3,9 @@
   <div *ngIf="!hasLiveResults()" @fadeOutOnLeave class="text-center h-auto fixed top-1/2 left-1/2 -translate-y-1/2 -translate-x-1/2">
     <div class="text-sm text-accent flex flex-row text-center" translate>{{hintText}}</div>
   </div>
-  
+
   <div *ngFor="let text of renderedResults()" @fadeInOnEnter @fadeOutOnLeave [classList]="classList()">
-    {{text | proper}}
+    {{text}}
   </div>
   <div class="absolute top-0 left-0 right-0 bottom-0 flex flex-col gap-3 items-center justify-center" *ngIf="isPaused()">
     <div class="badge badge-warning px-6 py-3 text-center h-auto" translate>HINTS.recognitionPaused</div>

packages/client/src/app/modules/media/services/azure-recognition.service.ts

@@ -54,22 +54,21 @@ export class AzureRecognitionService {
 		}
 	}
 
-	public connectToStream(): void {
-		if (!this.recognizer) {
-			throw new Error('Azure Recognition Engine not initialized!')
-		}
-
-		this.isStreaming = true;
-		this.recognizer.startContinuousRecognitionAsync(
-			() => {
-				console.log('recognizer started continuous async', this.isStreaming)
-			},
-			(err) => {
-				console.warn('start continuous error!', err)
-				this.isStreaming = false;
-				this.store.dispatch(RecognitionActions.error({error: err}))
-			}
-		)
+	public connectToStream(language: InterfaceLanguage | RecognitionDialect): void {
+		this.initialize(language).pipe(take(1)).subscribe((result) => {
+			this.store.dispatch(RecognitionActions.setToken(result))
+			this.isStreaming = true;
+			this.recognizer?.startContinuousRecognitionAsync(
+				() => {
+					console.log('recognizer started continuous async', this.isStreaming)
+				},
+				(err) => {
+					console.warn('start continuous error!', err)
+					this.isStreaming = false;
+					this.store.dispatch(RecognitionActions.error({error: err}))
+				}
+			)
+		})
 	}
 
 	public disconnectFromStream(): void {
@@ -82,6 +81,14 @@ export class AzureRecognitionService {
 				this.store.dispatch(RecognitionActions.error({error: err}))
 			}
 		)
+		this.recognizer?.close(() => {
+			console.log('recognizer closed')
+			this.recognizer = undefined;
+			},
+			(err) => {
+				this.store.dispatch(RecognitionActions.error({error: err}))
+			}
+		)
 	}
 
 	public pauseRecognition(): void {
@@ -103,28 +110,39 @@ export class AzureRecognitionService {
 	private _addEventListeners(): void {
 		if (this.recognizer) {
 			let segmentStart: Date | undefined;
+			this.recognizer.sessionStarted = (sender: sdk.Recognizer, event: sdk.SessionEventArgs) => {
+				console.log('session started', event.sessionId);
+				this._startSession(event.sessionId, Date.now()).pipe(take(1)).subscribe();
+			}
+			this.recognizer.sessionStopped = (sender: sdk.Recognizer, event: sdk.SessionEventArgs) => {
+				console.log('session stopped', event.sessionId);
+				this._endSession(event.sessionId, Date.now()).pipe(take(1)).subscribe(({userId, creditBalance}) => {
+					console.log('session ended');
+				});
+			}
 			this.recognizer.recognizing = (sender: sdk.Recognizer, event: sdk.SpeechRecognitionEventArgs) => {
-				console.log('recognizing', event, sender);
 				this.liveOutput.set(event.result.text);
 				if (!segmentStart) {
 					segmentStart = new Date()
 				}
 			}
 			this.recognizer.recognized = (sender: sdk.Recognizer, event: sdk.SpeechRecognitionEventArgs) => {
-				console.log('recognized', event, sender);
+				console.log('recognized', event.result.text);
+				this._updateSession(event.sessionId, Date.now()).pipe(take(1)).subscribe();
 				this.recognizedText.update((current: string[]) => {
 					current.push(event.result.text);
 					return current;
 				});
 				this.liveOutput.set('');
 				if (this.transcriptionEnabled()) {
 					this.store.dispatch(RecognitionActions.addTranscriptSegment({ text: event.result.text, start: segmentStart }))
-					segmentStart = undefined;
 				}
+				segmentStart = undefined;
 			}
 			this.recognizer.canceled = (sender: sdk.Recognizer,  event: sdk.SpeechRecognitionCanceledEventArgs) => {
-				console.log('cancelled', sender, event);
+				console.log('cancelled', event);
 				this.isStreaming = false;
+				this.store.dispatch(RecognitionActions.error({ error: 'Recognition was cancelled for some reason!'}));
 			}
 		}
 	}
@@ -133,6 +151,18 @@ export class AzureRecognitionService {
 		return this.http.get<{token: string; region: string}>(`${this.azureSttEndpoint}/get-token`);
 	}
 
+	private _startSession(sessionId: string, timestamp?: number): Observable<{id: string}> {
+		return this.http.post<{id: string}>(`${this.azureSttEndpoint}/start`, { sessionId, timestamp });
+	}
+
+	private _updateSession(sessionId: string, timestamp?: number): Observable<void> {
+		return this.http.post<void>(`${this.azureSttEndpoint}/track`, { sessionId, timestamp });
+	}
+
+	private _endSession(sessionId: string, timestamp?: number): Observable<{userId: string, creditBalance: number}> {
+		return this.http.post<{userId: string, creditBalance: number}>(`${this.azureSttEndpoint}/end`, { sessionId, timestamp });
+	}
+
 
 
 }

packages/client/src/app/modules/media/services/recognition.service.ts

@@ -61,7 +61,7 @@ export class RecognitionService {
 		if (this.provider() === 'web') {
 			this.webRecognition.connectToStream();
 		} else {
-			this.azureRecognition.connectToStream();
+			this.azureRecognition.connectToStream(this.activeLanguage());
 		}
   }
 

packages/client/src/app/modules/media/services/web-recognition.service.ts

@@ -10,7 +10,7 @@ import { SpeechRecognition } from '../../../models/recognition.model';
 import { ObsConnectionState } from '../../../reducers/obs.reducer';
 import { platformSelector } from '../../../selectors/app.selector';
 import { selectObsConnected } from '../../../selectors/obs.selectors';
-import { languageSelector, selectRenderHistoryLength, selectTranscriptionEnabled } from '../../../selectors/settings.selector';
+import { selectRenderHistoryLength, selectTranscriptionEnabled } from '../../../selectors/settings.selector';
 import { InterfaceLanguage, RecognitionDialect } from '../../settings/models/settings.model';
 
 // TODO: Fix missing definitions once https://github.com/microsoft/TypeScript-DOM-lib-generator/issues/1560 is resolved

packages/client/src/app/reducers/user.reducer.ts

@@ -97,5 +97,6 @@ export const userReducer = createReducer(
   on(UserActions.getSupporterProfileSuccess, (state: UserState, action: { profile: SupporterProfile | null}) => ({...state, supporter: action.profile || undefined })),
   on(UserActions.getSupporterProfileFailure, (state: UserState, action: { error: string }) => ({...state, error: action.error})),
 
+	on(UserActions.updateBalance, (state: UserState, action: { creditBalance: number }) => ({...state, profile: state.profile ? {...state.profile, creditBalance: action.creditBalance } : undefined }))
 );
 

packages/server/src/app/modules/azure-stt/azure-stt.controller.ts

@@ -1,16 +1,19 @@
-import { Controller, Get, HttpException, HttpStatus, Req, UseGuards } from "@nestjs/common";
+import { Body, Controller, Get, HttpException, HttpStatus, Post, Req, UseGuards } from "@nestjs/common";
 import { AzureSttService } from "./services/azure-stt.service";
 import { NoCache } from "../../decorators/no-cache.decorator";
 import { JwtAuthGuard } from "../../guards/jwt-auth.guard";
 import { UserService } from "../user/services/user.service";
 import { SupporterService } from "../../services/supporter/supporter.service";
+import { CacheService } from "../../services/cache/cache.service";
+import { UserController } from "../user/user.controller";
 
 @Controller('azure-stt')
 export class AzureSttController {
 	constructor(
 		private azureSttService: AzureSttService,
 		private userService: UserService,
 		private supporterService: SupporterService,
+		private cache: CacheService,
 	) {}
 
 	@Get('get-token')
@@ -27,4 +30,30 @@ export class AzureSttController {
 		}
 		return this.azureSttService.getToken();
 	}
+
+	@Post('start')
+	@UseGuards(JwtAuthGuard)
+	async startAzureSttSession(@Req() req, @Body() body: { sessionId: string, timestamp: number }): Promise<{id: string }> {
+		const userId = req.user.id;
+		return this.azureSttService.startExpenditure(userId, body.sessionId, body.timestamp);
+	}
+
+	@Post('track')
+	@UseGuards(JwtAuthGuard)
+	async trackAzureSttSession(@Req() req, @Body() body: { sessionId: string, timestamp: number }): Promise<void> {
+		return this.azureSttService.pingExpenditure(req.user.id, body.sessionId, body.timestamp);
+	}
+
+	@Post('end')
+	@UseGuards(JwtAuthGuard)
+	async endAzureSttSession(@Req() req, @Body() body: { sessionId: string, timestamp?: number }): Promise<{userId: string, creditBalance: number}> {
+		const result = await this.azureSttService.completeExpenditure(req.user.id, body.sessionId, body.timestamp);
+		await this.userService.updateUser(result.userId, { creditBalance: result.creditBalance })
+		this._burstCacheForKey(UserController.PROFILE_CACHE_KEY, { id: result.userId });
+		return result;
+	}
+
+	private _burstCacheForKey(key: string, params: { id: string }): void {
+    this.cache.del(`${key}-${params.id}`)
+  }
 }

packages/server/src/app/modules/azure-stt/azure-stt.module.ts

@@ -9,6 +9,8 @@ import { AzureSttController } from './azure-stt.controller';
 import { AzureSttService } from './services/azure-stt.service';
 import { UserService } from '../user/services/user.service';
 import { SupporterService } from '../../services/supporter/supporter.service';
+import { Expenditure, ExpenditureSchema } from '../user/models/expenditure.model';
+import { CacheService } from '../../services/cache/cache.service';
 
 @Module({
 	imports: [
@@ -18,7 +20,8 @@ import { SupporterService } from '../../services/supporter/supporter.service';
 		}),
 		MongooseModule.forFeature([
 			{ name: User.name, schema: UserSchema },
-			{ name: Supporter.name, schema: SupporterSchema }
+			{ name: Supporter.name, schema: SupporterSchema },
+			{ name: Expenditure.name, schema: ExpenditureSchema }
 		])
 	],
 	controllers: [
@@ -29,6 +32,7 @@ import { SupporterService } from '../../services/supporter/supporter.service';
 		AzureSttService,
 		UserService,
 		SupporterService,
+		CacheService
 	],
 })
 export class AzureSttModule {}

packages/server/src/app/modules/azure-stt/services/azure-stt.service.ts

@@ -1,12 +1,20 @@
 import { HttpService } from "@nestjs/axios";
 import { Injectable } from "@nestjs/common";
+import { InjectModel } from "@nestjs/mongoose";
 import { firstValueFrom } from "rxjs";
+import { Expenditure } from "../../user/models/expenditure.model";
+import { Model } from "mongoose";
+import { User } from "../../user/models/user.model";
 
 @Injectable()
 export class AzureSttService {
+	private readonly STT_CREDITS_PER_MINUTE = 60;
 	private speechKey = process.env.AZURE_SPEECH_KEY;
 	private speechRegion = process.env.AZURE_SPEECH_REGION;
-	constructor(private http: HttpService) { }
+	constructor(private http: HttpService,
+							@InjectModel(Expenditure.name) private expenditureModel: Model<Expenditure>,
+							@InjectModel(User.name) private userModel: Model<User>,
+	) { }
 
 	async getToken(): Promise<{token: string; region: string}> {
 		const headers = {
@@ -17,6 +25,54 @@ export class AzureSttService {
 			};
 		const tokenResponse = await firstValueFrom(this.http.post(`https://${this.speechRegion}.api.cognitive.microsoft.com/sts/v1.0/issueToken`, null, headers));
 		return { token: tokenResponse.data, region: this.speechRegion };
+	}
+
+	async startExpenditure(userId: string, sessionId: string, startedAtTimestamp?: number): Promise<{id: string}> {
+		console.log('start expenditure!', sessionId);
+		const user = await this.userModel.findOne({ id: userId });
+		if (!user || !user.creditBalance) {
+			throw new Error("Invalid user")
+		}
+		const createdAt = startedAtTimestamp ? new Date(startedAtTimestamp) : new Date();
+		const session = new this.expenditureModel({ userId, sessionId, createdAt, serviceName: 'Azure-STT' });
+		await session.save();
+		return { id: session.id }
+	}
 
+	async pingExpenditure(userId: string, sessionId: string, pingTimestamp?: number): Promise<void> {
+		console.log('ping expenditure!', sessionId);
+		const updatedAt = pingTimestamp ? new Date(pingTimestamp) : new Date();
+		const spend = await this.expenditureModel.findOne({ userId, sessionId });
+		if (!spend) {
+			throw new Error('Not found')
+		}
+		spend.updatedAt = updatedAt;
+		await spend.save();
 	}
+
+	async completeExpenditure(userId: string, sessionId: string, timestamp?: number): Promise<{userId: string, creditBalance: number}> {
+		console.log('complete expenditure!', sessionId);
+		const spend = await this.expenditureModel.findOne({userId, sessionId});
+		if (!spend) {
+			throw new Error('Not found')
+		}
+		const user = await this.userModel.findOne({ id: spend.userId });
+		if (!user) {
+			throw new Error("User not found")
+		}
+		const end = spend.updatedAt ? Math.max(spend.updatedAt.getTime(), timestamp) : timestamp;
+		console.log('end', end);
+		console.log('start', spend.createdAt.getTime());
+		spend.durationMs = end - spend.createdAt.getTime()
+		console.log('duration', spend.durationMs)
+		const cost = Math.min(1, Math.ceil((spend.durationMs / 1000 / 60) * this.STT_CREDITS_PER_MINUTE ));
+		console.log('cost', cost);
+		spend.creditsUsed = cost
+		await spend.save();
+		const curr = user.creditBalance || 0;
+		user.creditBalance = Math.max(0, curr - cost);
+		await user.save();
+		return { userId: user.id, creditBalance: user.creditBalance }
+	}
+
 }

packages/server/src/app/modules/user/models/credit-add.model.ts

@@ -10,6 +10,13 @@ export class CreditAdd {
 		})
 		id: string;
 
+		@Prop({
+			type: String,
+			required: true,
+			index: true
+		})
+		userId: string;
+
 		@Prop({
 			type: Date,
 			required: true,
@@ -30,4 +37,11 @@ export class CreditAdd {
 
 }
 
-export const CreditAddSchema = SchemaFactory.createForClass(CreditAdd);
+export const CreditAddSchema = SchemaFactory.createForClass(CreditAdd);
+
+CreditAddSchema.pre<CreditAdd>('validate', function(next) {
+	if (!this.createdAt) {
+		this.createdAt = new Date();
+	}
+	return next();
+})