From d9a60e0f27a23987e39cccc9571a3e13f1fee10c Mon Sep 17 00:00:00 2001 From: Sudeep Ratnaparkhe Date: Thu, 18 Jul 2024 17:34:04 +0530 Subject: [PATCH 1/4] Task #223068 Get Audio data from microphone and send it whisper cpp function --- src/components/Assesment/Assesment.jsx | 35 +++++++++++ src/utils/VoiceAnalyser.js | 87 ++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/src/components/Assesment/Assesment.jsx b/src/components/Assesment/Assesment.jsx index f2d569f1..96a87835 100644 --- a/src/components/Assesment/Assesment.jsx +++ b/src/components/Assesment/Assesment.jsx @@ -122,6 +122,40 @@ export const LanguageModal = ({ lang, setLang, setOpenLangModal }) => { }); }; + // Function to load model in whisper cpp module + const loadModelWhisper = async (modelName) => { + try { + window.whisperModule.FS_unlink("whisper.bin"); + } catch (e) { + // ignore + } + try { + const transaction = await db.transaction(["models"], "readonly"); + const store = transaction.objectStore("models"); + const request = await store.get(modelName); + + request.onsuccess = async () => { + const modelData = request.result; + let storeResponse = await window.whisperModule.FS_createDataFile( + "/", + "whisper.bin", + modelData, + true, + true + ); + setTimeout(console.log(window.whisperModule.init("whisper.bin")), 5000); + }; + + request.onerror = (err) => { + console.error(`Error to get model data: ${err}`); + }; + + console.log(`Stored model in whisper cpp memory`); + } catch (error) { + console.error("Error storing model in IndexedDB:", error); + } + }; + // Function to load model const loadModel = async () => { setLoading(true); @@ -136,6 +170,7 @@ export const LanguageModal = ({ lang, setLang, setOpenLangModal }) => { } else { console.log(`Model ${modelName} is already stored in IndexedDB`); } + await loadModelWhisper(modelName); } catch (error) { console.log(error.message); } finally { diff --git a/src/utils/VoiceAnalyser.js b/src/utils/VoiceAnalyser.js index c5324c2c..054f5669 100644 --- a/src/utils/VoiceAnalyser.js +++ b/src/utils/VoiceAnalyser.js @@ -90,6 +90,16 @@ function VoiceAnalyser(props) { await fetchFile(recordedBlob) ); + const nondenoiseddata = ffmpeg.FS("readFile", "recorded.webm"); + const nondenoisedBlob = new Blob([nondenoiseddata.buffer], { + type: "audio/webm", + }); + + if (callUpdateLearner) { + let nonDenoisedText = await getResponseText(nondenoisedBlob); + console.log("non denoised output -- ", nonDenoisedText); + } + const rnnoiseModelPath = "models/cb.rnnn"; // Ensure this path is correct and accessible await ffmpeg.FS( "writeFile", @@ -104,10 +114,16 @@ function VoiceAnalyser(props) { "arnndn=m=cb.rnnn", "output.wav" ); + const data = ffmpeg.FS("readFile", "output.wav"); const denoisedBlob = new Blob([data.buffer], { type: "audio/wav" }); const newDenoisedUrl = URL.createObjectURL(denoisedBlob); + if (callUpdateLearner) { + let denoisedText = await getResponseText(denoisedBlob); + console.log("denoised output -- ", denoisedText); + } + setRecordedAudio((prevUrl) => { if (prevUrl) { URL.revokeObjectURL(prevUrl); // Clean up the previous URL @@ -121,6 +137,77 @@ function VoiceAnalyser(props) { } }; + const getResponseText = async (audioBlob) => { + let denoised_response_text = ""; + let isWhisperRunning = false; + let audio0 = null; + let context = new AudioContext({ + sampleRate: 16000, + channelCount: 1, + echoCancellation: false, + autoGainControl: true, + noiseSuppression: true, + }); + + window.OfflineAudioContext = + window.OfflineAudioContext || window.webkitOfflineAudioContext; + + window.whisperModule.set_status(""); + + const blobToArrayBuffer = async (blob) => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onloadend = () => resolve(reader.result); + reader.onerror = reject; + reader.readAsArrayBuffer(blob); + }); + }; + + let audioBuf = await blobToArrayBuffer(audioBlob); + let audioBuffer = await context.decodeAudioData(audioBuf); + + var offlineContext = new OfflineAudioContext( + audioBuffer.numberOfChannels, + audioBuffer.length, + audioBuffer.sampleRate + ); + var source = offlineContext.createBufferSource(); + source.buffer = audioBuffer; + source.connect(offlineContext.destination); + source.start(0); + + let renderedBuffer = await offlineContext.startRendering(); + let audio = renderedBuffer.getChannelData(0); + let audioAll = new Float32Array( + audio0 == null ? audio.length : audio0.length + audio.length + ); + + if (audio0 != null) { + audioAll.set(audio0, 0); + } + audioAll.set(audio, audio0 == null ? 0 : audio0.length); + + window.whisperModule.set_audio(1, audioAll); + + let whisperStatus = ""; + + const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + + while (1) { + whisperStatus = window.whisperModule.get_status(); + if (whisperStatus === "running whisper ...") { + isWhisperRunning = true; + } + if (isWhisperRunning && whisperStatus === "waiting for audio ...") { + denoised_response_text = window.whisperModule.get_transcribed(); + break; + } + await delay(100); + } + + return denoised_response_text; + }; + useEffect(() => { const processAudio = async () => { if (loading || !recordedAudio) { From a4fa95ee34c18f4712e4b1250004fa913e652262 Mon Sep 17 00:00:00 2001 From: Sudeep Ratnaparkhe Date: Thu, 18 Jul 2024 18:20:41 +0530 Subject: [PATCH 2/4] Task #223068 error handling added --- src/components/Assesment/Assesment.jsx | 14 +++++++++++--- src/utils/VoiceAnalyser.js | 16 ++++++++++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/components/Assesment/Assesment.jsx b/src/components/Assesment/Assesment.jsx index 96a87835..fdb88990 100644 --- a/src/components/Assesment/Assesment.jsx +++ b/src/components/Assesment/Assesment.jsx @@ -130,9 +130,17 @@ export const LanguageModal = ({ lang, setLang, setOpenLangModal }) => { // ignore } try { - const transaction = await db.transaction(["models"], "readonly"); - const store = transaction.objectStore("models"); - const request = await store.get(modelName); + let transaction; + let store; + let request; + try { + transaction = await db.transaction(["models"], "readonly"); + store = transaction.objectStore("models"); + request = await store.get(modelName); + } catch (error) { + console.error("Error accessing IndexedDB:", error); + return; + } request.onsuccess = async () => { const modelData = request.result; diff --git a/src/utils/VoiceAnalyser.js b/src/utils/VoiceAnalyser.js index 054f5669..cae52b60 100644 --- a/src/utils/VoiceAnalyser.js +++ b/src/utils/VoiceAnalyser.js @@ -96,8 +96,12 @@ function VoiceAnalyser(props) { }); if (callUpdateLearner) { - let nonDenoisedText = await getResponseText(nondenoisedBlob); - console.log("non denoised output -- ", nonDenoisedText); + try { + let nonDenoisedText = await getResponseText(nondenoisedBlob); + console.log("non denoised output -- ", nonDenoisedText); + } catch (error) { + console.error("Error getting non denoised text:", error); + } } const rnnoiseModelPath = "models/cb.rnnn"; // Ensure this path is correct and accessible @@ -120,8 +124,12 @@ function VoiceAnalyser(props) { const newDenoisedUrl = URL.createObjectURL(denoisedBlob); if (callUpdateLearner) { - let denoisedText = await getResponseText(denoisedBlob); - console.log("denoised output -- ", denoisedText); + try { + let denoisedText = await getResponseText(denoisedBlob); + console.log("denoised output -- ", denoisedText); + } catch (error) { + console.error("Error getting denoised text:", error); + } } setRecordedAudio((prevUrl) => { From b8c60ce3992f4d977ef5a19512b56c204e5fc106 Mon Sep 17 00:00:00 2001 From: Sudeep Ratnaparkhe Date: Thu, 18 Jul 2024 18:59:37 +0530 Subject: [PATCH 3/4] Task #223068 added error handling for audio processing --- src/utils/VoiceAnalyser.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/utils/VoiceAnalyser.js b/src/utils/VoiceAnalyser.js index cae52b60..fcddad8a 100644 --- a/src/utils/VoiceAnalyser.js +++ b/src/utils/VoiceAnalyser.js @@ -172,7 +172,14 @@ function VoiceAnalyser(props) { }; let audioBuf = await blobToArrayBuffer(audioBlob); - let audioBuffer = await context.decodeAudioData(audioBuf); + + let audioBuffer; + try { + audioBuffer = await context.decodeAudioData(audioBuf); + } catch (error) { + console.error("Error decoding audio data:", error); + return ""; + } var offlineContext = new OfflineAudioContext( audioBuffer.numberOfChannels, From 0c99dcc505c7b6d4ee8de4419d8432abdc4999bc Mon Sep 17 00:00:00 2001 From: Sudeep Ratnaparkhe Date: Thu, 18 Jul 2024 20:05:06 +0530 Subject: [PATCH 4/4] Task #223068 added rnnoise model in const js and error handling added for ffmpeg FS --- src/utils/VoiceAnalyser.js | 23 +++++++++++++++++++---- src/utils/constants.js | 2 ++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/utils/VoiceAnalyser.js b/src/utils/VoiceAnalyser.js index fcddad8a..f418fd65 100644 --- a/src/utils/VoiceAnalyser.js +++ b/src/utils/VoiceAnalyser.js @@ -27,6 +27,7 @@ import { compareArrays, getLocalData, replaceAll, + rnnoiseModelPath, } from "./constants"; import config from "./urlConstants.json"; import { filterBadWords } from "./Badwords"; @@ -90,7 +91,13 @@ function VoiceAnalyser(props) { await fetchFile(recordedBlob) ); - const nondenoiseddata = ffmpeg.FS("readFile", "recorded.webm"); + let nondenoiseddata; + try { + nondenoiseddata = ffmpeg.FS("readFile", "recorded.webm"); + } catch (error) { + console.error("Error reading recorded file:", error); + return; + } const nondenoisedBlob = new Blob([nondenoiseddata.buffer], { type: "audio/webm", }); @@ -104,7 +111,6 @@ function VoiceAnalyser(props) { } } - const rnnoiseModelPath = "models/cb.rnnn"; // Ensure this path is correct and accessible await ffmpeg.FS( "writeFile", "cb.rnnn", @@ -119,7 +125,13 @@ function VoiceAnalyser(props) { "output.wav" ); - const data = ffmpeg.FS("readFile", "output.wav"); + let data; + try { + data = ffmpeg.FS("readFile", "output.wav"); + } catch (error) { + console.error("Error reading output file:", error); + return; + } const denoisedBlob = new Blob([data.buffer], { type: "audio/wav" }); const newDenoisedUrl = URL.createObjectURL(denoisedBlob); @@ -208,13 +220,16 @@ function VoiceAnalyser(props) { const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); - while (1) { + let checkWhisperStatus = true; + + while (checkWhisperStatus) { whisperStatus = window.whisperModule.get_status(); if (whisperStatus === "running whisper ...") { isWhisperRunning = true; } if (isWhisperRunning && whisperStatus === "waiting for audio ...") { denoised_response_text = window.whisperModule.get_transcribed(); + checkWhisperStatus = false; break; } await delay(100); diff --git a/src/utils/constants.js b/src/utils/constants.js index 354ac634..bffd145c 100644 --- a/src/utils/constants.js +++ b/src/utils/constants.js @@ -3550,3 +3550,5 @@ export const randomizeArray = (arr) => { } return wordsArr; }; + +export const rnnoiseModelPath = "models/cb.rnnn";