feat: DEV-4034: Rendering performance improvements for large-duration audio (#1138)

* wip: chunk decoding in webworker with ffmpeg

* wip: process chunks within the generator to allow quick fail and cleanup

* allow earlier render of timeline and updating of timecontrols for total duration

* refactor AudioDecoder from WaveformAudio

* fix chunk decoding to properly await the promise

* retain decoded audio for a single source so that quick switching between annotations is fluid

* colocate wasm built files with the js files importing them

* show chunk decoding progress information in the ui

* wip: render chunks

* wip: chunk render

* wip: render channel height correctly

* wip: interruptible render

* fix render of all buffer chunks

* do not need buffer allocator anymore, this was the problem

* fix mute/set volume of player

* fix duration reporting to be based on htmlmedia duration

* wip: quickview loading seems to double load some of the time

* fix the loading of decoded audio

* wip: working perf improvement on partial render

* render partial cache when samples to render becomes too high

* ensure amplitude scale change rerenders cache

* extract constant for default sample rate

* removing unused code

* update comments

* cleanup the async handling, wip audio decoder pool

* pool audio decoders so data can be reused efficiently while still being aggressive of memory eviction

* updates for review feedback

* bubble up an error message to the user if something goes wrong in audio load/decode

* fix resizing of waveform and timeline, make render cancellation possible for improved performance

Author: bmartel

package.json

@@ -44,7 +44,8 @@
     "start": "yarn run copy-examples && node dev-server.js",
     "test:coverage": "yarn test -- --coverage",
     "test:watch": "react-scripts test",
-    "test": "yarn jest src"
+    "test": "yarn jest src",
+    "postinstall": "sh scripts/postinstall.sh"
   },
   "husky": {
     "hooks": {
@@ -85,6 +86,7 @@
   ],
   "dependencies": {
     "@thi.ng/rle-pack": "^2.1.6",
+    "audio-file-decoder": "^2.3.0",
     "babel-preset-react-app": "^9.1.1",
     "d3": "^5.16.0",
     "magic-wand-js": "^1.0.0",
@@ -114,8 +116,8 @@
     "@types/jest": "^29.2.3",
     "@types/keymaster": "^1.6.30",
     "@types/lodash.ismatch": "^4.4.6",
-    "@types/offscreencanvas": "^2019.6.4",
     "@types/nanoid": "^3.0.0",
+    "@types/offscreencanvas": "^2019.6.4",
     "@types/react-dom": "^17.0.11",
     "@types/react-window": "^1.8.5",
     "@types/strman": "^2.0.0",
@@ -140,6 +142,7 @@
     "enzyme-to-json": "^3.5.0",
     "eslint": "^8.28.0",
     "eslint-webpack-plugin": "^3.0.1",
+    "file-loader": "^6.2.0",
     "html-webpack-plugin": "^5.3.1",
     "husky": "^3.1.0",
     "identity-obj-proxy": "^3.0.0",

scripts/postinstall.sh

@@ -0,0 +1 @@
+cp ./node_modules/audio-file-decoder/decode-audio.wasm ./node_modules/audio-file-decoder/dist/decode-audio.wasm

src/components/Timeline/Controls/AudioControl.tsx

@@ -6,7 +6,7 @@ import { IconSoundConfig, IconSoundMutedConfig } from '../../../assets/icons/tim
 import { ControlButton } from '../Controls';
 import { Slider } from './Slider';
 
-const MAX_VOL = 200;
+const MAX_VOL = 100;
 
 export interface AudioControlProps {
   volume: number;
@@ -38,15 +38,15 @@ export const AudioControl: FC<AudioControlProps> = ({
       onVolumeChange?.(0);
       return;
     }
-    if (_volumeValue > (MAX_VOL)) {
+    if (_volumeValue > MAX_VOL) {
       onVolumeChange?.(MAX_VOL / 100);
       return;
     } else if (_volumeValue < 0) {
       onVolumeChange?.(0);
       return;
     }
 
-    onVolumeChange?.(_volumeValue / MAX_VOL * 2);
+    onVolumeChange?.(_volumeValue / MAX_VOL);
   };
 
   const handleSetMute = () => {
@@ -60,7 +60,7 @@ export const AudioControl: FC<AudioControlProps> = ({
         <Slider
           min={0}
           max={MAX_VOL}
-          value={Math.round(volume * MAX_VOL / 2)}
+          value={Math.round(volume * MAX_VOL)}
           onChange={handleSetVolume}
           description={'Volume'}
           info={'Increase or decrease the volume of the audio'}

src/lib/AudioUltra/Controls/Player.ts

@@ -27,9 +27,9 @@ export class Player extends Destructable {
    * Get current playback speed
    */
   get rate() {
-    if (this.audio?.source?.playbackRate.value) {
-      if (this.audio.source.playbackRate.value !== this._rate) {
-        this.audio.source.playbackRate.value = this._rate; // restore the correct rate
+    if (this.audio) {
+      if (this.audio.speed !== this._rate) {
+        this.audio.speed = this._rate; // restore the correct rate
       }
     }
 
@@ -44,8 +44,8 @@ export class Player extends Destructable {
 
     this._rate = value;
 
-    if (this.audio?.source) {
-      this.audio.source.playbackRate.value = value;
+    if (this.audio) {
+      this.audio.speed = value;
 
       if (rateChanged) {
         this.wf.invoke('rateChanged', [value]);
@@ -54,22 +54,28 @@ export class Player extends Destructable {
   }
 
   get duration() {
-    return this.audio?.buffer?.duration ?? 0;
+    return this.audio?.duration ?? 0;
   }
 
   get volume() {
-    return this.audio?.gain?.gain.value ?? 1;
+    return this.audio?.volume ?? 1;
   }
 
   set volume(value: number) {
     if (this.audio) {
 
       const volumeChanged = this.volume !== value;
 
-      this.audio.volume = value;
-
       if (volumeChanged) {
-        this.wf.invoke('volumeChange', [value]);
+        if (value === 0) {
+          this.muted = true;
+        } else if(this.muted) {
+          this.muted = false;
+        } else {
+          this.audio.volume = value;
+        }
+
+        this.wf.invoke('volumeChanged', [this.volume]);
       }
     }
   }
@@ -91,13 +97,12 @@ export class Player extends Destructable {
   }
 
   get muted() {
-    return this.audio?.volume === 0;
+    return this.audio?.muted ?? false;
   }
 
   set muted(muted: boolean) {
     if (!this.audio) return;
-
-    if (this.audio.muted === muted) return;
+    if (this.muted === muted) return;
 
     if (muted) {
       this.audio.mute();
@@ -134,11 +139,14 @@ export class Player extends Destructable {
 
   handleEnded = () => {
     if (this.loop) return;
-    this.ended = true;
     this.updateCurrentTime(true);
+  };
+
+  private playEnded() {
+    this.ended = true;
     this.pause();
     this.wf.invoke('playend');
-  };
+  }
 
   pause() {
     if (this.isDestroyed || !this.playing || !this.audio) return;
@@ -183,7 +191,7 @@ export class Player extends Destructable {
     this.timestamp = performance.now();
     this.recreateSource();
 
-    if (!this.audio?.source) return;
+    if (!this.audio) return;
 
     this.playing = true;
 
@@ -196,8 +204,12 @@ export class Player extends Destructable {
       start = clamp(this.loop.start, 0, duration);
     }
 
-    this.audio.source.start(0, start ?? 0, duration ?? this.duration);
-    this.audio.source.addEventListener('ended', this.handleEnded);
+    if (this.audio.el) {
+      this.audio.el.currentTime = this.currentTime;
+      this.audio.el.addEventListener('ended', this.handleEnded);
+      this.audio.el.play();
+    }
+
     this.watch();
   }
 
@@ -241,16 +253,16 @@ export class Player extends Destructable {
   private disconnectSource() {
     if (this.isDestroyed || !this.audio || !this.connected) return;
     this.connected = false;
-    this.audio.source?.removeEventListener('ended', this.handleEnded);
-    this.audio.source?.stop(0);
+
+    if (this.audio.el) {
+      this.audio.el.removeEventListener('ended', this.handleEnded);
+    }
     this.audio.disconnect();
   }
 
   private cleanupSource() {
     if (this.isDestroyed || !this.audio) return;
     this.disconnectSource();
-
-    delete this.audio.source;
     delete this.audio;
   }
 
@@ -273,18 +285,25 @@ export class Player extends Destructable {
     }
   }
 
-  private updateCurrentTime(forceTimeToEnd?: boolean) {
+  private updateCurrentTime(forceEnd = false) {
     const now = performance.now();
-    const tick = (( now - this.timestamp) / 1000) * this.rate;
+    const tick = ((now - this.timestamp) / 1000) * this.rate;
 
     this.timestamp = now;
 
     const end = this.loop?.end ?? this.duration;
 
-    const newTime = forceTimeToEnd ? this.duration : clamp(this.time + tick, 0, end); 
+    const newTime = forceEnd ? this.duration : clamp(this.time + tick, 0, end); 
 
     this.time = newTime;
-    this.wf.invoke('playing', [this.time]);
+
+    if (!this.loop && this.time >= this.duration - tick) {
+      this.time = this.duration;
+      this.wf.invoke('playing', [this.duration]);
+      this.playEnded();
+    } else {
+      this.wf.invoke('playing', [this.time]);
+    }
   }
 
   private stopWatch() {