Audio comparison

src/App.tsx

@@ -2,6 +2,7 @@ import { useState } from "react";
 import "./App.css";
 import RecordAudio from "./RecordAudio";
 import { Audiovis } from "./Audiovis";
+import { Compare } from "./Compare";
 
 function App() {
   const [blobs, setBlobs] = useState<Blob[]>([]);
@@ -11,9 +12,11 @@ function App() {
       <h1>Audiovis</h1>
       <RecordAudio onCreated={(b) => setBlobs((prev) => [b, ...prev])} />
 
-      {blobs.map((blob, i) => (
-        <Audiovis key={i} srcObject={blob} />
-      ))}
+      {blobs.length === 2 ? (
+        <Compare a={blobs[0]} b={blobs[1]} />
+      ) : (
+        blobs.map((blob, i) => <Audiovis key={i} srcObject={blob} />)
+      )}
     </>
   );
 }

src/Compare.module.css

@@ -0,0 +1,22 @@
+.container {
+  display: grid;
+
+  margin-top: 3em;
+
+  grid-template-areas:
+    "a b"
+    ". c";
+}
+
+.container canvas:nth-child(1) {
+  grid-area: a;
+  /* transform: rotate(90deg); */
+}
+
+.container canvas:nth-child(2) {
+  grid-area: b;
+}
+
+.container canvas:nth-child(3) {
+  grid-area: c;
+}

src/Compare.tsx

@@ -0,0 +1,145 @@
+import { FC, useEffect, useMemo, useRef } from "react";
+import {
+  spectrum,
+  spectrumToImage,
+  useAudioBuffer,
+  valueToColor,
+} from "./util";
+import styles from "./Compare.module.css";
+
+export const Compare: FC<{ a: Blob; b: Blob }> = ({ a, b }) => {
+  const bufA = useAudioBuffer(a);
+  const bufB = useAudioBuffer(b);
+
+  const specA = useMemo(() => (bufA ? spectrum(bufA, 512) : null), [bufA]);
+  const specB = useMemo(() => (bufB ? spectrum(bufB, 512) : null), [bufB]);
+
+  // similiarity
+
+  const similarity = useMemo(() => {
+    if (specA && specB) {
+      const aLen = specA.length / 512;
+      const bLen = specB.length / 512;
+
+      const sim = new Float32Array(aLen * bLen);
+
+      for (let i = 0; i < aLen; i++) {
+        for (let j = 0; j < bLen; j++) {
+          let dotProduct = 0;
+          let normA = 0;
+          let normB = 0;
+          for (let k = 0; k < 512; k++) {
+            const aVal = specA[i * 512 + k];
+            const bVal = specB[j * 512 + k];
+            dotProduct += aVal * bVal;
+            normA += aVal * aVal;
+            normB += bVal * bVal;
+          }
+          sim[i * bLen + j] =
+            dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+        }
+      }
+
+      console.log(sim);
+
+      return sim;
+    }
+
+    return null;
+  }, [specA, specB]);
+
+  if (specA && specB && similarity) {
+    return (
+      <div className={styles.container}>
+        <Foobar spec={specA} rotate />
+        <Baz similarity={similarity} width={specA.length / 512} />
+        <Foobar spec={specB} />
+      </div>
+    );
+  }
+
+  return <h1>Compare</h1>;
+};
+
+const Foobar: FC<{ spec: Float32Array; rotate?: true }> = ({
+  spec,
+  rotate,
+}) => {
+  const canvas = useRef<HTMLCanvasElement>(null);
+
+  const imData = useMemo(() => spectrumToImage(spec, 512), [spec]);
+
+  useEffect(() => {
+    const ctx = canvas.current?.getContext("2d");
+    if (ctx) {
+      const im = rotate ? rotateImageData90(imData) : imData;
+
+      ctx.canvas.width = im.width;
+      ctx.canvas.height = im.height;
+
+      ctx.putImageData(im, 0, 0);
+    }
+  }, [imData, rotate]);
+
+  return <canvas ref={canvas} />;
+};
+
+const Baz: FC<{ similarity: Float32Array; width: number }> = ({
+  similarity,
+  width,
+}) => {
+  const canvas = useRef<HTMLCanvasElement>(null);
+
+  useEffect(() => {
+    const ctx = canvas.current?.getContext("2d");
+    if (ctx) {
+      const height = similarity.length / width;
+      const imData = new ImageData(width, height);
+
+      for (let i = 0; i < similarity.length; i++) {
+        const element = similarity[i];
+
+        const [r, g, b, a] = valueToColor(element);
+
+        imData.data[i * 4] = r;
+        imData.data[i * 4 + 1] = g;
+        imData.data[i * 4 + 2] = b;
+        imData.data[i * 4 + 3] = a;
+      }
+
+      ctx.canvas.width = imData.width;
+      ctx.canvas.height = imData.height;
+
+      ctx.putImageData(imData, 0, 0);
+    }
+  }, [similarity]);
+
+  return <canvas ref={canvas} />;
+};
+
+function rotateImageData90(imageData: ImageData) {
+  const { width, height, data } = imageData;
+  // Create a new ImageData object with swapped dimensions
+  const rotatedImageData = new ImageData(height, width);
+
+  const rotatedData = rotatedImageData.data;
+
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      // Calculate the source pixel index
+      const srcIndex = (y * width + x) * 4;
+
+      // Calculate the destination pixel index
+      // Transpose the coordinates and reverse the Y-axis
+      const dstIndex = (x * height + (height - y - 1)) * 4;
+
+      // Copy the RGBA values
+      rotatedData[dstIndex] = data[srcIndex]; // R
+      rotatedData[dstIndex + 1] = data[srcIndex + 1]; // G
+      rotatedData[dstIndex + 2] = data[srcIndex + 2]; // B
+      rotatedData[dstIndex + 3] = data[srcIndex + 3]; // A
+    }
+  }
+
+  return rotatedImageData;
+}

src/util.ts

@@ -0,0 +1,84 @@
+import FFT from "fft.js";
+import { useAudioCtx } from "./AudioCtxCtx";
+import { useEffect, useState } from "react";
+
+/**  load the spectrum for an entire audio buffer*/
+export function spectrum(
+  audio: AudioBuffer,
+  fftSize: 2 | 4 | 8 | 16 | 32 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096
+): Float32Array {
+  const fft = new FFT(fftSize);
+  const data = audio.getChannelData(0);
+
+  const chunks = Math.floor(audio.length / fftSize);
+  const target = new Float32Array(fftSize * chunks);
+
+  const sample = new Array(fftSize);
+  const out = fft.createComplexArray();
+
+  for (let i = 0; i < chunks; i++) {
+    const offset = i * fftSize;
+    for (let i = 0; i < sample.length; i++) sample[i] = data[i + offset];
+    fft.realTransform(out, sample);
+
+    for (let j = 0; j < fftSize; j++) {
+      target[i * fftSize + j] = Math.abs(out[j]);
+    }
+  }
+
+  return target;
+}
+
+/** Render a spectrum as a visible image to be written to canvas */
+export function spectrumToImage(
+  spectrum: Float32Array,
+  fftSize: 2 | 4 | 8 | 16 | 32 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096
+) {
+  const chunks = spectrum.length / fftSize;
+
+  const image = new ImageData(chunks, fftSize);
+
+  const max = spectrum.reduce((a, b) => Math.max(a, b), -Infinity);
+
+  for (let i = 0; i < chunks; i++) {
+    for (let j = 0; j < fftSize; j++) {
+      const idx = j * chunks + i;
+      const value = spectrum[i * fftSize + j];
+
+      const scaledValue = Math.pow(value / max, 0.3); // Apply a power scale
+
+      const [r, g, b, a] = valueToColor(scaledValue);
+
+      image.data[idx * 4] = r;
+      image.data[idx * 4 + 1] = g;
+      image.data[idx * 4 + 2] = b;
+      image.data[idx * 4 + 3] = a;
+    }
+  }
+
+  return image;
+}
+
+export function useAudioBuffer(src: Blob) {
+  const audioCtx = useAudioCtx();
+  const [buffer, setBuffer] = useState<AudioBuffer>();
+
+  useEffect(() => {
+    if (audioCtx) {
+      src
+        .arrayBuffer()
+        .then((bytes) => audioCtx.decodeAudioData(bytes))
+        .then(setBuffer);
+    }
+  }, [audioCtx, src]);
+
+  return buffer;
+}
+
+export function valueToColor(value: number) {
+  const r = value < 0.5 ? 0 : 255 * (value - 0.5) * 2;
+  const g = value < 0.5 ? 255 * value * 2 : 255 * (1 - value) * 2;
+  const b = value < 0.5 ? 255 * (1 - value * 2) : 0;
+  const a = value > 0.2 ? 255 : value * 5 * 255;
+  return [r, g, b, a];
+}