Skip to content

Commit e6eb16f

Browse files
benfoxallbenfoxall
committed
Audio comparison
1 parent 64cbce1 commit e6eb16f

File tree

4 files changed

+257
-3
lines changed

4 files changed

+257
-3
lines changed

‎src/App.tsx

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { useState } from "react";
22
import "./App.css";
33
import RecordAudio from "./RecordAudio";
44
import { Audiovis } from "./Audiovis";
5+
import { Compare } from "./Compare";
56

67
function App() {
78
const [blobs, setBlobs] = useState<Blob[]>([]);
@@ -11,9 +12,11 @@ function App() {
1112
<h1>Audiovis</h1>
1213
<RecordAudio onCreated={(b) => setBlobs((prev) => [b, ...prev])} />
1314

14-
{blobs.map((blob, i) => (
15-
<Audiovis key={i} srcObject={blob} />
16-
))}
15+
{blobs.length === 2 ? (
16+
<Compare a={blobs[0]} b={blobs[1]} />
17+
) : (
18+
blobs.map((blob, i) => <Audiovis key={i} srcObject={blob} />)
19+
)}
1720
</>
1821
);
1922
}

‎src/Compare.module.css

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
.container {
2+
display: grid;
3+
4+
margin-top: 3em;
5+
6+
grid-template-areas:
7+
"a b"
8+
". c";
9+
}
10+
11+
.container canvas:nth-child(1) {
12+
grid-area: a;
13+
/* transform: rotate(90deg); */
14+
}
15+
16+
.container canvas:nth-child(2) {
17+
grid-area: b;
18+
}
19+
20+
.container canvas:nth-child(3) {
21+
grid-area: c;
22+
}

‎src/Compare.tsx

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
import { FC, useEffect, useMemo, useRef } from "react";
2+
import {
3+
spectrum,
4+
spectrumToImage,
5+
useAudioBuffer,
6+
valueToColor,
7+
} from "./util";
8+
import styles from "./Compare.module.css";
9+
10+
export const Compare: FC<{ a: Blob; b: Blob }> = ({ a, b }) => {
11+
const bufA = useAudioBuffer(a);
12+
const bufB = useAudioBuffer(b);
13+
14+
const specA = useMemo(() => (bufA ? spectrum(bufA, 512) : null), [bufA]);
15+
const specB = useMemo(() => (bufB ? spectrum(bufB, 512) : null), [bufB]);
16+
17+
// similiarity
18+
19+
const similarity = useMemo(() => {
20+
if (specA && specB) {
21+
const aLen = specA.length / 512;
22+
const bLen = specB.length / 512;
23+
24+
const sim = new Float32Array(aLen * bLen);
25+
26+
for (let i = 0; i < aLen; i++) {
27+
for (let j = 0; j < bLen; j++) {
28+
let dotProduct = 0;
29+
let normA = 0;
30+
let normB = 0;
31+
for (let k = 0; k < 512; k++) {
32+
const aVal = specA[i * 512 + k];
33+
const bVal = specB[j * 512 + k];
34+
dotProduct += aVal * bVal;
35+
normA += aVal * aVal;
36+
normB += bVal * bVal;
37+
}
38+
sim[i * bLen + j] =
39+
dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
40+
}
41+
}
42+
43+
console.log(sim);
44+
45+
return sim;
46+
}
47+
48+
return null;
49+
}, [specA, specB]);
50+
51+
if (specA && specB && similarity) {
52+
return (
53+
<div className={styles.container}>
54+
<Foobar spec={specA} rotate />
55+
<Baz similarity={similarity} width={specA.length / 512} />
56+
<Foobar spec={specB} />
57+
</div>
58+
);
59+
}
60+
61+
return <h1>Compare</h1>;
62+
};
63+
64+
const Foobar: FC<{ spec: Float32Array; rotate?: true }> = ({
65+
spec,
66+
rotate,
67+
}) => {
68+
const canvas = useRef<HTMLCanvasElement>(null);
69+
70+
const imData = useMemo(() => spectrumToImage(spec, 512), [spec]);
71+
72+
useEffect(() => {
73+
const ctx = canvas.current?.getContext("2d");
74+
if (ctx) {
75+
const im = rotate ? rotateImageData90(imData) : imData;
76+
77+
ctx.canvas.width = im.width;
78+
ctx.canvas.height = im.height;
79+
80+
ctx.putImageData(im, 0, 0);
81+
}
82+
}, [imData, rotate]);
83+
84+
return <canvas ref={canvas} />;
85+
};
86+
87+
const Baz: FC<{ similarity: Float32Array; width: number }> = ({
88+
similarity,
89+
width,
90+
}) => {
91+
const canvas = useRef<HTMLCanvasElement>(null);
92+
93+
useEffect(() => {
94+
const ctx = canvas.current?.getContext("2d");
95+
if (ctx) {
96+
const height = similarity.length / width;
97+
const imData = new ImageData(width, height);
98+
99+
for (let i = 0; i < similarity.length; i++) {
100+
const element = similarity[i];
101+
102+
const [r, g, b, a] = valueToColor(element);
103+
104+
imData.data[i * 4] = r;
105+
imData.data[i * 4 + 1] = g;
106+
imData.data[i * 4 + 2] = b;
107+
imData.data[i * 4 + 3] = a;
108+
}
109+
110+
ctx.canvas.width = imData.width;
111+
ctx.canvas.height = imData.height;
112+
113+
ctx.putImageData(imData, 0, 0);
114+
}
115+
}, [similarity]);
116+
117+
return <canvas ref={canvas} />;
118+
};
119+
120+
function rotateImageData90(imageData: ImageData) {
121+
const { width, height, data } = imageData;
122+
// Create a new ImageData object with swapped dimensions
123+
const rotatedImageData = new ImageData(height, width);
124+
125+
const rotatedData = rotatedImageData.data;
126+
127+
for (let y = 0; y < height; y++) {
128+
for (let x = 0; x < width; x++) {
129+
// Calculate the source pixel index
130+
const srcIndex = (y * width + x) * 4;
131+
132+
// Calculate the destination pixel index
133+
// Transpose the coordinates and reverse the Y-axis
134+
const dstIndex = (x * height + (height - y - 1)) * 4;
135+
136+
// Copy the RGBA values
137+
rotatedData[dstIndex] = data[srcIndex]; // R
138+
rotatedData[dstIndex + 1] = data[srcIndex + 1]; // G
139+
rotatedData[dstIndex + 2] = data[srcIndex + 2]; // B
140+
rotatedData[dstIndex + 3] = data[srcIndex + 3]; // A
141+
}
142+
}
143+
144+
return rotatedImageData;
145+
}

‎src/util.ts

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import FFT from "fft.js";
2+
import { useAudioCtx } from "./AudioCtxCtx";
3+
import { useEffect, useState } from "react";
4+
5+
/** load the spectrum for an entire audio buffer*/
6+
export function spectrum(
7+
audio: AudioBuffer,
8+
fftSize: 2 | 4 | 8 | 16 | 32 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096
9+
): Float32Array {
10+
const fft = new FFT(fftSize);
11+
const data = audio.getChannelData(0);
12+
13+
const chunks = Math.floor(audio.length / fftSize);
14+
const target = new Float32Array(fftSize * chunks);
15+
16+
const sample = new Array(fftSize);
17+
const out = fft.createComplexArray();
18+
19+
for (let i = 0; i < chunks; i++) {
20+
const offset = i * fftSize;
21+
for (let i = 0; i < sample.length; i++) sample[i] = data[i + offset];
22+
fft.realTransform(out, sample);
23+
24+
for (let j = 0; j < fftSize; j++) {
25+
target[i * fftSize + j] = Math.abs(out[j]);
26+
}
27+
}
28+
29+
return target;
30+
}
31+
32+
/** Render a spectrum as a visible image to be written to canvas */
33+
export function spectrumToImage(
34+
spectrum: Float32Array,
35+
fftSize: 2 | 4 | 8 | 16 | 32 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096
36+
) {
37+
const chunks = spectrum.length / fftSize;
38+
39+
const image = new ImageData(chunks, fftSize);
40+
41+
const max = spectrum.reduce((a, b) => Math.max(a, b), -Infinity);
42+
43+
for (let i = 0; i < chunks; i++) {
44+
for (let j = 0; j < fftSize; j++) {
45+
const idx = j * chunks + i;
46+
const value = spectrum[i * fftSize + j];
47+
48+
const scaledValue = Math.pow(value / max, 0.3); // Apply a power scale
49+
50+
const [r, g, b, a] = valueToColor(scaledValue);
51+
52+
image.data[idx * 4] = r;
53+
image.data[idx * 4 + 1] = g;
54+
image.data[idx * 4 + 2] = b;
55+
image.data[idx * 4 + 3] = a;
56+
}
57+
}
58+
59+
return image;
60+
}
61+
62+
export function useAudioBuffer(src: Blob) {
63+
const audioCtx = useAudioCtx();
64+
const [buffer, setBuffer] = useState<AudioBuffer>();
65+
66+
useEffect(() => {
67+
if (audioCtx) {
68+
src
69+
.arrayBuffer()
70+
.then((bytes) => audioCtx.decodeAudioData(bytes))
71+
.then(setBuffer);
72+
}
73+
}, [audioCtx, src]);
74+
75+
return buffer;
76+
}
77+
78+
export function valueToColor(value: number) {
79+
const r = value < 0.5 ? 0 : 255 * (value - 0.5) * 2;
80+
const g = value < 0.5 ? 255 * value * 2 : 255 * (1 - value) * 2;
81+
const b = value < 0.5 ? 255 * (1 - value * 2) : 0;
82+
const a = value > 0.2 ? 255 : value * 5 * 255;
83+
return [r, g, b, a];
84+
}

0 commit comments

Comments
 (0)