diff --git a/src/App.tsx b/src/App.tsx index 5684350..45cab10 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -2,6 +2,7 @@ import { useState } from "react"; import "./App.css"; import RecordAudio from "./RecordAudio"; import { Audiovis } from "./Audiovis"; +import { Compare } from "./Compare"; function App() { const [blobs, setBlobs] = useState([]); @@ -11,9 +12,11 @@ function App() {

Audiovis

setBlobs((prev) => [b, ...prev])} /> - {blobs.map((blob, i) => ( - - ))} + {blobs.length === 2 ? ( + + ) : ( + blobs.map((blob, i) => ) + )} ); } diff --git a/src/Compare.module.css b/src/Compare.module.css new file mode 100644 index 0000000..dfe1f4d --- /dev/null +++ b/src/Compare.module.css @@ -0,0 +1,22 @@ +.container { + display: grid; + + margin-top: 3em; + + grid-template-areas: + "a b" + ". c"; +} + +.container canvas:nth-child(1) { + grid-area: a; + /* transform: rotate(90deg); */ +} + +.container canvas:nth-child(2) { + grid-area: b; +} + +.container canvas:nth-child(3) { + grid-area: c; +} diff --git a/src/Compare.tsx b/src/Compare.tsx new file mode 100644 index 0000000..35e8946 --- /dev/null +++ b/src/Compare.tsx @@ -0,0 +1,145 @@ +import { FC, useEffect, useMemo, useRef } from "react"; +import { + spectrum, + spectrumToImage, + useAudioBuffer, + valueToColor, +} from "./util"; +import styles from "./Compare.module.css"; + +export const Compare: FC<{ a: Blob; b: Blob }> = ({ a, b }) => { + const bufA = useAudioBuffer(a); + const bufB = useAudioBuffer(b); + + const specA = useMemo(() => (bufA ? spectrum(bufA, 512) : null), [bufA]); + const specB = useMemo(() => (bufB ? spectrum(bufB, 512) : null), [bufB]); + + // similiarity + + const similarity = useMemo(() => { + if (specA && specB) { + const aLen = specA.length / 512; + const bLen = specB.length / 512; + + const sim = new Float32Array(aLen * bLen); + + for (let i = 0; i < aLen; i++) { + for (let j = 0; j < bLen; j++) { + let dotProduct = 0; + let normA = 0; + let normB = 0; + for (let k = 0; k < 512; k++) { + const aVal = specA[i * 512 + k]; + const bVal = specB[j * 512 + k]; + dotProduct += aVal * bVal; + normA += aVal * aVal; + normB += bVal * bVal; + } + sim[i * bLen + j] = + dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); + } + } + + console.log(sim); + + return sim; + } + + return null; + }, [specA, specB]); + + if (specA && specB && similarity) { + return ( +
+ + + +
+ ); + } + + return

Compare

; +}; + +const Foobar: FC<{ spec: Float32Array; rotate?: true }> = ({ + spec, + rotate, +}) => { + const canvas = useRef(null); + + const imData = useMemo(() => spectrumToImage(spec, 512), [spec]); + + useEffect(() => { + const ctx = canvas.current?.getContext("2d"); + if (ctx) { + const im = rotate ? rotateImageData90(imData) : imData; + + ctx.canvas.width = im.width; + ctx.canvas.height = im.height; + + ctx.putImageData(im, 0, 0); + } + }, [imData, rotate]); + + return ; +}; + +const Baz: FC<{ similarity: Float32Array; width: number }> = ({ + similarity, + width, +}) => { + const canvas = useRef(null); + + useEffect(() => { + const ctx = canvas.current?.getContext("2d"); + if (ctx) { + const height = similarity.length / width; + const imData = new ImageData(width, height); + + for (let i = 0; i < similarity.length; i++) { + const element = similarity[i]; + + const [r, g, b, a] = valueToColor(element); + + imData.data[i * 4] = r; + imData.data[i * 4 + 1] = g; + imData.data[i * 4 + 2] = b; + imData.data[i * 4 + 3] = a; + } + + ctx.canvas.width = imData.width; + ctx.canvas.height = imData.height; + + ctx.putImageData(imData, 0, 0); + } + }, [similarity]); + + return ; +}; + +function rotateImageData90(imageData: ImageData) { + const { width, height, data } = imageData; + // Create a new ImageData object with swapped dimensions + const rotatedImageData = new ImageData(height, width); + + const rotatedData = rotatedImageData.data; + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + // Calculate the source pixel index + const srcIndex = (y * width + x) * 4; + + // Calculate the destination pixel index + // Transpose the coordinates and reverse the Y-axis + const dstIndex = (x * height + (height - y - 1)) * 4; + + // Copy the RGBA values + rotatedData[dstIndex] = data[srcIndex]; // R + rotatedData[dstIndex + 1] = data[srcIndex + 1]; // G + rotatedData[dstIndex + 2] = data[srcIndex + 2]; // B + rotatedData[dstIndex + 3] = data[srcIndex + 3]; // A + } + } + + return rotatedImageData; +} diff --git a/src/util.ts b/src/util.ts new file mode 100644 index 0000000..5da8de6 --- /dev/null +++ b/src/util.ts @@ -0,0 +1,84 @@ +import FFT from "fft.js"; +import { useAudioCtx } from "./AudioCtxCtx"; +import { useEffect, useState } from "react"; + +/** load the spectrum for an entire audio buffer*/ +export function spectrum( + audio: AudioBuffer, + fftSize: 2 | 4 | 8 | 16 | 32 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096 +): Float32Array { + const fft = new FFT(fftSize); + const data = audio.getChannelData(0); + + const chunks = Math.floor(audio.length / fftSize); + const target = new Float32Array(fftSize * chunks); + + const sample = new Array(fftSize); + const out = fft.createComplexArray(); + + for (let i = 0; i < chunks; i++) { + const offset = i * fftSize; + for (let i = 0; i < sample.length; i++) sample[i] = data[i + offset]; + fft.realTransform(out, sample); + + for (let j = 0; j < fftSize; j++) { + target[i * fftSize + j] = Math.abs(out[j]); + } + } + + return target; +} + +/** Render a spectrum as a visible image to be written to canvas */ +export function spectrumToImage( + spectrum: Float32Array, + fftSize: 2 | 4 | 8 | 16 | 32 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096 +) { + const chunks = spectrum.length / fftSize; + + const image = new ImageData(chunks, fftSize); + + const max = spectrum.reduce((a, b) => Math.max(a, b), -Infinity); + + for (let i = 0; i < chunks; i++) { + for (let j = 0; j < fftSize; j++) { + const idx = j * chunks + i; + const value = spectrum[i * fftSize + j]; + + const scaledValue = Math.pow(value / max, 0.3); // Apply a power scale + + const [r, g, b, a] = valueToColor(scaledValue); + + image.data[idx * 4] = r; + image.data[idx * 4 + 1] = g; + image.data[idx * 4 + 2] = b; + image.data[idx * 4 + 3] = a; + } + } + + return image; +} + +export function useAudioBuffer(src: Blob) { + const audioCtx = useAudioCtx(); + const [buffer, setBuffer] = useState(); + + useEffect(() => { + if (audioCtx) { + src + .arrayBuffer() + .then((bytes) => audioCtx.decodeAudioData(bytes)) + .then(setBuffer); + } + }, [audioCtx, src]); + + return buffer; +} + +export function valueToColor(value: number) { + const r = value < 0.5 ? 0 : 255 * (value - 0.5) * 2; + const g = value < 0.5 ? 255 * value * 2 : 255 * (1 - value) * 2; + const b = value < 0.5 ? 255 * (1 - value * 2) : 0; + const a = value > 0.2 ? 255 : value * 5 * 255; + return [r, g, b, a]; +}