diff --git a/src/lenses/SpectrogramLens/MelScale.tsx b/src/lenses/SpectrogramLens/MelScale.tsx new file mode 100644 index 00000000..471364da --- /dev/null +++ b/src/lenses/SpectrogramLens/MelScale.tsx @@ -0,0 +1,68 @@ +import * as d3 from 'd3'; + +interface MelScale { + (value: number): number; + toMelScale(value: number): number; + fromMelScale(frequency: number): number; + domain(): number[]; + domain(domain: number[]): MelScale; + range(): number[]; + range(range: number[]): MelScale; + copy(): MelScale; + invert(value: number): number; + ticks(count?: number): number[]; + tickFormat(count?: number, specifier?: string): (d: number) => string; +} + +const melScale = (): MelScale => { + // Create the base log scale + const linearScale = d3.scaleLinear(); + const logScale = d3.scaleLog(); + + // Our custom scale function + const scale: MelScale = ((value: number) => { + return linearScale(value); + }) as MelScale; + + scale.toMelScale = (frequency: number): number => { + return 2595 * Math.log10(1 + frequency / 700); + }; + + scale.fromMelScale = (mel: number): number => { + return 700 * (Math.pow(10, mel / 2595) - 1); + }; + + // Copy methods from the log scale + // eslint-disable-next-line @typescript-eslint/no-explicit-any + scale.domain = (domain?: number[]): any => { + if (domain === undefined) { + return linearScale.domain().map((d) => scale.toMelScale(d)); + } + logScale.domain(domain); + return domain ? (linearScale.domain(domain), scale) : linearScale.domain(); + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + scale.range = (range?: number[]): any => { + if (range) logScale.range(range); + return range ? (linearScale.range(range), scale) : linearScale.range(); + }; + scale.copy = () => { + return melScale().domain(scale.domain()).range(scale.range()); + }; + scale.invert = (value: number): number => { + return scale.fromMelScale(linearScale.invert(value)); + }; + + scale.ticks = (count?: number): number[] => { + const ticks = logScale.ticks(count).map((val) => { + return scale.toMelScale(val); + }); + if (!count) return ticks; + + return [ticks[0]].concat(ticks.slice(ticks.length - count, ticks.length)); + }; + + return scale; +}; + +export default melScale; diff --git a/src/lenses/SpectrogramLens/index.tsx b/src/lenses/SpectrogramLens/index.tsx index ff22e17a..cf7197bc 100644 --- a/src/lenses/SpectrogramLens/index.tsx +++ b/src/lenses/SpectrogramLens/index.tsx @@ -10,7 +10,8 @@ import { ColorsState, useColors } from '../../stores/colors'; import { Lens } from '../../types'; import useSetting from '../useSetting'; import MenuBar from './MenuBar'; -import { fixWindow, freqType, unitType, amplitudeToDb, hzToMel } from './Spectrogram'; +import melScale from './MelScale'; +import { fixWindow, freqType, unitType, amplitudeToDb } from './Spectrogram'; const Container = tw.div`flex flex-col w-full h-full items-stretch justify-center`; const EmptyNote = styled.p` @@ -22,7 +23,7 @@ interface WebAudio_ extends WebAudio { buffer: AudioBuffer; } -const LOG_DOMAIN_LOWER_LIMIT = 10; +const DOMAIN_LOWER_LIMIT = 10; const FFT_SAMPLES = 1024; /* @@ -41,6 +42,8 @@ const drawScale = ( numTicks = Math.round(height / 20); } else if (scale === 'linear') { numTicks = Math.round(height / 30); + } else if (scale === 'mel') { + numTicks = Math.round(height / 40); } else { numTicks = 5; } @@ -48,7 +51,7 @@ const drawScale = ( let axis; if (scale === 'logarithmic') { - const domain = [LOG_DOMAIN_LOWER_LIMIT, upperLimit]; + const domain = [DOMAIN_LOWER_LIMIT, upperLimit]; const range = [height, 0]; const scale = d3.scaleLog(domain, range); @@ -60,6 +63,23 @@ const drawScale = ( .ticks(numTicks, (x: number) => { return `${freqType(x).toFixed(1)} ${unitType(x)}`; }); + } else if (scale === 'mel') { + const domain: [number, number] = [DOMAIN_LOWER_LIMIT, upperLimit]; + const range: [number, number] = [height, 0]; + const scale = melScale().domain(domain).range(range); + + axis = d3 + .axisRight(scale) + .scale(scale) + .tickPadding(1) + .tickSizeOuter(0) + .ticks(numTicks) + .tickFormat( + (x: number) => + `${freqType(scale.fromMelScale(x).valueOf()).toFixed(1)} ${unitType( + scale.fromMelScale(x).valueOf() + )}` + ); } else { const domain = [upperLimit, 0]; const range = [0, height]; @@ -187,22 +207,22 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { const upperLimit = backend.buffer?.sampleRate / 2; // 10, ..., half-samplerate (default 22050) - const domain = [LOG_DOMAIN_LOWER_LIMIT, upperLimit]; + const domain = [DOMAIN_LOWER_LIMIT, upperLimit]; // 0, ..., canvas-height const range = [0, height]; const imageData = new ImageData(width, height); - const scaleFunc = d3.scaleLog(domain, range); - // Default to linear scale let heightScale = d3.scaleLinear(domain, [0, upperLimit]); if (freqScale === 'logarithmic') { heightScale = d3.scaleLinear(domain, [0, FFT_SAMPLES / 2 - 1]); } else if (freqScale === 'linear') { - heightScale = d3.scaleLinear(domain, [0, upperLimit]); + heightScale = d3.scaleLinear(range, [0, FFT_SAMPLES / 2]); + } else if (freqScale === 'mel') { + heightScale = d3.scaleLinear(domain, [0, FFT_SAMPLES / 2 - 1]); } const widthScale = d3.scaleLinear([0, width], [0, frequenciesData.length]); @@ -221,7 +241,6 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { ref = maxI; } } - //const top_db = 80; const amin = 1e-5; // Convert amplitudes to decibels @@ -241,24 +260,6 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { } } - drawData[i] = col; - } - } else if (ampScale === 'mel') { - for (let i = 0; i < frequenciesData.length; i++) { - const col = []; - - for (let j = 0; j < frequenciesData[i].length; j++) { - const amplitude = frequenciesData[i][j]; - col[j] = hzToMel(amplitude ** 2); - - if (col[j] > max) { - max = col[j]; - } - - if (col[j] < min) { - min = col[j]; - } - } drawData[i] = col; } } else { @@ -278,13 +279,21 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { } const colorScale = colorPalette.scale().domain([min, max]); + const scaleFunc = d3.scaleLog(domain, range); + for (let y = 0; y < height; y++) { let value = 0; if (freqScale === 'logarithmic') { value = heightScale(scaleFunc.invert(height - y)); } else if (freqScale === 'linear') { - value = Math.abs(heightScale(height - y)); + value = heightScale(height - y); + } else if (freqScale === 'mel') { + const scaleFunc = melScale() + .domain([DOMAIN_LOWER_LIMIT, melScale().toMelScale(upperLimit)]) + .range(range); + heightScale = d3.scaleLinear([0, upperLimit], [0, FFT_SAMPLES / 2]); + value = heightScale(scaleFunc.invert(height - y)); } const indexA = Math.floor(value); @@ -388,7 +397,7 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { const coords = d3.pointer(e); if (freqScale === 'logarithmic') { - const domain = [LOG_DOMAIN_LOWER_LIMIT, upperLimit]; + const domain = [DOMAIN_LOWER_LIMIT, upperLimit]; const range = [0, height]; const scale = d3.scaleLog(domain, range); @@ -399,6 +408,24 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { .attr('y', coords[1]) .attr('font-size', 10) .attr('fill', theme`colors.white`); + } else if (freqScale === 'mel') { + let y = height - coords[1]; + + const domain: [number, number] = [ + DOMAIN_LOWER_LIMIT, + melScale().toMelScale(upperLimit), + ]; + const range: [number, number] = [0, height]; + + const scaleFunc = melScale().domain(domain).range(range); + y = scaleFunc.invert(y); + + d3.select(mouseLabel.current) + .text(`${freqType(y).toFixed(1)} ${unitType(y)}`) + .attr('x', coords[0]) + .attr('y', coords[1]) + .attr('font-size', 10) + .attr('fill', theme`colors.white`); } else { //if (freqScale === 'linear') { const domain = [upperLimit, 0]; @@ -462,12 +489,11 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { // Add the menubar as last component, so that it is rendered on top // We don't use a z-index for this, because it interferes with the rendering of the contained menus } -