Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/melscale frequency #445

Merged
merged 6 commits into from
Mar 8, 2024
+125 −31
Merged

Feature/melscale frequency #445

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ba5de2b
use mel scale for frequencies
Dec 21, 2023
2d2fe03
create mel scale from d3 scale
Dec 21, 2023
acaf159
fix mel scale
Mar 8, 2024
6e29859
use mel scale for frequencies
Dec 21, 2023
9e5ba88
Merge branch 'main' into feature/melscale-frequency
Mar 8, 2024
008c5ff
outsource melscale to different file
Mar 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions src/lenses/SpectrogramLens/MelScale.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import * as d3 from 'd3';

interface MelScale {
(value: number): number;
toMelScale(value: number): number;
fromMelScale(frequency: number): number;
domain(): number[];
domain(domain: number[]): MelScale;
range(): number[];
range(range: number[]): MelScale;
copy(): MelScale;
invert(value: number): number;
ticks(count?: number): number[];
tickFormat(count?: number, specifier?: string): (d: number) => string;
}

const melScale = (): MelScale => {
// Create the base log scale
const linearScale = d3.scaleLinear();
const logScale = d3.scaleLog();

// Our custom scale function
const scale: MelScale = ((value: number) => {
return linearScale(value);
}) as MelScale;

scale.toMelScale = (frequency: number): number => {
return 2595 * Math.log10(1 + frequency / 700);
};

scale.fromMelScale = (mel: number): number => {
return 700 * (Math.pow(10, mel / 2595) - 1);
};

// Copy methods from the log scale
// eslint-disable-next-line @typescript-eslint/no-explicit-any
scale.domain = (domain?: number[]): any => {
if (domain === undefined) {
return linearScale.domain().map((d) => scale.toMelScale(d));
}
logScale.domain(domain);
return domain ? (linearScale.domain(domain), scale) : linearScale.domain();
};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
scale.range = (range?: number[]): any => {
if (range) logScale.range(range);
return range ? (linearScale.range(range), scale) : linearScale.range();
};
scale.copy = () => {
return melScale().domain(scale.domain()).range(scale.range());
};
scale.invert = (value: number): number => {
return scale.fromMelScale(linearScale.invert(value));
};

scale.ticks = (count?: number): number[] => {
const ticks = logScale.ticks(count).map((val) => {
return scale.toMelScale(val);
});
if (!count) return ticks;

return [ticks[0]].concat(ticks.slice(ticks.length - count, ticks.length));
};

return scale;
};

export default melScale;
88 changes: 57 additions & 31 deletions src/lenses/SpectrogramLens/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ import { ColorsState, useColors } from '../../stores/colors';
import { Lens } from '../../types';
import useSetting from '../useSetting';
import MenuBar from './MenuBar';
import { fixWindow, freqType, unitType, amplitudeToDb, hzToMel } from './Spectrogram';
import melScale from './MelScale';
import { fixWindow, freqType, unitType, amplitudeToDb } from './Spectrogram';

const Container = tw.div`flex flex-col w-full h-full items-stretch justify-center`;
const EmptyNote = styled.p`
Expand All @@ -22,7 +23,7 @@ interface WebAudio_ extends WebAudio {
buffer: AudioBuffer;
}

const LOG_DOMAIN_LOWER_LIMIT = 10;
const DOMAIN_LOWER_LIMIT = 10;
const FFT_SAMPLES = 1024;

/*
Expand All @@ -41,14 +42,16 @@ const drawScale = (
numTicks = Math.round(height / 20);
} else if (scale === 'linear') {
numTicks = Math.round(height / 30);
} else if (scale === 'mel') {
numTicks = Math.round(height / 40);
} else {
numTicks = 5;
}

let axis;

if (scale === 'logarithmic') {
const domain = [LOG_DOMAIN_LOWER_LIMIT, upperLimit];
const domain = [DOMAIN_LOWER_LIMIT, upperLimit];
const range = [height, 0];
const scale = d3.scaleLog(domain, range);

Expand All @@ -60,6 +63,23 @@ const drawScale = (
.ticks(numTicks, (x: number) => {
return `${freqType(x).toFixed(1)} ${unitType(x)}`;
});
} else if (scale === 'mel') {
const domain: [number, number] = [DOMAIN_LOWER_LIMIT, upperLimit];
const range: [number, number] = [height, 0];
const scale = melScale().domain(domain).range(range);

axis = d3
.axisRight(scale)
.scale(scale)
.tickPadding(1)
.tickSizeOuter(0)
.ticks(numTicks)
.tickFormat(
(x: number) =>
`${freqType(scale.fromMelScale(x).valueOf()).toFixed(1)} ${unitType(
scale.fromMelScale(x).valueOf()
)}`
);
} else {
const domain = [upperLimit, 0];
const range = [0, height];
Expand Down Expand Up @@ -187,22 +207,22 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => {
const upperLimit = backend.buffer?.sampleRate / 2;

// 10, ..., half-samplerate (default 22050)
const domain = [LOG_DOMAIN_LOWER_LIMIT, upperLimit];
const domain = [DOMAIN_LOWER_LIMIT, upperLimit];

// 0, ..., canvas-height
const range = [0, height];

const imageData = new ImageData(width, height);

const scaleFunc = d3.scaleLog(domain, range);

// Default to linear scale
let heightScale = d3.scaleLinear(domain, [0, upperLimit]);

if (freqScale === 'logarithmic') {
heightScale = d3.scaleLinear(domain, [0, FFT_SAMPLES / 2 - 1]);
} else if (freqScale === 'linear') {
heightScale = d3.scaleLinear(domain, [0, upperLimit]);
heightScale = d3.scaleLinear(range, [0, FFT_SAMPLES / 2]);
} else if (freqScale === 'mel') {
heightScale = d3.scaleLinear(domain, [0, FFT_SAMPLES / 2 - 1]);
}

const widthScale = d3.scaleLinear([0, width], [0, frequenciesData.length]);
Expand All @@ -221,7 +241,6 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => {
ref = maxI;
}
}
//const top_db = 80;
const amin = 1e-5;

// Convert amplitudes to decibels
Expand All @@ -241,24 +260,6 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => {
}
}

drawData[i] = col;
}
} else if (ampScale === 'mel') {
for (let i = 0; i < frequenciesData.length; i++) {
const col = [];

for (let j = 0; j < frequenciesData[i].length; j++) {
const amplitude = frequenciesData[i][j];
col[j] = hzToMel(amplitude ** 2);

if (col[j] > max) {
max = col[j];
}

if (col[j] < min) {
min = col[j];
}
}
drawData[i] = col;
}
} else {
Expand All @@ -278,13 +279,21 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => {
}
const colorScale = colorPalette.scale().domain([min, max]);

const scaleFunc = d3.scaleLog(domain, range);

for (let y = 0; y < height; y++) {
let value = 0;

if (freqScale === 'logarithmic') {
value = heightScale(scaleFunc.invert(height - y));
} else if (freqScale === 'linear') {
value = Math.abs(heightScale(height - y));
value = heightScale(height - y);
} else if (freqScale === 'mel') {
const scaleFunc = melScale()
.domain([DOMAIN_LOWER_LIMIT, melScale().toMelScale(upperLimit)])
.range(range);
heightScale = d3.scaleLinear([0, upperLimit], [0, FFT_SAMPLES / 2]);
value = heightScale(scaleFunc.invert(height - y));
}

const indexA = Math.floor(value);
Expand Down Expand Up @@ -388,7 +397,7 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => {
const coords = d3.pointer(e);

if (freqScale === 'logarithmic') {
const domain = [LOG_DOMAIN_LOWER_LIMIT, upperLimit];
const domain = [DOMAIN_LOWER_LIMIT, upperLimit];
const range = [0, height];

const scale = d3.scaleLog(domain, range);
Expand All @@ -399,6 +408,24 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => {
.attr('y', coords[1])
.attr('font-size', 10)
.attr('fill', theme`colors.white`);
} else if (freqScale === 'mel') {
let y = height - coords[1];

const domain: [number, number] = [
DOMAIN_LOWER_LIMIT,
melScale().toMelScale(upperLimit),
];
const range: [number, number] = [0, height];

const scaleFunc = melScale().domain(domain).range(range);
y = scaleFunc.invert(y);

d3.select(mouseLabel.current)
.text(`${freqType(y).toFixed(1)} ${unitType(y)}`)
.attr('x', coords[0])
.attr('y', coords[1])
.attr('font-size', 10)
.attr('fill', theme`colors.white`);
} else {
//if (freqScale === 'linear') {
const domain = [upperLimit, 0];
Expand Down Expand Up @@ -462,12 +489,11 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => {
// Add the menubar as last component, so that it is rendered on top
// We don't use a z-index for this, because it interferes with the rendering of the contained menus
}

<MenuBar
availableFreqScales={['linear', 'logarithmic']}
availableFreqScales={['linear', 'logarithmic', 'mel']}
freqScale={freqScale}
onChangeFreqScale={handleFreqScaleChange}
availableAmpScales={['decibel', 'linear', 'mel']}
availableAmpScales={['decibel', 'linear']}
ampScale={ampScale}
onChangeAmpScale={handleAmpScaleChange}
/>
Expand Down