From e4c82c0a64b744622301ca3d326e29365ed640f3 Mon Sep 17 00:00:00 2001 From: Boof2015 <75185879+Boof2015@users.noreply.github.com> Date: Tue, 19 May 2026 13:13:24 -0400 Subject: [PATCH] complete overhaul of spectrogram --- native/binding.gyp | 1 + native/src/main.cpp | 88 +++ native/src/spectrogram.cpp | 488 +++++++++++++++ native/src/spectrogram.h | 90 +++ src/preload/index.ts | 1 + src/renderer/audio/native/index.ts | 43 +- src/renderer/audio/native/visualizer-dsp.d.ts | 30 + src/renderer/components/ScopeModule.tsx | 1 + .../components/ScopeSettingsSection.tsx | 16 +- src/renderer/visualizers/Spectrogram.ts | 576 +++++------------- src/shared/profileState.ts | 8 +- src/types/settings.ts | 4 +- src/types/spectrogram.ts | 19 + test/renderer-helpers.test.ts | 255 ++++---- 14 files changed, 1079 insertions(+), 541 deletions(-) create mode 100644 native/src/spectrogram.cpp create mode 100644 native/src/spectrogram.h diff --git a/native/binding.gyp b/native/binding.gyp index 50b2859..2bb46bb 100644 --- a/native/binding.gyp +++ b/native/binding.gyp @@ -9,6 +9,7 @@ "src/main.cpp", "src/oscilloscope.cpp", "src/spectrum.cpp", + "src/spectrogram.cpp", "src/vectorscope.cpp", "src/dsp_utils.cpp" ], diff --git a/native/src/main.cpp b/native/src/main.cpp index f0251e0..b06f9d4 100644 --- a/native/src/main.cpp +++ b/native/src/main.cpp @@ -1,15 +1,18 @@ #include #include +#include #include "linux_capture.h" #include "macos_capture.h" #include "windows_capture.h" #include "oscilloscope.h" #include "spectrum.h" +#include "spectrogram.h" #include "vectorscope.h" // Global instances static Visualizer::Oscilloscope oscilloscope; static Visualizer::Spectrum spectrum(2048); +static Visualizer::SpectrogramAnalyzer spectrogramAnalyzer; static Visualizer::Vectorscope vectorscope; // ============== Oscilloscope ============== @@ -239,6 +242,84 @@ Napi::Value SpectrumReset(const Napi::CallbackInfo& info) { return info.Env().Undefined(); } +// ============== Spectrogram ============== + +namespace { +float GetObjectFloat(const Napi::Object& obj, const char* key, float fallback) { + Napi::Value value = obj.Get(key); + return value.IsNumber() ? value.As().FloatValue() : fallback; +} + +size_t GetObjectSize(const Napi::Object& obj, const char* key, size_t fallback) { + Napi::Value value = obj.Get(key); + return value.IsNumber() ? static_cast(value.As().Uint32Value()) : fallback; +} + +std::string GetObjectString(const Napi::Object& obj, const char* key, const std::string& fallback) { + Napi::Value value = obj.Get(key); + return value.IsString() ? value.As().Utf8Value() : fallback; +} +} // namespace + +Napi::Value SpectrogramConfigure(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() < 1 || !info[0].IsObject()) { + Napi::TypeError::New(env, "Expected spectrogram options object").ThrowAsJavaScriptException(); + return env.Null(); + } + + Napi::Object options = info[0].As(); + Visualizer::SpectrogramConfig config; + config.fftSize = GetObjectSize(options, "fftSize", config.fftSize); + config.sampleRate = GetObjectFloat(options, "sampleRate", config.sampleRate); + config.rowCount = GetObjectSize(options, "rowCount", config.rowCount); + config.minFrequency = GetObjectFloat(options, "minFrequency", config.minFrequency); + config.maxFrequency = GetObjectFloat(options, "maxFrequency", config.maxFrequency); + config.minDecibels = GetObjectFloat(options, "minDecibels", config.minDecibels); + config.maxDecibels = GetObjectFloat(options, "maxDecibels", config.maxDecibels); + config.scrollSpeed = GetObjectFloat(options, "scrollSpeed", config.scrollSpeed); + config.contrast = GetObjectFloat(options, "contrast", config.contrast); + config.tiltDbPerOctave = GetObjectFloat(options, "tiltDbPerOctave", config.tiltDbPerOctave); + config.clarityMode = GetObjectString(options, "clarityMode", config.clarityMode); + config.scaleMode = GetObjectString(options, "scaleMode", config.scaleMode); + config.orientation = GetObjectString(options, "orientation", config.orientation); + + spectrogramAnalyzer.configure(config); + return env.Undefined(); +} + +Napi::Value SpectrogramProcess(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() < 1 || !info[0].IsTypedArray()) { + Napi::TypeError::New(env, "Expected Float32Array").ThrowAsJavaScriptException(); + return env.Null(); + } + + Napi::Float32Array audioData = info[0].As(); + auto result = spectrogramAnalyzer.process(audioData.Data(), audioData.ElementLength()); + + Napi::Float32Array display = Napi::Float32Array::New(env, result.display.size()); + Napi::Float32Array heat = Napi::Float32Array::New(env, result.heat.size()); + if (!result.display.empty()) { + memcpy(display.Data(), result.display.data(), result.display.size() * sizeof(float)); + } + if (!result.heat.empty()) { + memcpy(heat.Data(), result.heat.data(), result.heat.size() * sizeof(float)); + } + + Napi::Object obj = Napi::Object::New(env); + obj.Set("display", display); + obj.Set("heat", heat); + obj.Set("columnCount", Napi::Number::New(env, static_cast(result.columnCount))); + obj.Set("rowCount", Napi::Number::New(env, static_cast(result.rowCount))); + return obj; +} + +Napi::Value SpectrogramReset(const Napi::CallbackInfo& info) { + spectrogramAnalyzer.reset(); + return info.Env().Undefined(); +} + // ============== Vectorscope ============== Napi::Value VectorscopeSetSampleRate(const Napi::CallbackInfo& info) { @@ -378,6 +459,13 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) { specExports.Set("reset", Napi::Function::New(env, SpectrumReset)); exports.Set("spectrum", specExports); + // Spectrogram + Napi::Object spectrogramExports = Napi::Object::New(env); + spectrogramExports.Set("configure", Napi::Function::New(env, SpectrogramConfigure)); + spectrogramExports.Set("process", Napi::Function::New(env, SpectrogramProcess)); + spectrogramExports.Set("reset", Napi::Function::New(env, SpectrogramReset)); + exports.Set("spectrogram", spectrogramExports); + // Vectorscope Napi::Object vecExports = Napi::Object::New(env); vecExports.Set("setSampleRate", Napi::Function::New(env, VectorscopeSetSampleRate)); diff --git a/native/src/spectrogram.cpp b/native/src/spectrogram.cpp new file mode 100644 index 0000000..2d51c64 --- /dev/null +++ b/native/src/spectrogram.cpp @@ -0,0 +1,488 @@ +#define _USE_MATH_DEFINES +#include "spectrogram.h" +#include +#include +#include +#include + +namespace Visualizer { + +namespace { +constexpr size_t FFT_PAD_FACTOR = 4; +constexpr float DISPLAY_GAIN_DB = 2.0f; +constexpr float HEAT_GAIN_COMPENSATION_DB = 6.0f; +constexpr float TILT_REFERENCE_HZ = 1000.0f; +constexpr float HEAT_MIN_DB = -100.0f; +constexpr float HEAT_MAX_DB = -20.0f; +constexpr float SLANEY_F_SP = 200.0f / 3.0f; +constexpr float SLANEY_MIN_LOG_HZ = 1000.0f; +constexpr float SLANEY_MIN_LOG_MEL = SLANEY_MIN_LOG_HZ / SLANEY_F_SP; +constexpr float SLANEY_LOG_STEP = 1.8562979903656263f / 27.0f; // log(6.4) / 27 + +bool isPowerOfTwo(size_t value) { + return value >= 2 && (value & (value - 1)) == 0; +} + +float clamp01(float value) { + return std::max(0.0f, std::min(1.0f, value)); +} + +float normalizeHeatDb(float db) { + if (!std::isfinite(db)) { + return 0.0f; + } + return clamp01((db - HEAT_MIN_DB) / (HEAT_MAX_DB - HEAT_MIN_DB)); +} + +float hzToMelSlaney(float frequencyHz) { + if (frequencyHz < SLANEY_MIN_LOG_HZ) { + return frequencyHz / SLANEY_F_SP; + } + return SLANEY_MIN_LOG_MEL + (std::log(frequencyHz / SLANEY_MIN_LOG_HZ) / SLANEY_LOG_STEP); +} + +float melToHzSlaney(float mel) { + if (mel < SLANEY_MIN_LOG_MEL) { + return mel * SLANEY_F_SP; + } + return SLANEY_MIN_LOG_HZ * std::exp(SLANEY_LOG_STEP * (mel - SLANEY_MIN_LOG_MEL)); +} + +float wrapPhase(float value) { + const float twoPi = static_cast(2.0 * M_PI); + float wrapped = std::remainder(value, twoPi); + if (!std::isfinite(wrapped)) { + return 0.0f; + } + return wrapped; +} +} // namespace + +SpectrogramAnalyzer::SpectrogramAnalyzer() + : fftSize_(0) + , paddedSize_(0) + , frameFill_(0) + , haveLastPhase_(false) { + configureFft(config_.fftSize); + rebuildFrequencyMapping(); +} + +void SpectrogramAnalyzer::configure(const SpectrogramConfig& config) { + SpectrogramConfig next = config; + + if (!isPowerOfTwo(next.fftSize)) { + next.fftSize = 4096; + } + next.fftSize = std::clamp(next.fftSize, static_cast(128), static_cast(16384)); + next.sampleRate = std::isfinite(next.sampleRate) && next.sampleRate > 0.0f ? next.sampleRate : 48000.0f; + next.rowCount = std::clamp(next.rowCount, static_cast(1), static_cast(8192)); + next.minFrequency = std::isfinite(next.minFrequency) && next.minFrequency > 0.0f ? next.minFrequency : 20.0f; + next.maxFrequency = std::isfinite(next.maxFrequency) && next.maxFrequency > 0.0f ? next.maxFrequency : 20000.0f; + next.minDecibels = std::isfinite(next.minDecibels) ? next.minDecibels : -90.0f; + next.maxDecibels = std::isfinite(next.maxDecibels) ? next.maxDecibels : -12.0f; + if (next.maxDecibels <= next.minDecibels) { + next.maxDecibels = next.minDecibels + 1.0f; + } + next.scrollSpeed = std::isfinite(next.scrollSpeed) ? next.scrollSpeed : 2.0f; + next.contrast = std::isfinite(next.contrast) ? next.contrast : 1.0f; + next.contrast = std::clamp(next.contrast, 0.1f, 8.0f); + next.tiltDbPerOctave = std::isfinite(next.tiltDbPerOctave) ? next.tiltDbPerOctave : 4.0f; + next.tiltDbPerOctave = std::clamp(next.tiltDbPerOctave, -12.0f, 12.0f); + if (next.scaleMode != "linear" && next.scaleMode != "mel" && next.scaleMode != "log") { + next.scaleMode = "log"; + } + if (next.orientation != "vertical") { + next.orientation = "horizontal"; + } + if (next.clarityMode != "classic" && next.clarityMode != "sharp" && next.clarityMode != "sharper") { + next.clarityMode = "sharper"; + } + + const bool fftChanged = next.fftSize != fftSize_; + const bool sampleRateChanged = next.sampleRate != config_.sampleRate; + const bool mappingChanged = fftChanged + || sampleRateChanged + || next.rowCount != config_.rowCount + || next.minFrequency != config_.minFrequency + || next.maxFrequency != config_.maxFrequency + || next.scaleMode != config_.scaleMode + || next.orientation != config_.orientation; + + config_ = next; + + if (fftChanged) { + configureFft(config_.fftSize); + } else if (sampleRateChanged) { + haveLastPhase_ = false; + } + + if (mappingChanged) { + rebuildFrequencyMapping(); + } +} + +void SpectrogramAnalyzer::configureFft(size_t fftSize) { + fftSize_ = fftSize; + paddedSize_ = fftSize_ * FFT_PAD_FACTOR; + fft_ = std::make_unique(paddedSize_); + frameBuffer_.assign(fftSize_, 0.0f); + window_.assign(fftSize_, 1.0f); + windowedInput_.assign(paddedSize_, 0.0f); + fftOutput_.assign(paddedSize_, std::complex(0.0f, 0.0f)); + magnitudesDb_.assign(paddedSize_ / 2, -200.0f); + magnitudesLinear_.assign(paddedSize_ / 2, 0.0f); + phases_.assign(paddedSize_ / 2, 0.0f); + lastPhases_.assign(paddedSize_ / 2, 0.0f); + frameFill_ = 0; + haveLastPhase_ = false; + + if (fftSize_ <= 1) { + return; + } + + for (size_t index = 0; index < fftSize_; index += 1) { + window_[index] = 0.5f * (1.0f - std::cos((2.0f * static_cast(M_PI) * index) / (fftSize_ - 1))); + } +} + +void SpectrogramAnalyzer::reset() { + std::fill(frameBuffer_.begin(), frameBuffer_.end(), 0.0f); + std::fill(lastPhases_.begin(), lastPhases_.end(), 0.0f); + frameFill_ = 0; + haveLastPhase_ = false; +} + +size_t SpectrogramAnalyzer::resolveHopSize() const { + const float baseHopDivisor = 8.0f; + const float speed = std::isfinite(config_.scrollSpeed) ? config_.scrollSpeed : 2.0f; + const int divisor = std::clamp(static_cast(std::lround(baseHopDivisor * speed)), 2, 64); + return std::max(static_cast(1), fftSize_ / static_cast(divisor)); +} + +void SpectrogramAnalyzer::rebuildFrequencyMapping() { + const size_t rowCount = std::max(static_cast(1), config_.rowCount); + const float sampleRate = std::max(1.0f, config_.sampleRate); + const float nyquist = sampleRate * 0.5f; + const float minFrequency = std::max(1.0f, std::min(config_.minFrequency, nyquist)); + const float maxFrequency = std::max(minFrequency + 1.0f, std::min(config_.maxFrequency, nyquist)); + config_.minFrequency = minFrequency; + config_.maxFrequency = maxFrequency; + + rowCenterBins_.assign(rowCount, 0.0f); + rowBandStartBins_.assign(rowCount, 0.0f); + rowBandEndBins_.assign(rowCount, 0.0f); + rowCenterFrequencies_.assign(rowCount, minFrequency); + standardRaw_.assign(rowCount, 0.0f); + standardHeat_.assign(rowCount, 0.0f); + reassignedPower_.assign(rowCount, 0.0f); + blendedRaw_.assign(rowCount, 0.0f); + blendedHeat_.assign(rowCount, 0.0f); + + const float rowSpan = static_cast(std::max(static_cast(1), rowCount - 1)); + const float numBins = static_cast(std::max(static_cast(1), paddedSize_ / 2)); + const float binWidth = nyquist / numBins; + + for (size_t row = 0; row < rowCount; row += 1) { + const float rowF = static_cast(row); + const float normalizedPosition = config_.orientation == "vertical" + ? rowF / rowSpan + : 1.0f - (rowF / rowSpan); + + float upperEdgeNormalized; + float lowerEdgeNormalized; + if (config_.orientation == "vertical") { + upperEdgeNormalized = row == rowCount - 1 ? 1.0f : (rowF + 0.5f) / rowSpan; + lowerEdgeNormalized = row == 0 ? 0.0f : (rowF - 0.5f) / rowSpan; + } else { + upperEdgeNormalized = row == 0 ? 1.0f : 1.0f - ((rowF - 0.5f) / rowSpan); + lowerEdgeNormalized = row == rowCount - 1 ? 0.0f : 1.0f - ((rowF + 0.5f) / rowSpan); + } + + const float centerFrequency = frequencyFromScale(normalizedPosition); + const float lowerFrequency = frequencyFromScale(clamp01(lowerEdgeNormalized)); + const float upperFrequency = frequencyFromScale(clamp01(upperEdgeNormalized)); + + rowCenterFrequencies_[row] = centerFrequency; + rowCenterBins_[row] = std::clamp(centerFrequency / binWidth, 0.0f, numBins - 1.0f); + rowBandStartBins_[row] = std::clamp(std::min(lowerFrequency, upperFrequency) / binWidth, 0.0f, numBins); + rowBandEndBins_[row] = std::clamp(std::max(lowerFrequency, upperFrequency) / binWidth, 0.0f, numBins); + } +} + +float SpectrogramAnalyzer::frequencyFromScale(float normalizedPosition) const { + const float t = clamp01(normalizedPosition); + const float minFrequency = std::max(1.0f, config_.minFrequency); + const float maxFrequency = std::max(minFrequency + 1.0f, config_.maxFrequency); + + if (config_.scaleMode == "linear") { + return minFrequency + (t * (maxFrequency - minFrequency)); + } + + if (config_.scaleMode == "mel") { + const float melMin = hzToMelSlaney(minFrequency); + const float melMax = hzToMelSlaney(maxFrequency); + return melToHzSlaney(melMin + (t * (melMax - melMin))); + } + + const float logMin = std::log10(minFrequency); + const float logMax = std::log10(maxFrequency); + return std::pow(10.0f, logMin + (t * (logMax - logMin))); +} + +float SpectrogramAnalyzer::frequencyToRow(float frequency) const { + const float minFrequency = std::max(1.0f, config_.minFrequency); + const float maxFrequency = std::max(minFrequency + 1.0f, config_.maxFrequency); + const float clampedFrequency = std::clamp(frequency, minFrequency, maxFrequency); + float normalized = 0.0f; + + if (config_.scaleMode == "linear") { + normalized = (clampedFrequency - minFrequency) / std::max(maxFrequency - minFrequency, std::numeric_limits::epsilon()); + } else if (config_.scaleMode == "mel") { + const float melMin = hzToMelSlaney(minFrequency); + const float melMax = hzToMelSlaney(maxFrequency); + normalized = (hzToMelSlaney(clampedFrequency) - melMin) / std::max(melMax - melMin, std::numeric_limits::epsilon()); + } else { + const float logMin = std::log10(minFrequency); + const float logMax = std::log10(maxFrequency); + normalized = (std::log10(clampedFrequency) - logMin) / std::max(logMax - logMin, std::numeric_limits::epsilon()); + } + + const float rowSpan = static_cast(std::max(static_cast(1), config_.rowCount - 1)); + return config_.orientation == "vertical" + ? clamp01(normalized) * rowSpan + : (1.0f - clamp01(normalized)) * rowSpan; +} + +float SpectrogramAnalyzer::applyDisplayTilt(float db, float frequency) const { + const float safeFrequency = std::max(1.0f, frequency); + const float tiltAmount = config_.tiltDbPerOctave * std::log2(safeFrequency / TILT_REFERENCE_HZ); + return db + tiltAmount + DISPLAY_GAIN_DB; +} + +float SpectrogramAnalyzer::displayDbToIntensity(float db) const { + const float range = std::max(1.0e-6f, config_.maxDecibels - config_.minDecibels); + return clamp01((db - config_.minDecibels) / range); +} + +float SpectrogramAnalyzer::sampleDbAtBin(float bin) const { + if (magnitudesDb_.empty()) { + return -200.0f; + } + + const float clampedBin = std::clamp(bin, 0.0f, static_cast(magnitudesDb_.size() - 1)); + const size_t i1 = static_cast(std::floor(clampedBin)); + const float frac = clampedBin - static_cast(i1); + const size_t i0 = i1 > 0 ? i1 - 1 : i1; + const size_t i2 = std::min(magnitudesDb_.size() - 1, i1 + 1); + const size_t i3 = std::min(magnitudesDb_.size() - 1, i1 + 2); + const float m0 = magnitudesDb_[i0]; + const float m1 = magnitudesDb_[i1]; + const float m2 = magnitudesDb_[i2]; + const float m3 = magnitudesDb_[i3]; + const float f2 = frac * frac; + const float f3 = f2 * frac; + + return 0.5f * ( + (2.0f * m1) + + ((-m0 + m2) * frac) + + ((2.0f * m0 - 5.0f * m1 + 4.0f * m2 - m3) * f2) + + ((-m0 + 3.0f * m1 - 3.0f * m2 + m3) * f3) + ); +} + +void SpectrogramAnalyzer::computeStandardSpectrum() { + const size_t rowCount = config_.rowCount; + for (size_t row = 0; row < rowCount; row += 1) { + const float displayDb = applyDisplayTilt(sampleDbAtBin(rowCenterBins_[row]), rowCenterFrequencies_[row]); + standardRaw_[row] = displayDbToIntensity(displayDb); + standardHeat_[row] = normalizeHeatDb(displayDb + HEAT_GAIN_COMPENSATION_DB); + } +} + +void SpectrogramAnalyzer::computeReassignedSpectrum() { + std::fill(reassignedPower_.begin(), reassignedPower_.end(), 0.0f); + if (!haveLastPhase_ || magnitudesLinear_.size() < 3 || config_.rowCount == 0) { + return; + } + + const float sampleRate = std::max(1.0f, config_.sampleRate); + const float binWidth = sampleRate / static_cast(paddedSize_); + const float hopDt = static_cast(resolveHopSize()) / sampleRate; + const float ampThreshold = std::pow(10.0f, config_.minDecibels / 20.0f); + const float twoPi = static_cast(2.0 * M_PI); + + for (size_t bin = 1; bin + 1 < magnitudesLinear_.size(); bin += 1) { + const float mag = magnitudesLinear_[bin]; + if (mag <= ampThreshold) { + continue; + } + if (mag < magnitudesLinear_[bin - 1] || mag < magnitudesLinear_[bin + 1]) { + continue; + } + + const float nominalFrequency = static_cast(bin) * binWidth; + if (nominalFrequency < config_.minFrequency || nominalFrequency > config_.maxFrequency) { + continue; + } + + const float expected = twoPi * nominalFrequency * hopDt; + float correctionHz = wrapPhase(phases_[bin] - lastPhases_[bin] - expected) / (twoPi * hopDt); + correctionHz = std::clamp(correctionHz, -1.5f * binWidth, 1.5f * binWidth); + float reassignedFrequency = nominalFrequency + correctionHz; + + const float leftWeight = magnitudesLinear_[bin - 1]; + const float centerWeight = mag; + const float rightWeight = magnitudesLinear_[bin + 1]; + const float weightSum = leftWeight + centerWeight + rightWeight; + if (weightSum > std::numeric_limits::epsilon()) { + const float centroidFrequency = ( + (static_cast(bin - 1) * binWidth * leftWeight) + + (nominalFrequency * centerWeight) + + (static_cast(bin + 1) * binWidth * rightWeight) + ) / weightSum; + reassignedFrequency = 0.5f * reassignedFrequency + 0.5f * centroidFrequency; + } + + reassignedFrequency = std::clamp(reassignedFrequency, config_.minFrequency, config_.maxFrequency); + const float rowF = frequencyToRow(reassignedFrequency); + const size_t row0 = static_cast(std::floor(std::clamp(rowF, 0.0f, static_cast(config_.rowCount - 1)))); + const float frac = rowF - static_cast(row0); + const float power = mag * mag; + + reassignedPower_[row0] += power * (1.0f - frac); + if (row0 + 1 < config_.rowCount) { + reassignedPower_[row0 + 1] += power * frac; + } + } +} + +SpectrogramAnalyzer::ClarityProfile SpectrogramAnalyzer::clarityProfile(const std::string& mode) { + if (mode == "classic") { + return {1.4f, 0.0f, 3.0f}; + } + if (mode == "sharp") { + return {1.5f, 2.5f, 3.0f}; + } + return {2.0f, 5.0f, 2.0f}; +} + +void SpectrogramAnalyzer::blendAndShapeColumn(std::vector& display, std::vector& heat) { + const size_t rowCount = config_.rowCount; + const ClarityProfile clarity = clarityProfile(config_.clarityMode); + const float standardWeight = config_.clarityMode == "classic" ? 0.8f : (config_.clarityMode == "sharp" ? 0.6f : 0.45f); + const float reassignedWeight = config_.clarityMode == "classic" ? 0.85f : 1.0f; + + for (size_t row = 0; row < rowCount; row += 1) { + float reassignedRaw = 0.0f; + float reassignedHeat = 0.0f; + if (reassignedPower_[row] > 0.0f) { + const float reassignedMag = std::sqrt(reassignedPower_[row]); + const float reassignedDb = 20.0f * std::log10(std::max(reassignedMag, 1.0e-10f)); + const float displayDb = applyDisplayTilt(reassignedDb, rowCenterFrequencies_[row]); + reassignedRaw = displayDbToIntensity(displayDb); + reassignedHeat = normalizeHeatDb(displayDb + HEAT_GAIN_COMPENSATION_DB); + } + + blendedRaw_[row] = std::max(standardRaw_[row] * standardWeight, reassignedRaw * reassignedWeight); + blendedHeat_[row] = std::max(standardHeat_[row] * standardWeight, reassignedHeat * reassignedWeight); + } + + if (clarity.sharpness > 0.0f) { + const std::vector peakSource = blendedRaw_; + const float mainlobePaddedBins = 4.0f * static_cast(FFT_PAD_FACTOR); + const float detailPreserve = config_.clarityMode == "sharp" ? 0.18f : 0.14f; + + for (size_t row = 0; row < rowCount; row += 1) { + const float bandWidthPerRow = std::max(0.1f, rowBandEndBins_[row] - rowBandStartBins_[row]); + const float mainlobePixels = mainlobePaddedBins / bandWidthPerRow; + const int halfWindow = std::max(2, std::min(50, static_cast(std::lround(mainlobePixels * 0.5f)))); + const float scaleFactor = std::max(1.0f, mainlobePixels / clarity.lineWidth); + const float effectiveSharpness = clarity.sharpness * scaleFactor; + + float localMax = peakSource[row]; + for (int offset = 1; offset <= halfWindow; offset += 1) { + if (row >= static_cast(offset)) { + localMax = std::max(localMax, peakSource[row - static_cast(offset)]); + } + if (row + static_cast(offset) < rowCount) { + localMax = std::max(localMax, peakSource[row + static_cast(offset)]); + } + } + + if (localMax > 1.0e-6f) { + const float ratio = blendedRaw_[row] / localMax; + const float suppression = std::pow(clamp01(ratio), effectiveSharpness); + const float rawBefore = blendedRaw_[row]; + const float heatBefore = blendedHeat_[row]; + blendedRaw_[row] = std::max(rawBefore * suppression, rawBefore * detailPreserve); + blendedHeat_[row] = std::max(heatBefore * suppression, heatBefore * detailPreserve); + } + } + } + + const float effectiveGamma = clarity.gamma * config_.contrast; + const size_t offset = display.size(); + display.resize(offset + rowCount); + heat.resize(offset + rowCount); + for (size_t row = 0; row < rowCount; row += 1) { + display[offset + row] = std::pow(clamp01(blendedRaw_[row]), effectiveGamma); + heat[offset + row] = clamp01(blendedHeat_[row]); + } +} + +void SpectrogramAnalyzer::processFrame(std::vector& display, std::vector& heat) { + std::fill(windowedInput_.begin(), windowedInput_.end(), 0.0f); + for (size_t index = 0; index < fftSize_; index += 1) { + windowedInput_[index] = frameBuffer_[index] * window_[index]; + } + + fft_->forward(windowedInput_.data(), fftOutput_.data()); + + const size_t numBins = paddedSize_ / 2; + const float scale = 2.0f / static_cast(fftSize_); + for (size_t bin = 0; bin < numBins; bin += 1) { + const float re = fftOutput_[bin].real(); + const float im = fftOutput_[bin].imag(); + const float magnitude = std::sqrt((re * re) + (im * im)) * scale; + magnitudesLinear_[bin] = magnitude; + magnitudesDb_[bin] = 20.0f * std::log10(std::max(magnitude, 1.0e-10f)); + phases_[bin] = std::atan2(im, re); + } + + computeStandardSpectrum(); + computeReassignedSpectrum(); + blendAndShapeColumn(display, heat); + + lastPhases_ = phases_; + haveLastPhase_ = true; +} + +SpectrogramProcessResult SpectrogramAnalyzer::process(const float* samples, size_t length) { + SpectrogramProcessResult result; + result.rowCount = config_.rowCount; + if (!samples || length == 0 || fftSize_ == 0 || config_.rowCount == 0) { + return result; + } + + const size_t hopSize = resolveHopSize(); + const size_t overlapSamples = fftSize_ - hopSize; + + for (size_t index = 0; index < length; index += 1) { + frameBuffer_[frameFill_] = samples[index]; + frameFill_ += 1; + + if (frameFill_ >= fftSize_) { + processFrame(result.display, result.heat); + result.columnCount += 1; + + if (overlapSamples > 0) { + std::memmove(frameBuffer_.data(), frameBuffer_.data() + hopSize, overlapSamples * sizeof(float)); + } + frameFill_ = overlapSamples; + } + } + + return result; +} + +} // namespace Visualizer diff --git a/native/src/spectrogram.h b/native/src/spectrogram.h new file mode 100644 index 0000000..48605f7 --- /dev/null +++ b/native/src/spectrogram.h @@ -0,0 +1,90 @@ +#pragma once + +#include "dsp_utils.h" +#include +#include +#include +#include + +namespace Visualizer { + +struct SpectrogramConfig { + size_t fftSize = 4096; + float sampleRate = 48000.0f; + size_t rowCount = 1; + float minFrequency = 20.0f; + float maxFrequency = 20000.0f; + float minDecibels = -90.0f; + float maxDecibels = -12.0f; + float scrollSpeed = 2.0f; + float contrast = 1.0f; + float tiltDbPerOctave = 4.0f; + std::string clarityMode = "sharper"; + std::string scaleMode = "log"; + std::string orientation = "horizontal"; +}; + +struct SpectrogramProcessResult { + std::vector display; + std::vector heat; + size_t columnCount = 0; + size_t rowCount = 0; +}; + +class SpectrogramAnalyzer { +public: + SpectrogramAnalyzer(); + + void configure(const SpectrogramConfig& config); + SpectrogramProcessResult process(const float* samples, size_t length); + void reset(); + +private: + struct ClarityProfile { + float gamma; + float sharpness; + float lineWidth; + }; + + SpectrogramConfig config_; + size_t fftSize_; + size_t paddedSize_; + size_t frameFill_; + bool haveLastPhase_; + + std::unique_ptr fft_; + std::vector frameBuffer_; + std::vector window_; + std::vector windowedInput_; + std::vector> fftOutput_; + std::vector magnitudesDb_; + std::vector magnitudesLinear_; + std::vector phases_; + std::vector lastPhases_; + + std::vector rowCenterBins_; + std::vector rowBandStartBins_; + std::vector rowBandEndBins_; + std::vector rowCenterFrequencies_; + std::vector standardRaw_; + std::vector standardHeat_; + std::vector reassignedPower_; + std::vector blendedRaw_; + std::vector blendedHeat_; + + void configureFft(size_t fftSize); + void rebuildFrequencyMapping(); + void processFrame(std::vector& display, std::vector& heat); + void computeStandardSpectrum(); + void computeReassignedSpectrum(); + void blendAndShapeColumn(std::vector& display, std::vector& heat); + size_t resolveHopSize() const; + float sampleDbAtBin(float bin) const; + float frequencyFromScale(float normalizedPosition) const; + float frequencyToRow(float frequency) const; + float applyDisplayTilt(float db, float frequency) const; + float displayDbToIntensity(float db) const; + static ClarityProfile clarityProfile(const std::string& mode); +}; + +} // namespace Visualizer diff --git a/src/preload/index.ts b/src/preload/index.ts index e93106d..9a3fb98 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -245,6 +245,7 @@ const visualizerAPI = nativeAddonModule ? { oscilloscope: nativeAddonModule.oscilloscope, spectrum: nativeAddonModule.spectrum, + spectrogram: nativeAddonModule.spectrogram, vectorscope: nativeAddonModule.vectorscope, } : null diff --git a/src/renderer/audio/native/index.ts b/src/renderer/audio/native/index.ts index e877bda..99bf9b1 100644 --- a/src/renderer/audio/native/index.ts +++ b/src/renderer/audio/native/index.ts @@ -1,7 +1,14 @@ // Native visualizer DSP module loader // This loads the native C++ addon for high-performance audio visualization -import type { VisualizerDSP, OscilloscopeResult, VectorscopeResult, VectorscopePointsResult } from './visualizer-dsp' +import type { + VisualizerDSP, + OscilloscopeResult, + SpectrogramNativeOptions, + SpectrogramNativeResult, + VectorscopeResult, + VectorscopePointsResult, +} from './visualizer-dsp' let nativeModule: VisualizerDSP | null = null let loadError: Error | null = null @@ -153,6 +160,32 @@ export const spectrum = { } } +export interface SpectrogramNativeAnalyzer { + configure(options: SpectrogramNativeOptions): void + process(audioData: Float32Array): SpectrogramNativeResult | null + reset(): void + isAvailable?: () => boolean +} + +export const spectrogram: SpectrogramNativeAnalyzer = { + isAvailable: (): boolean => { + return Boolean(nativeModule?.spectrogram) + }, + + configure: (options: SpectrogramNativeOptions): void => { + nativeModule?.spectrogram?.configure(options) + }, + + process: (audioData: Float32Array): SpectrogramNativeResult | null => { + if (!nativeModule?.spectrogram) return null + return nativeModule.spectrogram.process(audioData) + }, + + reset: (): void => { + nativeModule?.spectrogram?.reset() + }, +} + export const vectorscope = { setSampleRate: (sampleRate: number): void => { nativeModule?.vectorscope.setSampleRate(sampleRate) @@ -196,4 +229,10 @@ export const vectorscope = { } } -export type { OscilloscopeResult, VectorscopeResult, VectorscopePointsResult } +export type { + OscilloscopeResult, + SpectrogramNativeOptions, + SpectrogramNativeResult, + VectorscopeResult, + VectorscopePointsResult, +} diff --git a/src/renderer/audio/native/visualizer-dsp.d.ts b/src/renderer/audio/native/visualizer-dsp.d.ts index 0f7cb50..72ff7aa 100644 --- a/src/renderer/audio/native/visualizer-dsp.d.ts +++ b/src/renderer/audio/native/visualizer-dsp.d.ts @@ -18,6 +18,29 @@ export interface VectorscopePointsResult { count: number; } +export interface SpectrogramNativeOptions { + fftSize: number; + sampleRate: number; + rowCount: number; + minFrequency: number; + maxFrequency: number; + minDecibels: number; + maxDecibels: number; + scrollSpeed: number; + contrast: number; + tiltDbPerOctave: number; + clarityMode: string; + scaleMode: string; + orientation: string; +} + +export interface SpectrogramNativeResult { + display: Float32Array; + heat: Float32Array; + columnCount: number; + rowCount: number; +} + // Circular buffer size (must match native code) export const OSCILLOSCOPE_BUFFER_SIZE = 32768; @@ -62,6 +85,12 @@ export interface SpectrumModule { reset(): void; } +export interface SpectrogramModule { + configure(options: SpectrogramNativeOptions): void; + process(audioData: Float32Array): SpectrogramNativeResult; + reset(): void; +} + export interface VectorscopeModule { setSampleRate(sampleRate: number): void; pushSamples(leftChannel: Float32Array, rightChannel: Float32Array): void; @@ -76,6 +105,7 @@ export interface VectorscopeModule { export interface VisualizerDSP { oscilloscope: OscilloscopeModule; spectrum: SpectrumModule; + spectrogram: SpectrogramModule; vectorscope: VectorscopeModule; } diff --git a/src/renderer/components/ScopeModule.tsx b/src/renderer/components/ScopeModule.tsx index 5728a85..2dc358c 100644 --- a/src/renderer/components/ScopeModule.tsx +++ b/src/renderer/components/ScopeModule.tsx @@ -247,6 +247,7 @@ export function scopeSettingsToOptions( heatColors: t.heatColors, backgroundColor: t.background, fftSize: s.fftSize, + tiltDbPerOctave: s.tiltDbPerOctave, scrollSpeed: s.scrollSpeed, contrast: s.contrast, clarityMode: s.clarityMode, diff --git a/src/renderer/components/ScopeSettingsSection.tsx b/src/renderer/components/ScopeSettingsSection.tsx index 8e1ae7d..c027982 100644 --- a/src/renderer/components/ScopeSettingsSection.tsx +++ b/src/renderer/components/ScopeSettingsSection.tsx @@ -4,8 +4,11 @@ import { SCOPE_LABELS } from '../../types/scope' import type { ScopeSettings } from '../../types/settings' import { MAX_SPECTROGRAM_CONTRAST, + MAX_SPECTROGRAM_TILT_DB_PER_OCTAVE, MIN_SPECTROGRAM_CONTRAST, + MIN_SPECTROGRAM_TILT_DB_PER_OCTAVE, SPECTROGRAM_CONTRAST_STEP, + SPECTROGRAM_TILT_STEP, } from '../../types/spectrogram' import { DEFAULT_VU_REFERENCE_DBFS, @@ -490,7 +493,7 @@ export default function ScopeSettingsSection({ value={current.fftSize} onChange={(value) => onUpdate('spectrogram', { fftSize: Number(value) })} > - {[512, 1024, 2048, 4096].map((option) => ( + {[512, 1024, 2048, 4096, 8192].map((option) => ( @@ -556,6 +559,17 @@ export default function ScopeSettingsSection({ fullWidth={false} onChange={(value) => onUpdate('spectrogram', { contrast: value })} /> + + onUpdate('spectrogram', { tiltDbPerOctave: value })} + /> ) })()} diff --git a/src/renderer/visualizers/Spectrogram.ts b/src/renderer/visualizers/Spectrogram.ts index 0d5bac4..76caed8 100644 --- a/src/renderer/visualizers/Spectrogram.ts +++ b/src/renderer/visualizers/Spectrogram.ts @@ -1,4 +1,10 @@ import { audioRouter } from '../audio/AudioRouter' +import { + spectrogram as nativeSpectrogram, + type SpectrogramNativeAnalyzer, + type SpectrogramNativeOptions, + type SpectrogramNativeResult, +} from '../audio/native' import { parseColorToRgba, resolveColorToRgb, type RgbaColor } from '../utils/color' import { defaultVisualizerSessionSource, type VisualizerSessionSource } from './dataSource' import { FrameScheduler } from './frameScheduler' @@ -9,8 +15,10 @@ import { DEFAULT_SPECTROGRAM_ORIENTATION, DEFAULT_SPECTROGRAM_SCALE_MODE, DEFAULT_SPECTROGRAM_SCROLL_SPEED, + DEFAULT_SPECTROGRAM_TILT_DB_PER_OCTAVE, clampSpectrogramContrast, clampSpectrogramScrollSpeed, + clampSpectrogramTiltDbPerOctave, isSpectrogramClarityMode, isSpectrogramOrientation, isSpectrogramScaleMode, @@ -30,6 +38,7 @@ export interface SpectrogramDataSource extends VisualizerSessionSource { export interface SpectrogramOptions { fftSize?: number + tiltDbPerOctave?: number minFrequency?: number maxFrequency?: number minDecibels?: number @@ -45,23 +54,14 @@ export interface SpectrogramOptions { backgroundColor?: string dataSource?: SpectrogramDataSource frameScheduler?: FrameScheduler + nativeAnalyzer?: SpectrogramNativeAnalyzer | null } -type ResolvedSpectrogramOptions = Required> - -interface SpectrogramClarityProfile { - gamma: number // contrast curve exponent - sharpness: number // local peak suppression exponent (0 = off, higher = thinner lines) - lineWidth: number // target visible line width in pixels (smaller = tighter peaks) -} - -const SPECTROGRAM_DISPLAY_GAIN_DB = 2 -const SPECTROGRAM_HEAT_GAIN_COMPENSATION_DB = 6 -const SPECTROGRAM_TILT_DB_PER_OCTAVE = 4 -const SPECTROGRAM_TILT_REFERENCE_HZ = 1000 +type ResolvedSpectrogramOptions = Required> const defaultOptions: ResolvedSpectrogramOptions = { fftSize: 4096, + tiltDbPerOctave: DEFAULT_SPECTROGRAM_TILT_DB_PER_OCTAVE, minFrequency: 20, maxFrequency: 20000, minDecibels: -90, @@ -82,17 +82,6 @@ const defaultSpectrogramDataSource: SpectrogramDataSource = { ...defaultVisualizerSessionSource, } -function getClarityProfile(mode: SpectrogramClarityMode): SpectrogramClarityProfile { - switch (mode) { - case 'classic': - return { gamma: 1.4, sharpness: 0, lineWidth: 3 } - case 'sharp': - return { gamma: 1.5, sharpness: 2.5, lineWidth: 3 } - case 'sharper': - return { gamma: 2.0, sharpness: 5.0, lineWidth: 2 } - } -} - function resolveClarityMode(value: unknown, fallback: SpectrogramClarityMode): SpectrogramClarityMode { return isSpectrogramClarityMode(value) ? value : fallback } @@ -108,6 +97,9 @@ function resolveOrientation(value: unknown, fallback: SpectrogramOrientation): S function resolveOptions(base: ResolvedSpectrogramOptions, overrides: Partial): ResolvedSpectrogramOptions { return { fftSize: typeof overrides.fftSize === 'number' ? overrides.fftSize : base.fftSize, + tiltDbPerOctave: overrides.tiltDbPerOctave === undefined + ? base.tiltDbPerOctave + : clampSpectrogramTiltDbPerOctave(overrides.tiltDbPerOctave), minFrequency: typeof overrides.minFrequency === 'number' ? overrides.minFrequency : base.minFrequency, maxFrequency: typeof overrides.maxFrequency === 'number' ? overrides.maxFrequency : base.maxFrequency, minDecibels: typeof overrides.minDecibels === 'number' ? overrides.minDecibels : base.minDecibels, @@ -128,118 +120,6 @@ function resolveOptions(base: ResolvedSpectrogramOptions, overrides: Partial> 1 - while (j & bit) { - j ^= bit - bit >>= 1 - } - j ^= bit - - if (i < j) { - let tmp = re[i] - re[i] = re[j] - re[j] = tmp - tmp = im[i] - im[i] = im[j] - im[j] = tmp - } - } - - for (let len = 2; len <= n; len <<= 1) { - const halfLen = len >> 1 - const angle = -2 * Math.PI / len - const wRe = Math.cos(angle) - const wIm = Math.sin(angle) - - for (let i = 0; i < n; i += len) { - let curRe = 1 - let curIm = 0 - - for (let k = 0; k < halfLen; k += 1) { - const evenIdx = i + k - const oddIdx = i + k + halfLen - - const tRe = curRe * re[oddIdx] - curIm * im[oddIdx] - const tIm = curRe * im[oddIdx] + curIm * re[oddIdx] - - re[oddIdx] = re[evenIdx] - tRe - im[oddIdx] = im[evenIdx] - tIm - re[evenIdx] += tRe - im[evenIdx] += tIm - - const nextRe = curRe * wRe - curIm * wIm - curIm = curRe * wIm + curIm * wRe - curRe = nextRe - } - } - } -} - -const hannWindowCache = new Map() - -function getHannWindow(size: number): Float32Array { - let window = hannWindowCache.get(size) - if (window) return window - - window = new Float32Array(size) - for (let i = 0; i < size; i += 1) { - window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (size - 1))) - } - hannWindowCache.set(size, window) - return window -} - type ColorStop = { at: number color: [number, number, number, number] @@ -340,42 +220,22 @@ function buildHeatLUT(colors: [string, string, string]): Uint8ClampedArray { return lut } -// Zero-pad FFT for finer frequency resolution (visual interpolation) -const FFT_PAD_FACTOR = 4 - export class Spectrogram { private canvas: HTMLCanvasElement private ctx: CanvasRenderingContext2D private options: ResolvedSpectrogramOptions private dataSource: SpectrogramDataSource + private nativeAnalyzer: SpectrogramNativeAnalyzer | null private frameLoop: VisualizerFrameLoop - private fftRe: Float32Array - private fftIm: Float32Array - private fftMagnitudes: Float32Array - private sampleBuffer: Float32Array - private sampleBufferPos = 0 - private waterfallCanvas: HTMLCanvasElement private waterfallCtx: CanvasRenderingContext2D - private rowCenterBins = new Float32Array(0) - private rowBandStartBins = new Float32Array(0) - private rowBandEndBins = new Float32Array(0) private columnValues = new Float32Array(0) - private rawColumnValues = new Float32Array(0) - private heatColumnValues = new Float32Array(0) private columnImageData: ImageData | null = null private heatLut: Uint8ClampedArray - private lastWidth = 0 - private lastHeight = 0 - private lastFftSize = 0 - private lastSampleRate = 0 - private lastMinFrequency = 0 - private lastMaxFrequency = 0 - private lastScaleMode: SpectrogramScaleMode | null = null - private lastOrientation: SpectrogramOrientation | null = null + private lastNativeConfigKey: string | null = null private unsubscribeSessionChange: (() => void) | null = null constructor(canvas: HTMLCanvasElement, options: SpectrogramOptions = {}) { @@ -384,9 +244,10 @@ export class Spectrogram { if (!ctx) throw new Error('Could not get 2D context') this.ctx = ctx - const { dataSource, frameScheduler, ...optionOverrides } = options + const { dataSource, frameScheduler, nativeAnalyzer, ...optionOverrides } = options this.options = resolveOptions(defaultOptions, optionOverrides) this.dataSource = dataSource ?? defaultSpectrogramDataSource + this.nativeAnalyzer = nativeAnalyzer === undefined ? nativeSpectrogram : nativeAnalyzer this.heatLut = buildHeatLUT(this.options.heatColors) this.frameLoop = new VisualizerFrameLoop({ frameScheduler, @@ -394,13 +255,6 @@ export class Spectrogram { onFrame: this.drawFrame, }) - const windowSize = this.options.fftSize - const paddedSize = windowSize * FFT_PAD_FACTOR - this.fftRe = new Float32Array(paddedSize) - this.fftIm = new Float32Array(paddedSize) - this.fftMagnitudes = new Float32Array(paddedSize / 2) - this.sampleBuffer = new Float32Array(windowSize) - this.waterfallCanvas = document.createElement('canvas') this.waterfallCanvas.width = canvas.width this.waterfallCanvas.height = canvas.height @@ -424,36 +278,36 @@ export class Spectrogram { } private resetDisplay(): void { - this.sampleBufferPos = 0 + this.nativeAnalyzer?.reset() + this.lastNativeConfigKey = null this.waterfallCtx.clearRect(0, 0, this.waterfallCanvas.width, this.waterfallCanvas.height) this.invalidate() } setOptions(options: Partial): void { - const { dataSource, frameScheduler: _frameScheduler, ...optionUpdates } = options + const { dataSource, frameScheduler: _frameScheduler, nativeAnalyzer, ...optionUpdates } = options const previousOptions = this.options this.options = resolveOptions(previousOptions, optionUpdates) this.heatLut = buildHeatLUT(this.options.heatColors) + if (nativeAnalyzer !== undefined && nativeAnalyzer !== this.nativeAnalyzer) { + this.nativeAnalyzer = nativeAnalyzer + this.lastNativeConfigKey = null + this.resetDisplay() + } + if (dataSource && dataSource !== this.dataSource) { this.dataSource = dataSource this.subscribeToSessionChanges() this.resetDisplay() } - if (this.options.fftSize !== previousOptions.fftSize) { - const windowSize = this.options.fftSize - const paddedSize = windowSize * FFT_PAD_FACTOR - this.fftRe = new Float32Array(paddedSize) - this.fftIm = new Float32Array(paddedSize) - this.fftMagnitudes = new Float32Array(paddedSize / 2) - this.sampleBuffer = new Float32Array(windowSize) - this.sampleBufferPos = 0 - this.lastFftSize = 0 - this.resetDisplay() - } else if ( - this.options.scaleMode !== previousOptions.scaleMode + if ( + this.options.fftSize !== previousOptions.fftSize + || this.options.scaleMode !== previousOptions.scaleMode || this.options.orientation !== previousOptions.orientation + || this.options.minFrequency !== previousOptions.minFrequency + || this.options.maxFrequency !== previousOptions.maxFrequency ) { this.resetDisplay() } @@ -474,8 +328,6 @@ export class Spectrogram { } resize(): void { - this.lastWidth = 0 - this.lastHeight = 0 this.invalidate() } @@ -497,8 +349,6 @@ export class Spectrogram { } this.columnValues = new Float32Array(pixelCount) - this.rawColumnValues = new Float32Array(pixelCount) - this.heatColumnValues = new Float32Array(pixelCount) this.columnImageData = new ImageData(imageWidth, imageHeight) } @@ -527,120 +377,120 @@ export class Spectrogram { } } - private ensureBandMapping(): void { - const { canvas, options } = this - const width = canvas.width - const height = canvas.height - const fftSize = options.fftSize - const frequencyPixelCount = this.getFrequencyPixelCount(width, height) - const sampleRate = Math.max(1, this.dataSource.getSampleRate()) - const nyquist = sampleRate / 2 - const minFrequency = Math.max(1, Math.min(options.minFrequency, nyquist)) - const maxFrequency = Math.max(minFrequency + 1, Math.min(options.maxFrequency, nyquist)) - - if ( - width === this.lastWidth - && height === this.lastHeight - && fftSize === this.lastFftSize - && sampleRate === this.lastSampleRate - && minFrequency === this.lastMinFrequency - && maxFrequency === this.lastMaxFrequency - && options.scaleMode === this.lastScaleMode - && options.orientation === this.lastOrientation - ) { - return + private isNativeAnalyzerReady(): boolean { + if (!this.nativeAnalyzer) { + return false } - - this.lastWidth = width - this.lastHeight = height - this.lastFftSize = fftSize - this.lastSampleRate = sampleRate - this.lastMinFrequency = minFrequency - this.lastMaxFrequency = maxFrequency - this.lastScaleMode = options.scaleMode - this.lastOrientation = options.orientation - - const numBins = (fftSize * FFT_PAD_FACTOR) / 2 - const rowSpan = Math.max(1, frequencyPixelCount - 1) - const binWidth = nyquist / numBins - - this.rowCenterBins = new Float32Array(frequencyPixelCount) - this.rowBandStartBins = new Float32Array(frequencyPixelCount) - this.rowBandEndBins = new Float32Array(frequencyPixelCount) - for (let row = 0; row < frequencyPixelCount; row += 1) { - const normalizedPosition = options.orientation === 'vertical' - ? row / rowSpan - : 1 - (row / rowSpan) - const centerFrequency = frequencyFromScale( - options.scaleMode, - minFrequency, - maxFrequency, - normalizedPosition - ) - const upperEdgeNormalized = options.orientation === 'vertical' - ? row === frequencyPixelCount - 1 - ? 1 - : (row + 0.5) / rowSpan - : row === 0 - ? 1 - : 1 - ((row - 0.5) / rowSpan) - const lowerEdgeNormalized = options.orientation === 'vertical' - ? row === 0 - ? 0 - : (row - 0.5) / rowSpan - : row === height - 1 - ? 0 - : 1 - ((row + 0.5) / rowSpan) - const upperEdgeFrequency = frequencyFromScale( - options.scaleMode, - minFrequency, - maxFrequency, - upperEdgeNormalized - ) - const lowerEdgeFrequency = frequencyFromScale( - options.scaleMode, - minFrequency, - maxFrequency, - lowerEdgeNormalized - ) - - this.rowCenterBins[row] = Math.max(0, Math.min(numBins - 1, centerFrequency / binWidth)) - this.rowBandStartBins[row] = Math.max(0, Math.min(numBins, lowerEdgeFrequency / binWidth)) - this.rowBandEndBins[row] = Math.max(0, Math.min(numBins, upperEdgeFrequency / binWidth)) - } - - this.ensureColumnBuffers(frequencyPixelCount) + return this.nativeAnalyzer.isAvailable?.() ?? true } - private processFFT(samples: Float32Array): Float32Array { - const windowSize = samples.length - const paddedSize = windowSize * FFT_PAD_FACTOR - const window = getHannWindow(windowSize) - - // Apply window to audio samples - for (let index = 0; index < windowSize; index += 1) { - this.fftRe[index] = samples[index] * window[index] + private buildNativeConfig(width: number, height: number): SpectrogramNativeOptions { + return { + fftSize: this.options.fftSize, + sampleRate: Math.max(1, this.dataSource.getSampleRate()), + rowCount: this.getFrequencyPixelCount(width, height), + minFrequency: this.options.minFrequency, + maxFrequency: this.options.maxFrequency, + minDecibels: this.options.minDecibels, + maxDecibels: this.options.maxDecibels, + scrollSpeed: this.options.scrollSpeed, + contrast: this.options.contrast, + tiltDbPerOctave: this.options.tiltDbPerOctave, + clarityMode: this.options.clarityMode, + scaleMode: this.options.scaleMode, + orientation: this.options.orientation, } - // Zero-pad the rest for finer frequency interpolation - for (let index = windowSize; index < paddedSize; index += 1) { - this.fftRe[index] = 0 - } - this.fftIm.fill(0) + } - fft(this.fftRe, this.fftIm) - - const numBins = paddedSize / 2 - const magnitudes = this.fftMagnitudes - const scale = 2 / windowSize // normalize by window size, not padded size - - for (let index = 0; index < numBins; index += 1) { - const re = this.fftRe[index] - const im = this.fftIm[index] - const magnitude = Math.sqrt((re * re) + (im * im)) * scale - magnitudes[index] = 20 * Math.log10(Math.max(magnitude, 1e-10)) + private configureNativeAnalyzer(config: SpectrogramNativeOptions): boolean { + if (!this.nativeAnalyzer || config.rowCount <= 0) { + return false } - return magnitudes + const key = [ + config.fftSize, + config.sampleRate, + config.rowCount, + config.minFrequency, + config.maxFrequency, + config.minDecibels, + config.maxDecibels, + config.scrollSpeed, + config.contrast, + config.tiltDbPerOctave, + config.clarityMode, + config.scaleMode, + config.orientation, + ].join('|') + + if (key !== this.lastNativeConfigKey) { + this.nativeAnalyzer.configure(config) + this.lastNativeConfigKey = key + } + + return true + } + + private isValidNativeResult(result: SpectrogramNativeResult | null, rowCount: number): result is SpectrogramNativeResult { + if (!result) { + return false + } + + const columnCount = Math.max(0, Math.floor(result.columnCount)) + if (result.rowCount !== rowCount || columnCount !== result.columnCount) { + return false + } + + const expectedLength = rowCount * columnCount + return result.display.length >= expectedLength && result.heat.length >= expectedLength + } + + private tryDrawNativeColumns(pendingSamples: Float32Array[], width: number, height: number): boolean { + if (!this.isNativeAnalyzerReady()) { + return false + } + + const config = this.buildNativeConfig(width, height) + if (config.rowCount <= 0) { + return false + } + + this.ensureColumnBuffers(config.rowCount) + + const results: SpectrogramNativeResult[] = [] + try { + if (!this.configureNativeAnalyzer(config)) { + return false + } + + for (const chunk of pendingSamples) { + const result = this.nativeAnalyzer?.process(chunk) ?? null + if (!this.isValidNativeResult(result, config.rowCount)) { + return false + } + if (result.columnCount > 0) { + results.push(result) + } + } + } catch (error) { + console.warn('Spectrogram: native analyzer failed', error) + this.nativeAnalyzer?.reset() + this.lastNativeConfigKey = null + return false + } + + for (const result of results) { + for (let column = 0; column < result.columnCount; column += 1) { + const start = column * result.rowCount + const end = start + result.rowCount + this.shiftAndPaintColumn( + result.display.subarray(start, end), + result.heat.subarray(start, end), + ) + } + } + + return true } private paintColumnImage(values: Float32Array, heatValues: Float32Array = values): void { @@ -671,103 +521,13 @@ export class Spectrogram { } } - private drawColumn(magnitudes: Float32Array): Float32Array { - const width = this.waterfallCanvas.width - const height = this.waterfallCanvas.height - const frequencyPixelCount = this.getFrequencyPixelCount(width, height) - if (frequencyPixelCount <= 0) return this.columnValues - - this.ensureColumnBuffers(frequencyPixelCount) - const values = this.columnValues - const raw = this.rawColumnValues - const heat = this.heatColumnValues - const numBins = magnitudes.length - - const clarity = getClarityProfile(this.options.clarityMode) - const minDecibels = this.options.minDecibels - const dbRange = Math.max(1e-6, this.options.maxDecibels - minDecibels) - - // Compute bin width for frequency-based tilt - const sampleRate = Math.max(1, this.dataSource.getSampleRate()) - const binWidth = (sampleRate / 2) / numBins - - // Pass 1: sub-bin interpolation + tilt/gain -> raw normalized values (no gamma yet) - for (let row = 0; row < frequencyPixelCount; row += 1) { - const centerBin = this.rowCenterBins[row] - - // 4-point Catmull–Rom cubic interpolation in dB — captures the Hann - // mainlobe curvature so a tone between bins lands on a single row - // instead of smearing linearly across two. - const i1 = Math.floor(centerBin) - const frac = centerBin - i1 - const i0 = Math.max(0, i1 - 1) - const i2 = Math.min(numBins - 1, i1 + 1) - const i3 = Math.min(numBins - 1, i1 + 2) - const m0 = magnitudes[i0] - const m1 = magnitudes[i1] - const m2 = magnitudes[i2] - const m3 = magnitudes[i3] - const f2 = frac * frac - const f3 = f2 * frac - const db = 0.5 * ( - (2 * m1) - + (-m0 + m2) * frac - + (2 * m0 - 5 * m1 + 4 * m2 - m3) * f2 - + (-m0 + 3 * m1 - 3 * m2 + m3) * f3 - ) - - // Frequency-based tilt — dB per octave from reference, scale-mode independent - const centerFreq = Math.max(1, centerBin * binWidth) - const tiltAmount = SPECTROGRAM_TILT_DB_PER_OCTAVE * Math.log2(centerFreq / SPECTROGRAM_TILT_REFERENCE_HZ) - const displayDb = db + tiltAmount + SPECTROGRAM_DISPLAY_GAIN_DB - raw[row] = clamp01((displayDb - minDecibels) / dbRange) - heat[row] = normalizeHeatDb(displayDb + SPECTROGRAM_HEAT_GAIN_COMPENSATION_DB) + private paintWaterfall(width: number, height: number): void { + this.ctx.clearRect(0, 0, width, height) + if (this.options.backgroundColor !== 'transparent') { + this.ctx.fillStyle = this.options.backgroundColor + this.ctx.fillRect(0, 0, width, height) } - - // Pass 2: local peak suppression — thin spectral lines for sharp/sharper modes - const sharpness = clarity.sharpness - if (sharpness > 0) { - // Hann mainlobe = 4 original bins = 4 * FFT_PAD_FACTOR padded bins - const mainlobePaddedBins = 4 * FFT_PAD_FACTOR - // Target visual line width in pixels — suppression scales to achieve this - const TARGET_LINE_WIDTH = clarity.lineWidth - - for (let row = 0; row < frequencyPixelCount; row += 1) { - // Adaptive window: mainlobe width in pixel rows at this frequency - const bandWidthPerRow = Math.max(0.1, this.rowBandEndBins[row] - this.rowBandStartBins[row]) - const mainlobePixels = mainlobePaddedBins / bandWidthPerRow - const halfWin = Math.max(2, Math.min(50, Math.round(mainlobePixels / 2))) - - // Scale suppression by how wide the mainlobe is vs target width - // At low freqs (mainlobe=26px, target=3px): 8.7x stronger suppression - // At high freqs (mainlobe=2px, target=3px): 1x base suppression - const scaleFactor = Math.max(1, mainlobePixels / TARGET_LINE_WIDTH) - const effectiveSharpness = sharpness * scaleFactor - - // Find local peak in neighborhood - let localMax = raw[row] - for (let d = 1; d <= halfWin; d += 1) { - if (row - d >= 0 && raw[row - d] > localMax) localMax = raw[row - d] - if (row + d < frequencyPixelCount && raw[row + d] > localMax) localMax = raw[row + d] - } - - // Suppress off-peak values: peak stays bright, slopes get crushed - if (localMax > 1e-6) { - const ratio = raw[row] / localMax - const suppression = Math.pow(ratio, effectiveSharpness) - raw[row] *= suppression - heat[row] *= suppression - } - } - } - - // Pass 3: apply gamma scaled by user contrast (1.0 = profile default) - const effectiveGamma = clarity.gamma * this.options.contrast - for (let row = 0; row < frequencyPixelCount; row += 1) { - values[row] = Math.pow(raw[row], effectiveGamma) - } - - return values + this.ctx.drawImage(this.waterfallCanvas, 0, 0) } private drawFrame = (): void => { @@ -816,56 +576,18 @@ export class Spectrogram { ) } } - - this.lastWidth = 0 } - this.ensureBandMapping() - if (!this.dataSource.isPlaying()) { this.dataSource.getPendingSpectrogramSamples() // Freeze waterfall in place instead of blanking - this.ctx.clearRect(0, 0, width, height) - if (this.options.backgroundColor !== 'transparent') { - this.ctx.fillStyle = this.options.backgroundColor - this.ctx.fillRect(0, 0, width, height) - } - this.ctx.drawImage(this.waterfallCanvas, 0, 0) + this.paintWaterfall(width, height) return } const pendingSamples = this.dataSource.getPendingSpectrogramSamples() - const fftSize = this.options.fftSize - - // Scroll speed solely controls temporal resolution (hop divisor) - const BASE_HOP_DIVISOR = 8 - const effectiveHopDivisor = Math.max(2, Math.min(64, Math.round(BASE_HOP_DIVISOR * this.options.scrollSpeed))) - const hopSize = Math.max(1, Math.floor(fftSize / effectiveHopDivisor)) - const overlapSamples = fftSize - hopSize - - for (const chunk of pendingSamples) { - for (let index = 0; index < chunk.length; index += 1) { - this.sampleBuffer[this.sampleBufferPos] = chunk[index] - this.sampleBufferPos += 1 - - if (this.sampleBufferPos >= fftSize) { - const magnitudes = this.processFFT(this.sampleBuffer) - const values = this.drawColumn(magnitudes) - // Each FFT hop = exactly 1 pixel slice. No accumulation, no duplication. - this.shiftAndPaintColumn(values, this.heatColumnValues) - - this.sampleBuffer.copyWithin(0, hopSize) - this.sampleBufferPos = overlapSamples - } - } - } - - this.ctx.clearRect(0, 0, width, height) - if (this.options.backgroundColor !== 'transparent') { - this.ctx.fillStyle = this.options.backgroundColor - this.ctx.fillRect(0, 0, width, height) - } - this.ctx.drawImage(this.waterfallCanvas, 0, 0) + this.tryDrawNativeColumns(pendingSamples, width, height) + this.paintWaterfall(width, height) } dispose(): void { diff --git a/src/shared/profileState.ts b/src/shared/profileState.ts index 75c7eb8..24f9066 100644 --- a/src/shared/profileState.ts +++ b/src/shared/profileState.ts @@ -16,7 +16,10 @@ import { AUDIO_SCOPE_KINDS, SCOPE_KINDS, normalizeScopeKind, type ScopeKind } fr import { DEFAULT_SCOPE_SETTINGS, type ScopeSettings } from '../types/settings' import { isLUFSMeterReadout } from '../types/lufsmeter' import { normalizeSpectrumPeakInfoMode } from '../types/spectrum' -import { isSpectrogramOrientation } from '../types/spectrogram' +import { + clampSpectrogramTiltDbPerOctave, + isSpectrogramOrientation, +} from '../types/spectrogram' import { isVUMeterNeedleChannels, sanitizeVUReferenceDbfs } from '../types/vumeter' import { clampWaveformScrollSpeed } from '../types/waveform' @@ -168,6 +171,9 @@ export function mergeScopeSettings(raw: unknown): ScopeSettings { orientation: isSpectrogramOrientation(rawSpectrogram.orientation) ? rawSpectrogram.orientation : DEFAULT_SCOPE_SETTINGS.spectrogram.orientation, + tiltDbPerOctave: clampSpectrogramTiltDbPerOctave( + rawSpectrogram.tiltDbPerOctave ?? DEFAULT_SCOPE_SETTINGS.spectrogram.tiltDbPerOctave + ), }, vumeter: { ...DEFAULT_SCOPE_SETTINGS.vumeter, diff --git a/src/types/settings.ts b/src/types/settings.ts index a8db46b..e6c68c2 100644 --- a/src/types/settings.ts +++ b/src/types/settings.ts @@ -2,6 +2,7 @@ import type { VectorscopeMode } from '../renderer/visualizers/Vectorscope' import { DEFAULT_SPECTROGRAM_CONTRAST, DEFAULT_SPECTROGRAM_ORIENTATION, + DEFAULT_SPECTROGRAM_TILT_DB_PER_OCTAVE, type SpectrogramClarityMode, type SpectrogramOrientation, type SpectrogramScaleMode, @@ -39,6 +40,7 @@ export interface ScopeSettings { } spectrogram: { fftSize: number + tiltDbPerOctave: number scrollSpeed: number contrast: number clarityMode: SpectrogramClarityMode @@ -75,7 +77,7 @@ export const DEFAULT_SCOPE_SETTINGS: ScopeSettings = { spectrum: { fftSize: 2048, tiltDbPerOctave: 2.0, heatmap: false, heatmapTiltDbPerOctave: 2.0, heatmapSmoothing: 0.5, showGrid: true, smoothing: 0.9, fillGradient: true, showSideLine: false, peakInfoMode: DEFAULT_SPECTRUM_PEAK_INFO_MODE }, oscilloscope: { pitchLock: true, underfillEnabled: false, showGrid: true, lineWidth: 2 }, vectorscope: { mode: 'lissajous', multiband: false, showGrid: true, persistence: 0.10, lineWidth: 1.5 }, - spectrogram: { fftSize: 2048, scrollSpeed: 2, contrast: DEFAULT_SPECTROGRAM_CONTRAST, clarityMode: 'sharper', scaleMode: 'log', orientation: DEFAULT_SPECTROGRAM_ORIENTATION, colorScheme: 'heat' }, + spectrogram: { fftSize: 4096, tiltDbPerOctave: DEFAULT_SPECTROGRAM_TILT_DB_PER_OCTAVE, scrollSpeed: 2, contrast: DEFAULT_SPECTROGRAM_CONTRAST, clarityMode: 'sharper', scaleMode: 'log', orientation: DEFAULT_SPECTROGRAM_ORIENTATION, colorScheme: 'heat' }, vumeter: { mode: 'bar', orientation: 'horizontal', needleChannels: 'stereo', referenceDb: DEFAULT_VU_REFERENCE_DBFS }, lufsmeter: { mode: 'bar', readout: DEFAULT_LUFS_METER_READOUT }, waveform: { mode: DEFAULT_WAVEFORM_MODE, scrollSpeed: 1, multiband: false }, diff --git a/src/types/spectrogram.ts b/src/types/spectrogram.ts index 688b818..b76b503 100644 --- a/src/types/spectrogram.ts +++ b/src/types/spectrogram.ts @@ -30,6 +30,11 @@ export const MAX_SPECTROGRAM_CONTRAST = 2.0 export const SPECTROGRAM_CONTRAST_STEP = 0.1 export const DEFAULT_SPECTROGRAM_CONTRAST = 1.0 +export const DEFAULT_SPECTROGRAM_TILT_DB_PER_OCTAVE = 4.0 +export const MIN_SPECTROGRAM_TILT_DB_PER_OCTAVE = -2.0 +export const MAX_SPECTROGRAM_TILT_DB_PER_OCTAVE = 8.0 +export const SPECTROGRAM_TILT_STEP = 0.1 + export function isSpectrogramClarityMode(value: unknown): value is SpectrogramClarityMode { return typeof value === 'string' && SPECTROGRAM_CLARITY_MODES.includes(value as SpectrogramClarityMode) } @@ -61,3 +66,17 @@ export function clampSpectrogramContrast(value: unknown): number { const snapped = Math.round(numeric / SPECTROGRAM_CONTRAST_STEP) * SPECTROGRAM_CONTRAST_STEP return Math.min(MAX_SPECTROGRAM_CONTRAST, Math.max(MIN_SPECTROGRAM_CONTRAST, snapped)) } + +export function clampSpectrogramTiltDbPerOctave(value: unknown): number { + const numeric = Number(value) + if (!Number.isFinite(numeric)) { + return DEFAULT_SPECTROGRAM_TILT_DB_PER_OCTAVE + } + + const snapped = Math.round(numeric / SPECTROGRAM_TILT_STEP) * SPECTROGRAM_TILT_STEP + const rounded = Math.round(snapped * 10) / 10 + return Math.min( + MAX_SPECTROGRAM_TILT_DB_PER_OCTAVE, + Math.max(MIN_SPECTROGRAM_TILT_DB_PER_OCTAVE, rounded), + ) +} diff --git a/test/renderer-helpers.test.ts b/test/renderer-helpers.test.ts index b0cf968..960e42d 100644 --- a/test/renderer-helpers.test.ts +++ b/test/renderer-helpers.test.ts @@ -75,6 +75,11 @@ import { NativeVisualizerTransport, type NativeVisualizerTransportBridge, } from '../src/renderer/audio/NativeVisualizerTransport' +import type { + SpectrogramNativeAnalyzer, + SpectrogramNativeOptions, + SpectrogramNativeResult, +} from '../src/renderer/audio/native' import { HEAT_LOW_DB, HEAT_MAX_DB, @@ -1327,6 +1332,13 @@ test('default profile starts with spectrum peak info disabled', () => { assert.equal(normalizeSpectrumPeakInfoMode('nope'), DEFAULT_SPECTRUM_PEAK_INFO_MODE) }) +test('default profile starts with high-detail spectrogram FFT size', () => { + const profile = createDefaultProfile('Default') + + assert.equal(profile.scopeSettings.spectrogram.fftSize, 4096) + assert.equal(profile.scopeSettings.spectrogram.tiltDbPerOctave, 4) +}) + test('spectrum pitch helpers format nearest note with octave and cents', () => { assert.equal(formatSpectrumPitchInfo(resolveSpectrumPitchInfo(440)), 'A4 0c') assert.equal(formatSpectrumPitchInfo(resolveSpectrumPitchInfo(261.6255653005986)), 'C4 0c') @@ -1567,96 +1579,6 @@ test('Spectrogram keeps the historical display dB range for line thickness', () } }) -test('Spectrogram display gain feeds both display and heat intensity', () => { - const dom = installFakeCanvasDom() - const dataSource = { - getPendingSpectrogramSamples: () => [], - getSampleRate: () => 48000, - isPlaying: () => false, - subscribeToSessionChanges: () => () => {}, - } - const canvas = createFakeCanvas() - canvas.width = 1 - canvas.height = 1 - const spectrogram = new Spectrogram(canvas, { - dataSource, - clarityMode: 'classic', - minDecibels: -90, - maxDecibels: -12, - }) - - try { - const state = spectrogram as unknown as { - drawColumn: (magnitudes: Float32Array) => Float32Array - heatColumnValues: Float32Array - rowCenterBins: Float32Array - rowBandStartBins: Float32Array - rowBandEndBins: Float32Array - } - state.rowCenterBins = Float32Array.from([1]) - state.rowBandStartBins = Float32Array.from([0.5]) - state.rowBandEndBins = Float32Array.from([1.5]) - - const magnitudes = new Float32Array(24) - magnitudes.fill(-120) - magnitudes[1] = -66 - const values = state.drawColumn(magnitudes) - const expectedDisplay = Math.pow((-64 - (-90)) / (-12 - (-90)), 1.4) - const expectedHeat = normalizeHeatDb(-58) - - assertAlmostEqual(values[0], expectedDisplay, 1e-6, 'display intensity should include spectrogram display gain') - assertAlmostEqual(state.heatColumnValues[0], expectedHeat, 1e-6, 'heat color should include display gain before heat compensation') - } finally { - spectrogram.dispose() - dom.restore() - } -}) - -test('Spectrogram tilt adds 4 dB per octave above the reference frequency', () => { - const dom = installFakeCanvasDom() - const dataSource = { - getPendingSpectrogramSamples: () => [], - getSampleRate: () => 48000, - isPlaying: () => false, - subscribeToSessionChanges: () => () => {}, - } - const canvas = createFakeCanvas() - canvas.width = 1 - canvas.height = 1 - const spectrogram = new Spectrogram(canvas, { - dataSource, - clarityMode: 'classic', - minDecibels: -90, - maxDecibels: -12, - }) - - try { - const state = spectrogram as unknown as { - drawColumn: (magnitudes: Float32Array) => Float32Array - heatColumnValues: Float32Array - rowCenterBins: Float32Array - rowBandStartBins: Float32Array - rowBandEndBins: Float32Array - } - state.rowCenterBins = Float32Array.from([2]) - state.rowBandStartBins = Float32Array.from([1.5]) - state.rowBandEndBins = Float32Array.from([2.5]) - - const magnitudes = new Float32Array(24) - magnitudes.fill(-120) - magnitudes[2] = -66 - const values = state.drawColumn(magnitudes) - const expectedDisplay = Math.pow((-60 - (-90)) / (-12 - (-90)), 1.4) - const expectedHeat = normalizeHeatDb(-54) - - assertAlmostEqual(values[0], expectedDisplay, 1e-6, 'display intensity should include +4 dB/oct spectrogram tilt') - assertAlmostEqual(state.heatColumnValues[0], expectedHeat, 1e-6, 'heat color should include +4 dB/oct spectrogram tilt') - } finally { - spectrogram.dispose() - dom.restore() - } -}) - test('Spectrogram custom heat colors land on shared low mid and high thresholds', () => { const imageData = renderSpectrogramColumnImage({ heatColors: [ @@ -1736,35 +1658,148 @@ test('Spectrogram vertical orientation shifts existing rows upward with copy com assert.equal(drawCall?.args[2], -1) }) -test('Spectrogram vertical orientation maps low frequencies to the left', () => { - const dom = installFakeCanvasDom() +test('Spectrogram paints multiple native analyzer columns in order', () => { + const recorder = createFakeCanvasRecorder() + const dom = installFakeCanvasDom(() => createFakeCanvas(recorder)) + const canvas = createFakeCanvas(recorder, 4, 2) + let pending = [Float32Array.from([0, 1, 0, -1])] const dataSource = { - getPendingSpectrogramSamples: () => [], + getPendingSpectrogramSamples: () => { + const chunks = pending + pending = [] + return chunks + }, getSampleRate: () => 48000, - isPlaying: () => false, + isPlaying: () => true, subscribeToSessionChanges: () => () => {}, } - const canvas = createFakeCanvas() - canvas.width = 3 - canvas.height = 5 + const nativeAnalyzer: SpectrogramNativeAnalyzer = { + isAvailable: () => true, + configure: () => {}, + process: () => ({ + display: Float32Array.from([0.25, 0.5, 0.75, 1]), + heat: Float32Array.from([0.25, 0.5, 0.75, 1]), + columnCount: 2, + rowCount: 2, + }), + reset: () => {}, + } const spectrogram = new Spectrogram(canvas, { dataSource, - orientation: 'vertical', - scaleMode: 'linear', - minFrequency: 100, - maxFrequency: 300, + nativeAnalyzer, + colorScheme: 'mono', + lineColor: 'rgb(10, 20, 30)', }) try { - const state = spectrogram as unknown as { - ensureBandMapping: () => void - rowCenterBins: Float32Array - } - state.ensureBandMapping() + const state = spectrogram as unknown as { drawFrame: () => void } + state.drawFrame() - assert.equal(state.rowCenterBins.length, 3) - assert.equal(state.rowCenterBins[0] < state.rowCenterBins[1], true) - assert.equal(state.rowCenterBins[1] < state.rowCenterBins[2], true) + const writes = recorder.imageDataWrites + assert.equal(writes.length, 2) + assert.deepEqual(writes[0].data.slice(0, 8), [10, 20, 30, 64, 10, 20, 30, 128]) + assert.deepEqual(writes[1].data.slice(0, 8), [10, 20, 30, 191, 10, 20, 30, 255]) + } finally { + spectrogram.dispose() + dom.restore() + } +}) + +test('Spectrogram forwards orientation and row count to the native analyzer', () => { + const dom = installFakeCanvasDom() + const canvas = createFakeCanvas(null, 3, 5) + let capturedConfig: SpectrogramNativeOptions | null = null + let pending = [Float32Array.from([0, 0])] + const dataSource = { + getPendingSpectrogramSamples: () => { + const chunks = pending + pending = [] + return chunks + }, + getSampleRate: () => 44100, + isPlaying: () => true, + subscribeToSessionChanges: () => () => {}, + } + const nativeAnalyzer: SpectrogramNativeAnalyzer = { + isAvailable: () => true, + configure: (options) => { + capturedConfig = options + }, + process: (): SpectrogramNativeResult => ({ + display: new Float32Array(0), + heat: new Float32Array(0), + columnCount: 0, + rowCount: 3, + }), + reset: () => {}, + } + const spectrogram = new Spectrogram(canvas, { + dataSource, + nativeAnalyzer, + orientation: 'vertical', + scaleMode: 'linear', + fftSize: 8192, + scrollSpeed: 4, + tiltDbPerOctave: 5.5, + }) + + try { + const state = spectrogram as unknown as { drawFrame: () => void } + state.drawFrame() + + assert.equal(capturedConfig?.orientation, 'vertical') + assert.equal(capturedConfig?.rowCount, 3) + assert.equal(capturedConfig?.scaleMode, 'linear') + assert.equal(capturedConfig?.fftSize, 8192) + assert.equal(capturedConfig?.scrollSpeed, 4) + assert.equal(capturedConfig?.tiltDbPerOctave, 5.5) + assert.equal(capturedConfig?.sampleRate, 44100) + } finally { + spectrogram.dispose() + dom.restore() + } +}) + +test('Spectrogram freezes the waterfall when native analyzer is unavailable', () => { + const recorder = createFakeCanvasRecorder() + const dom = installFakeCanvasDom() + const canvas = createFakeCanvas(recorder, 4, 4) + let pending = [Float32Array.from([0, 1, 0, -1])] + let pendingFlushes = 0 + let nativeProcessCalls = 0 + const dataSource = { + getPendingSpectrogramSamples: () => { + pendingFlushes += 1 + const chunks = pending + pending = [] + return chunks + }, + getSampleRate: () => 48000, + isPlaying: () => true, + subscribeToSessionChanges: () => () => {}, + } + const nativeAnalyzer: SpectrogramNativeAnalyzer = { + isAvailable: () => false, + configure: () => {}, + process: () => { + nativeProcessCalls += 1 + return null + }, + reset: () => {}, + } + const spectrogram = new Spectrogram(canvas, { + dataSource, + nativeAnalyzer, + }) + + try { + const state = spectrogram as unknown as { drawFrame: () => void } + state.drawFrame() + + assert.equal(pendingFlushes, 1) + assert.equal(nativeProcessCalls, 0) + assert.equal(recorder.imageDataWrites.length, 0) + assert.equal(recorder.drawImageCalls.length, 1) } finally { spectrogram.dispose() dom.restore() @@ -2297,6 +2332,7 @@ test('Vectorscope keeps the original linear projection behavior', () => { test('scopeSettingsToOptions forwards themed backgrounds and track colors to spectrogram, VU, and LUFS modules', () => { const profile = createDefaultProfile('Default') profile.scopeSettings.spectrogram.orientation = 'vertical' + profile.scopeSettings.spectrogram.tiltDbPerOctave = 5.2 profile.scopeSettings.lufsmeter.readout = 'shortTerm' profile.scopeSettings.vumeter.needleChannels = 'combined' const authoredTheme = createDefaultTheme() @@ -2312,6 +2348,7 @@ test('scopeSettingsToOptions forwards themed backgrounds and track colors to spe const spectrogram = scopeSettingsToOptions('spectrogram', profile.scopeSettings.spectrogram, theme.spectrogram) assert.equal(spectrogram.backgroundColor, 'rgb(6, 7, 8)') assert.equal(spectrogram.orientation, 'vertical') + assert.equal(spectrogram.tiltDbPerOctave, 5.2) const vumeter = scopeSettingsToOptions('vumeter', profile.scopeSettings.vumeter, theme.vumeter) assert.equal(vumeter.backgroundColor, 'rgb(6, 7, 8)')