From d9cf2638564bd781faadc041099403f4d5f0d656 Mon Sep 17 00:00:00 2001 From: Boof2015 <75185879+Boof2015@users.noreply.github.com> Date: Tue, 19 May 2026 19:02:44 -0400 Subject: [PATCH] rewrite multiband splitter to native --- native/binding.gyp | 2 + native/src/dsp_utils.cpp | 14 + native/src/dsp_utils.h | 1 + native/src/main.cpp | 104 +++++++ native/src/multiband.cpp | 63 +++++ native/src/multiband.h | 42 +++ native/src/vectorscope.cpp | 60 +++- native/src/vectorscope.h | 12 + native/src/waveform.cpp | 152 +++++++++++ native/src/waveform.h | 48 ++++ src/preload/index.ts | 1 + src/renderer/audio/native/index.ts | 86 +++++- src/renderer/audio/native/visualizer-dsp.d.ts | 15 + src/renderer/visualizers/Vectorscope.ts | 128 +++++++-- src/renderer/visualizers/Waveform.ts | 117 +++++++- test/renderer-helpers.test.ts | 257 ++++++++++++++++++ 16 files changed, 1064 insertions(+), 38 deletions(-) create mode 100644 native/src/multiband.cpp create mode 100644 native/src/multiband.h create mode 100644 native/src/waveform.cpp create mode 100644 native/src/waveform.h diff --git a/native/binding.gyp b/native/binding.gyp index a6a3ce1..1e224ca 100644 --- a/native/binding.gyp +++ b/native/binding.gyp @@ -11,6 +11,8 @@ "src/spectrum.cpp", "src/spectrogram.cpp", "src/vectorscope.cpp", + "src/multiband.cpp", + "src/waveform.cpp", "src/vumeter.cpp", "src/lufsmeter.cpp", "src/dsp_utils.cpp" diff --git a/native/src/dsp_utils.cpp b/native/src/dsp_utils.cpp index 3fba83b..ae7f8d0 100644 --- a/native/src/dsp_utils.cpp +++ b/native/src/dsp_utils.cpp @@ -88,6 +88,20 @@ void BiquadFilter::setLowpass(float frequency, float sampleRate, float Q) { a2_ = (1.0f - alpha) / a0; } +void BiquadFilter::setHighpass(float frequency, float sampleRate, float Q) { + float omega = 2.0f * M_PI * frequency / sampleRate; + float sinOmega = sinf(omega); + float cosOmega = cosf(omega); + float alpha = sinOmega / (2.0f * Q); + + float a0 = 1.0f + alpha; + b0_ = (1.0f + cosOmega) / 2.0f / a0; + b1_ = -(1.0f + cosOmega) / a0; + b2_ = (1.0f + cosOmega) / 2.0f / a0; + a1_ = -2.0f * cosOmega / a0; + a2_ = (1.0f - alpha) / a0; +} + void BiquadFilter::setBandpass(float frequency, float sampleRate, float Q) { float omega = 2.0f * M_PI * frequency / sampleRate; float sinOmega = sinf(omega); diff --git a/native/src/dsp_utils.h b/native/src/dsp_utils.h index 92136bd..db90a4f 100644 --- a/native/src/dsp_utils.h +++ b/native/src/dsp_utils.h @@ -32,6 +32,7 @@ class BiquadFilter { public: BiquadFilter(); void setLowpass(float frequency, float sampleRate, float Q = 0.707f); + void setHighpass(float frequency, float sampleRate, float Q = 0.707f); void setBandpass(float frequency, float sampleRate, float Q = 2.0f); void setHighShelf(float frequency, float sampleRate, float gainDB, float Q = 0.707f); float process(float input); diff --git a/native/src/main.cpp b/native/src/main.cpp index 0be8c10..093cd05 100644 --- a/native/src/main.cpp +++ b/native/src/main.cpp @@ -9,6 +9,7 @@ #include "spectrum.h" #include "spectrogram.h" #include "vectorscope.h" +#include "waveform.h" #include "vumeter.h" #include "lufsmeter.h" @@ -17,6 +18,7 @@ static Visualizer::Oscilloscope oscilloscope; static Visualizer::Spectrum spectrum(2048); static Visualizer::SpectrogramAnalyzer spectrogramAnalyzer; static Visualizer::Vectorscope vectorscope; +static Visualizer::WaveformMultibandAnalyzer waveform; static Visualizer::VUMeterAnalyzer vuMeter; static Visualizer::LUFSMeterAnalyzer lufsMeter; @@ -350,6 +352,19 @@ Napi::Value VectorscopePushSamples(const Napi::CallbackInfo& info) { return env.Undefined(); } +Napi::Value VectorscopePushMultibandSamples(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() < 2 || !info[0].IsTypedArray() || !info[1].IsTypedArray()) { + Napi::TypeError::New(env, "Expected two Float32Arrays (left, right)").ThrowAsJavaScriptException(); + return env.Null(); + } + Napi::Float32Array leftData = info[0].As(); + Napi::Float32Array rightData = info[1].As(); + size_t length = std::min(leftData.ElementLength(), rightData.ElementLength()); + vectorscope.pushMultibandSamples(leftData.Data(), rightData.Data(), length); + return env.Undefined(); +} + Napi::Value VectorscopeGetPoints(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (info.Length() < 1 || !info[0].IsNumber()) { @@ -376,6 +391,31 @@ Napi::Value VectorscopeGetPoints(const Napi::CallbackInfo& info) { return result; } +Napi::Value VectorscopeGetMultibandPoints(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() < 1 || !info[0].IsNumber()) { + Napi::TypeError::New(env, "Expected max points count").ThrowAsJavaScriptException(); + return env.Null(); + } + + const size_t maxPoints = static_cast(info[0].As().Uint32Value()); + Napi::Float32Array data = Napi::Float32Array::New(env, maxPoints * Visualizer::MULTIBAND_POINT_STRIDE); + const size_t actual = vectorscope.getMultibandPoints(data.Data(), maxPoints); + + Napi::Object result = Napi::Object::New(env); + if (actual < maxPoints) { + Napi::Float32Array trimmed = Napi::Float32Array::New(env, actual * Visualizer::MULTIBAND_POINT_STRIDE); + if (actual > 0) { + memcpy(trimmed.Data(), data.Data(), actual * Visualizer::MULTIBAND_POINT_STRIDE * sizeof(float)); + } + result.Set("data", trimmed); + } else { + result.Set("data", data); + } + result.Set("count", Napi::Number::New(env, static_cast(actual))); + return result; +} + Napi::Value VectorscopeFillPoints(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (info.Length() < 2 || !info[0].IsTypedArray() || !info[1].IsTypedArray()) { @@ -431,6 +471,60 @@ Napi::Value VectorscopeReset(const Napi::CallbackInfo& info) { return info.Env().Undefined(); } +// ============== Waveform ============== + +Napi::Value WaveformConfigure(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() < 2 || !info[0].IsNumber() || !info[1].IsNumber()) { + Napi::TypeError::New(env, "Expected sample rate and samples per column").ThrowAsJavaScriptException(); + return env.Null(); + } + + const float sampleRate = info[0].As().FloatValue(); + const size_t samplesPerColumn = static_cast(info[1].As().Uint32Value()); + waveform.configure(sampleRate, samplesPerColumn); + return env.Undefined(); +} + +Napi::Value WaveformProcessMono(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() < 1 || !info[0].IsTypedArray()) { + Napi::TypeError::New(env, "Expected Float32Array").ThrowAsJavaScriptException(); + return env.Null(); + } + + Napi::Float32Array samples = info[0].As(); + const auto& summaries = waveform.processMono(samples.Data(), samples.ElementLength()); + Napi::Float32Array result = Napi::Float32Array::New(env, summaries.size()); + if (!summaries.empty()) { + memcpy(result.Data(), summaries.data(), summaries.size() * sizeof(float)); + } + return result; +} + +Napi::Value WaveformProcessStereo(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() < 2 || !info[0].IsTypedArray() || !info[1].IsTypedArray()) { + Napi::TypeError::New(env, "Expected two Float32Arrays (left, right)").ThrowAsJavaScriptException(); + return env.Null(); + } + + Napi::Float32Array leftData = info[0].As(); + Napi::Float32Array rightData = info[1].As(); + const size_t length = std::min(leftData.ElementLength(), rightData.ElementLength()); + const auto& summaries = waveform.processStereo(leftData.Data(), rightData.Data(), length); + Napi::Float32Array result = Napi::Float32Array::New(env, summaries.size()); + if (!summaries.empty()) { + memcpy(result.Data(), summaries.data(), summaries.size() * sizeof(float)); + } + return result; +} + +Napi::Value WaveformReset(const Napi::CallbackInfo& info) { + waveform.reset(); + return info.Env().Undefined(); +} + // ============== VU Meter ============== Napi::Value VUMeterSetSampleRate(const Napi::CallbackInfo& info) { @@ -570,14 +664,24 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) { Napi::Object vecExports = Napi::Object::New(env); vecExports.Set("setSampleRate", Napi::Function::New(env, VectorscopeSetSampleRate)); vecExports.Set("pushSamples", Napi::Function::New(env, VectorscopePushSamples)); + vecExports.Set("pushMultibandSamples", Napi::Function::New(env, VectorscopePushMultibandSamples)); vecExports.Set("fillPoints", Napi::Function::New(env, VectorscopeFillPoints)); vecExports.Set("getPoints", Napi::Function::New(env, VectorscopeGetPoints)); + vecExports.Set("getMultibandPoints", Napi::Function::New(env, VectorscopeGetMultibandPoints)); vecExports.Set("setBufferSize", Napi::Function::New(env, VectorscopeSetBufferSize)); vecExports.Set("getBufferSize", Napi::Function::New(env, VectorscopeGetBufferSize)); vecExports.Set("process", Napi::Function::New(env, VectorscopeProcess)); vecExports.Set("reset", Napi::Function::New(env, VectorscopeReset)); exports.Set("vectorscope", vecExports); + // Waveform + Napi::Object waveformExports = Napi::Object::New(env); + waveformExports.Set("configure", Napi::Function::New(env, WaveformConfigure)); + waveformExports.Set("processMono", Napi::Function::New(env, WaveformProcessMono)); + waveformExports.Set("processStereo", Napi::Function::New(env, WaveformProcessStereo)); + waveformExports.Set("reset", Napi::Function::New(env, WaveformReset)); + exports.Set("waveform", waveformExports); + // VU Meter Napi::Object vuExports = Napi::Object::New(env); vuExports.Set("setSampleRate", Napi::Function::New(env, VUMeterSetSampleRate)); diff --git a/native/src/multiband.cpp b/native/src/multiband.cpp new file mode 100644 index 0000000..f1de8d6 --- /dev/null +++ b/native/src/multiband.cpp @@ -0,0 +1,63 @@ +#include "multiband.h" +#include +#include + +namespace Visualizer { + +namespace { +constexpr float MULTIBAND_FILTER_Q = 1.41421356237f; +} + +MultibandSplitter::MultibandSplitter() + : configuredSampleRate_(0.0f) { + configure(48000.0f); +} + +void MultibandSplitter::configure(float sampleRate) { + const float nextSampleRate = std::max(1.0f, sampleRate); + if (nextSampleRate == configuredSampleRate_) { + return; + } + + configuredSampleRate_ = nextSampleRate; + + lowLpL_.setLowpass(MULTIBAND_LOW_MID_CROSSOVER, configuredSampleRate_, MULTIBAND_FILTER_Q); + lowLpR_.setLowpass(MULTIBAND_LOW_MID_CROSSOVER, configuredSampleRate_, MULTIBAND_FILTER_Q); + + midHpL_.setHighpass(MULTIBAND_LOW_MID_CROSSOVER, configuredSampleRate_, MULTIBAND_FILTER_Q); + midHpR_.setHighpass(MULTIBAND_LOW_MID_CROSSOVER, configuredSampleRate_, MULTIBAND_FILTER_Q); + midLpL_.setLowpass(MULTIBAND_MID_HIGH_CROSSOVER, configuredSampleRate_, MULTIBAND_FILTER_Q); + midLpR_.setLowpass(MULTIBAND_MID_HIGH_CROSSOVER, configuredSampleRate_, MULTIBAND_FILTER_Q); + + highHpL_.setHighpass(MULTIBAND_MID_HIGH_CROSSOVER, configuredSampleRate_, MULTIBAND_FILTER_Q); + highHpR_.setHighpass(MULTIBAND_MID_HIGH_CROSSOVER, configuredSampleRate_, MULTIBAND_FILTER_Q); + + reset(); +} + +MultibandSample MultibandSplitter::process(float left, float right) { + const float midTmpL = midHpL_.process(left); + const float midTmpR = midHpR_.process(right); + + return { + lowLpL_.process(left), + lowLpR_.process(right), + midLpL_.process(midTmpL), + midLpR_.process(midTmpR), + highHpL_.process(left), + highHpR_.process(right), + }; +} + +void MultibandSplitter::reset() { + lowLpL_.reset(); + lowLpR_.reset(); + midHpL_.reset(); + midHpR_.reset(); + midLpL_.reset(); + midLpR_.reset(); + highHpL_.reset(); + highHpR_.reset(); +} + +} // namespace Visualizer diff --git a/native/src/multiband.h b/native/src/multiband.h new file mode 100644 index 0000000..683777c --- /dev/null +++ b/native/src/multiband.h @@ -0,0 +1,42 @@ +#pragma once + +#include "dsp_utils.h" +#include + +namespace Visualizer { + +constexpr float MULTIBAND_LOW_MID_CROSSOVER = 250.0f; +constexpr float MULTIBAND_MID_HIGH_CROSSOVER = 2500.0f; +constexpr size_t MULTIBAND_POINT_STRIDE = 6; + +struct MultibandSample { + float lowL; + float lowR; + float midL; + float midR; + float highL; + float highR; +}; + +class MultibandSplitter { +public: + MultibandSplitter(); + + void configure(float sampleRate); + MultibandSample process(float left, float right); + void reset(); + +private: + float configuredSampleRate_; + + DSP::BiquadFilter lowLpL_; + DSP::BiquadFilter lowLpR_; + DSP::BiquadFilter midHpL_; + DSP::BiquadFilter midHpR_; + DSP::BiquadFilter midLpL_; + DSP::BiquadFilter midLpR_; + DSP::BiquadFilter highHpL_; + DSP::BiquadFilter highHpR_; +}; + +} // namespace Visualizer diff --git a/native/src/vectorscope.cpp b/native/src/vectorscope.cpp index ff1c5a7..dea93e5 100644 --- a/native/src/vectorscope.cpp +++ b/native/src/vectorscope.cpp @@ -8,10 +8,18 @@ Vectorscope::Vectorscope() : sampleRate_(48000.0f) , bufferSize_(1024) , writePos_(0) - , validSamples_(0) { + , validSamples_(0) + , multibandWritePos_(0) + , multibandValidSamples_(0) { leftBuffer_.resize(VECTORSCOPE_BUFFER_SIZE, 0.0f); rightBuffer_.resize(VECTORSCOPE_BUFFER_SIZE, 0.0f); + lowLeftBuffer_.resize(VECTORSCOPE_BUFFER_SIZE, 0.0f); + lowRightBuffer_.resize(VECTORSCOPE_BUFFER_SIZE, 0.0f); + midLeftBuffer_.resize(VECTORSCOPE_BUFFER_SIZE, 0.0f); + midRightBuffer_.resize(VECTORSCOPE_BUFFER_SIZE, 0.0f); + highLeftBuffer_.resize(VECTORSCOPE_BUFFER_SIZE, 0.0f); + highRightBuffer_.resize(VECTORSCOPE_BUFFER_SIZE, 0.0f); points_.reserve(1024); // Cascaded lowpass at 8kHz, Butterworth (Q=0.707) @@ -21,6 +29,7 @@ Vectorscope::Vectorscope() leftLowpass2_.setLowpass(8000.0f, sampleRate_, 0.707f); rightLowpass1_.setLowpass(8000.0f, sampleRate_, 0.707f); rightLowpass2_.setLowpass(8000.0f, sampleRate_, 0.707f); + multibandSplitter_.configure(sampleRate_); } void Vectorscope::setSampleRate(float sampleRate) { @@ -30,6 +39,7 @@ void Vectorscope::setSampleRate(float sampleRate) { leftLowpass2_.setLowpass(8000.0f, sampleRate_, 0.707f); rightLowpass1_.setLowpass(8000.0f, sampleRate_, 0.707f); rightLowpass2_.setLowpass(8000.0f, sampleRate_, 0.707f); + multibandSplitter_.configure(sampleRate_); } void Vectorscope::setBufferSize(size_t size) { @@ -60,6 +70,28 @@ void Vectorscope::pushSamples( } } +void Vectorscope::pushMultibandSamples( + const float* leftChannel, + const float* rightChannel, + size_t length +) { + for (size_t i = 0; i < length; i++) { + const MultibandSample bands = multibandSplitter_.process(leftChannel[i], rightChannel[i]); + + lowLeftBuffer_[multibandWritePos_] = bands.lowL; + lowRightBuffer_[multibandWritePos_] = bands.lowR; + midLeftBuffer_[multibandWritePos_] = bands.midL; + midRightBuffer_[multibandWritePos_] = bands.midR; + highLeftBuffer_[multibandWritePos_] = bands.highL; + highRightBuffer_[multibandWritePos_] = bands.highR; + + multibandWritePos_ = (multibandWritePos_ + 1) % VECTORSCOPE_BUFFER_SIZE; + if (multibandValidSamples_ < VECTORSCOPE_BUFFER_SIZE) { + multibandValidSamples_++; + } + } +} + size_t Vectorscope::getPoints(float* xOut, float* yOut, size_t maxPoints) const { size_t count = std::min(maxPoints, validSamples_); @@ -73,6 +105,23 @@ size_t Vectorscope::getPoints(float* xOut, float* yOut, size_t maxPoints) const return count; } +size_t Vectorscope::getMultibandPoints(float* output, size_t maxPoints) const { + const size_t count = std::min(maxPoints, multibandValidSamples_); + + for (size_t i = 0; i < count; i++) { + const size_t idx = (multibandWritePos_ + VECTORSCOPE_BUFFER_SIZE - count + i) % VECTORSCOPE_BUFFER_SIZE; + const size_t base = i * MULTIBAND_POINT_STRIDE; + output[base] = lowLeftBuffer_[idx]; + output[base + 1] = lowRightBuffer_[idx]; + output[base + 2] = midLeftBuffer_[idx]; + output[base + 3] = midRightBuffer_[idx]; + output[base + 4] = highLeftBuffer_[idx]; + output[base + 5] = highRightBuffer_[idx]; + } + + return count; +} + // Legacy process method (routes through new pipeline) const std::vector& Vectorscope::process( const float* leftChannel, @@ -98,12 +147,21 @@ const std::vector& Vectorscope::process( void Vectorscope::reset() { writePos_ = 0; validSamples_ = 0; + multibandWritePos_ = 0; + multibandValidSamples_ = 0; std::fill(leftBuffer_.begin(), leftBuffer_.end(), 0.0f); std::fill(rightBuffer_.begin(), rightBuffer_.end(), 0.0f); + std::fill(lowLeftBuffer_.begin(), lowLeftBuffer_.end(), 0.0f); + std::fill(lowRightBuffer_.begin(), lowRightBuffer_.end(), 0.0f); + std::fill(midLeftBuffer_.begin(), midLeftBuffer_.end(), 0.0f); + std::fill(midRightBuffer_.begin(), midRightBuffer_.end(), 0.0f); + std::fill(highLeftBuffer_.begin(), highLeftBuffer_.end(), 0.0f); + std::fill(highRightBuffer_.begin(), highRightBuffer_.end(), 0.0f); leftLowpass1_.reset(); leftLowpass2_.reset(); rightLowpass1_.reset(); rightLowpass2_.reset(); + multibandSplitter_.reset(); points_.clear(); } diff --git a/native/src/vectorscope.h b/native/src/vectorscope.h index 0042c09..061a6ea 100644 --- a/native/src/vectorscope.h +++ b/native/src/vectorscope.h @@ -1,6 +1,7 @@ #pragma once #include "dsp_utils.h" +#include "multiband.h" #include #include @@ -25,10 +26,12 @@ public: // Push stereo samples into circular buffer (called per worklet chunk) void pushSamples(const float* leftChannel, const float* rightChannel, size_t length); + void pushMultibandSamples(const float* leftChannel, const float* rightChannel, size_t length); // Get the most recent N points for rendering (from circular buffer) // Returns count of valid points written to output arrays size_t getPoints(float* xOut, float* yOut, size_t maxPoints) const; + size_t getMultibandPoints(float* output, size_t maxPoints) const; // Get number of valid samples in buffer size_t getValidSamples() const { return validSamples_; } @@ -52,12 +55,21 @@ private: // Circular buffers for filtered L/R std::vector leftBuffer_; std::vector rightBuffer_; + std::vector lowLeftBuffer_; + std::vector lowRightBuffer_; + std::vector midLeftBuffer_; + std::vector midRightBuffer_; + std::vector highLeftBuffer_; + std::vector highRightBuffer_; // Cascaded lowpass filters (4th order Butterworth at 8kHz per channel) DSP::BiquadFilter leftLowpass1_; DSP::BiquadFilter leftLowpass2_; DSP::BiquadFilter rightLowpass1_; DSP::BiquadFilter rightLowpass2_; + MultibandSplitter multibandSplitter_; + size_t multibandWritePos_; + size_t multibandValidSamples_; // Legacy std::vector points_; diff --git a/native/src/waveform.cpp b/native/src/waveform.cpp new file mode 100644 index 0000000..bdd9c9e --- /dev/null +++ b/native/src/waveform.cpp @@ -0,0 +1,152 @@ +#include "waveform.h" +#include +#include + +namespace Visualizer { + +WaveformMultibandAnalyzer::WaveformMultibandAnalyzer() + : sampleRate_(48000.0f) + , samplesPerColumn_(1) + , columnPos_(0) + , leftMin_(0.0f) + , leftMax_(0.0f) + , rightMin_(0.0f) + , rightMax_(0.0f) + , leftLowSum_(0.0f) + , leftMidSum_(0.0f) + , leftHighSum_(0.0f) + , rightLowSum_(0.0f) + , rightMidSum_(0.0f) + , rightHighSum_(0.0f) { + splitter_.configure(sampleRate_); +} + +void WaveformMultibandAnalyzer::configure(float sampleRate, size_t samplesPerColumn) { + const float nextSampleRate = std::max(1.0f, sampleRate); + const size_t nextSamplesPerColumn = std::max(1, samplesPerColumn); + + if (nextSampleRate == sampleRate_ && nextSamplesPerColumn == samplesPerColumn_) { + return; + } + + sampleRate_ = nextSampleRate; + samplesPerColumn_ = nextSamplesPerColumn; + splitter_.configure(sampleRate_); + reset(); +} + +const std::vector& WaveformMultibandAnalyzer::processMono(const float* samples, size_t length) { + summaries_.clear(); + + for (size_t i = 0; i < length; i++) { + const float sample = samples[i]; + const MultibandSample bands = splitter_.process(sample, sample); + accumulateLeft(sample, bands); + + columnPos_++; + if (columnPos_ >= samplesPerColumn_) { + flushMonoColumn(); + resetColumn(); + } + } + + return summaries_; +} + +const std::vector& WaveformMultibandAnalyzer::processStereo( + const float* left, + const float* right, + size_t length +) { + summaries_.clear(); + + for (size_t i = 0; i < length; i++) { + const MultibandSample bands = splitter_.process(left[i], right[i]); + accumulateLeft(left[i], bands); + accumulateRight(right[i], bands); + + columnPos_++; + if (columnPos_ >= samplesPerColumn_) { + flushStereoColumn(); + resetColumn(); + } + } + + return summaries_; +} + +void WaveformMultibandAnalyzer::reset() { + splitter_.reset(); + summaries_.clear(); + resetColumn(); +} + +void WaveformMultibandAnalyzer::resetColumn() { + columnPos_ = 0; + leftMin_ = 0.0f; + leftMax_ = 0.0f; + rightMin_ = 0.0f; + rightMax_ = 0.0f; + leftLowSum_ = 0.0f; + leftMidSum_ = 0.0f; + leftHighSum_ = 0.0f; + rightLowSum_ = 0.0f; + rightMidSum_ = 0.0f; + rightHighSum_ = 0.0f; +} + +void WaveformMultibandAnalyzer::accumulateLeft(float sample, const MultibandSample& bands) { + if (columnPos_ == 0) { + leftMin_ = sample; + leftMax_ = sample; + } else { + leftMin_ = std::min(leftMin_, sample); + leftMax_ = std::max(leftMax_, sample); + } + + leftLowSum_ += bands.lowL * bands.lowL; + leftMidSum_ += bands.midL * bands.midL; + leftHighSum_ += bands.highL * bands.highL; +} + +void WaveformMultibandAnalyzer::accumulateRight(float sample, const MultibandSample& bands) { + if (columnPos_ == 0) { + rightMin_ = sample; + rightMax_ = sample; + } else { + rightMin_ = std::min(rightMin_, sample); + rightMax_ = std::max(rightMax_, sample); + } + + rightLowSum_ += bands.lowR * bands.lowR; + rightMidSum_ += bands.midR * bands.midR; + rightHighSum_ += bands.highR * bands.highR; +} + +void WaveformMultibandAnalyzer::flushMonoColumn() { + summaries_.push_back(leftMin_); + summaries_.push_back(leftMax_); + summaries_.push_back(rms(leftLowSum_)); + summaries_.push_back(rms(leftMidSum_)); + summaries_.push_back(rms(leftHighSum_)); +} + +void WaveformMultibandAnalyzer::flushStereoColumn() { + summaries_.push_back(leftMin_); + summaries_.push_back(leftMax_); + summaries_.push_back(rms(leftLowSum_)); + summaries_.push_back(rms(leftMidSum_)); + summaries_.push_back(rms(leftHighSum_)); + summaries_.push_back(rightMin_); + summaries_.push_back(rightMax_); + summaries_.push_back(rms(rightLowSum_)); + summaries_.push_back(rms(rightMidSum_)); + summaries_.push_back(rms(rightHighSum_)); +} + +float WaveformMultibandAnalyzer::rms(float sum) const { + const size_t count = std::max(1, columnPos_); + return std::sqrt(sum / static_cast(count)); +} + +} // namespace Visualizer diff --git a/native/src/waveform.h b/native/src/waveform.h new file mode 100644 index 0000000..c8f5169 --- /dev/null +++ b/native/src/waveform.h @@ -0,0 +1,48 @@ +#pragma once + +#include "multiband.h" +#include +#include + +namespace Visualizer { + +constexpr size_t WAVEFORM_MONO_SUMMARY_STRIDE = 5; +constexpr size_t WAVEFORM_STEREO_SUMMARY_STRIDE = 10; + +class WaveformMultibandAnalyzer { +public: + WaveformMultibandAnalyzer(); + + void configure(float sampleRate, size_t samplesPerColumn); + const std::vector& processMono(const float* samples, size_t length); + const std::vector& processStereo(const float* left, const float* right, size_t length); + void reset(); + +private: + void resetColumn(); + void accumulateLeft(float sample, const MultibandSample& bands); + void accumulateRight(float sample, const MultibandSample& bands); + void flushMonoColumn(); + void flushStereoColumn(); + float rms(float sum) const; + + MultibandSplitter splitter_; + float sampleRate_; + size_t samplesPerColumn_; + size_t columnPos_; + + float leftMin_; + float leftMax_; + float rightMin_; + float rightMax_; + float leftLowSum_; + float leftMidSum_; + float leftHighSum_; + float rightLowSum_; + float rightMidSum_; + float rightHighSum_; + + std::vector summaries_; +}; + +} // namespace Visualizer diff --git a/src/preload/index.ts b/src/preload/index.ts index 031acf1..db87368 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -247,6 +247,7 @@ const visualizerAPI = nativeAddonModule spectrum: nativeAddonModule.spectrum, spectrogram: nativeAddonModule.spectrogram, vectorscope: nativeAddonModule.vectorscope, + waveform: nativeAddonModule.waveform, vumeter: nativeAddonModule.vumeter, lufsmeter: nativeAddonModule.lufsmeter, } diff --git a/src/renderer/audio/native/index.ts b/src/renderer/audio/native/index.ts index 074d053..51892b4 100644 --- a/src/renderer/audio/native/index.ts +++ b/src/renderer/audio/native/index.ts @@ -7,6 +7,7 @@ import type { LUFSMeterNativeSnapshot, SpectrogramNativeOptions, SpectrogramNativeResult, + VectorscopeMultibandPointsResult, VectorscopeResult, VectorscopePointsResult, VUMeterNativeSnapshot, @@ -185,6 +186,25 @@ export interface VUMeterNativeAnalyzer { isAvailable?: () => boolean } +export interface VectorscopeNativeAnalyzer { + setSampleRate(sampleRate: number): void + pushSamples(leftChannel: Float32Array, rightChannel: Float32Array): void + pushMultibandSamples?: (leftChannel: Float32Array, rightChannel: Float32Array) => void + fillPoints(xOut: Float32Array, yOut: Float32Array): number + getMultibandPoints?: (maxPoints: number) => VectorscopeMultibandPointsResult | null + reset(): void + isAvailable?: () => boolean + isMultibandAvailable?: () => boolean +} + +export interface WaveformNativeAnalyzer { + configure(sampleRate: number, samplesPerColumn: number): void + processMono(samples: Float32Array): Float32Array | null + processStereo(leftChannel: Float32Array, rightChannel: Float32Array): Float32Array | null + reset(): void + isAvailable?: () => boolean +} + export const spectrogram: SpectrogramNativeAnalyzer = { isAvailable: (): boolean => { return Boolean(nativeModule?.spectrogram) @@ -204,17 +224,37 @@ export const spectrogram: SpectrogramNativeAnalyzer = { }, } -export const vectorscope = { +export const vectorscope: VectorscopeNativeAnalyzer & { + getPoints: (maxPoints: number) => VectorscopePointsResult | null + getBufferSize: () => number + setBufferSize: (size: number) => void + process: (leftChannel: Float32Array, rightChannel: Float32Array) => VectorscopeResult | null +} = { + isAvailable: (): boolean => { + return Boolean(nativeModule?.vectorscope) + }, + + isMultibandAvailable: (): boolean => { + return Boolean( + nativeModule?.vectorscope?.pushMultibandSamples + && nativeModule?.vectorscope?.getMultibandPoints, + ) + }, + setSampleRate: (sampleRate: number): void => { - nativeModule?.vectorscope.setSampleRate(sampleRate) + nativeModule?.vectorscope?.setSampleRate(sampleRate) }, pushSamples: (leftChannel: Float32Array, rightChannel: Float32Array): void => { - nativeModule?.vectorscope.pushSamples(leftChannel, rightChannel) + nativeModule?.vectorscope?.pushSamples(leftChannel, rightChannel) + }, + + pushMultibandSamples: (leftChannel: Float32Array, rightChannel: Float32Array): void => { + nativeModule?.vectorscope?.pushMultibandSamples?.(leftChannel, rightChannel) }, fillPoints: (xOut: Float32Array, yOut: Float32Array): number => { - if (!nativeModule) return 0 + if (!nativeModule?.vectorscope) return 0 const result = nativeModule.vectorscope.getPoints(Math.min(xOut.length, yOut.length)) const count = Math.min(xOut.length, yOut.length, result.count, result.x.length, result.y.length) if (count > 0) { @@ -224,13 +264,18 @@ export const vectorscope = { return count }, + getMultibandPoints: (maxPoints: number): VectorscopeMultibandPointsResult | null => { + if (!nativeModule?.vectorscope?.getMultibandPoints) return null + return nativeModule.vectorscope.getMultibandPoints(maxPoints) + }, + getPoints: (maxPoints: number): VectorscopePointsResult | null => { - if (!nativeModule) return null + if (!nativeModule?.vectorscope) return null return nativeModule.vectorscope.getPoints(maxPoints) }, setBufferSize: (size: number): void => { - nativeModule?.vectorscope.setBufferSize(size) + nativeModule?.vectorscope?.setBufferSize(size) }, getBufferSize: (): number => { @@ -238,15 +283,39 @@ export const vectorscope = { }, process: (leftChannel: Float32Array, rightChannel: Float32Array): VectorscopeResult | null => { - if (!nativeModule) return null + if (!nativeModule?.vectorscope) return null return nativeModule.vectorscope.process(leftChannel, rightChannel) }, reset: (): void => { - nativeModule?.vectorscope.reset() + nativeModule?.vectorscope?.reset() } } +export const waveform: WaveformNativeAnalyzer = { + isAvailable: (): boolean => { + return Boolean(nativeModule?.waveform) + }, + + configure: (sampleRate: number, samplesPerColumn: number): void => { + nativeModule?.waveform?.configure(sampleRate, samplesPerColumn) + }, + + processMono: (samples: Float32Array): Float32Array | null => { + if (!nativeModule?.waveform) return null + return nativeModule.waveform.processMono(samples) + }, + + processStereo: (leftChannel: Float32Array, rightChannel: Float32Array): Float32Array | null => { + if (!nativeModule?.waveform) return null + return nativeModule.waveform.processStereo(leftChannel, rightChannel) + }, + + reset: (): void => { + nativeModule?.waveform?.reset() + }, +} + export const vumeter: VUMeterNativeAnalyzer = { isAvailable: (): boolean => { return Boolean(nativeModule?.vumeter) @@ -300,5 +369,6 @@ export type { SpectrogramNativeResult, VectorscopeResult, VectorscopePointsResult, + VectorscopeMultibandPointsResult, VUMeterNativeSnapshot, } diff --git a/src/renderer/audio/native/visualizer-dsp.d.ts b/src/renderer/audio/native/visualizer-dsp.d.ts index b86f74f..3abb137 100644 --- a/src/renderer/audio/native/visualizer-dsp.d.ts +++ b/src/renderer/audio/native/visualizer-dsp.d.ts @@ -18,6 +18,11 @@ export interface VectorscopePointsResult { count: number; } +export interface VectorscopeMultibandPointsResult { + data: Float32Array; + count: number; +} + export interface SpectrogramNativeOptions { fftSize: number; sampleRate: number; @@ -117,14 +122,23 @@ export interface SpectrogramModule { export interface VectorscopeModule { setSampleRate(sampleRate: number): void; pushSamples(leftChannel: Float32Array, rightChannel: Float32Array): void; + pushMultibandSamples?: (leftChannel: Float32Array, rightChannel: Float32Array) => void; fillPoints(xOut: Float32Array, yOut: Float32Array): number; getPoints(maxPoints: number): VectorscopePointsResult; + getMultibandPoints?: (maxPoints: number) => VectorscopeMultibandPointsResult; setBufferSize(size: number): void; getBufferSize(): number; process(leftChannel: Float32Array, rightChannel: Float32Array): VectorscopeResult; reset(): void; } +export interface WaveformModule { + configure(sampleRate: number, samplesPerColumn: number): void; + processMono(samples: Float32Array): Float32Array; + processStereo(leftChannel: Float32Array, rightChannel: Float32Array): Float32Array; + reset(): void; +} + export interface LUFSMeterModule { setSampleRate(sampleRate: number): void; pushSamples(leftChannel: Float32Array, rightChannel: Float32Array): void; @@ -144,6 +158,7 @@ export interface VisualizerDSP { spectrum: SpectrumModule; spectrogram: SpectrogramModule; vectorscope: VectorscopeModule; + waveform?: WaveformModule; vumeter: VUMeterModule; lufsmeter: LUFSMeterModule; } diff --git a/src/renderer/visualizers/Vectorscope.ts b/src/renderer/visualizers/Vectorscope.ts index b4f3ae9..054d2a2 100644 --- a/src/renderer/visualizers/Vectorscope.ts +++ b/src/renderer/visualizers/Vectorscope.ts @@ -1,5 +1,5 @@ import { audioRouter } from '../audio/AudioRouter' -import { vectorscope as nativeVectorscope, isNativeAvailable } from '../audio/native' +import { vectorscope as nativeVectorscope, type VectorscopeNativeAnalyzer } from '../audio/native' import { drawVectorscopeGridForMode, getVectorscopeLayout, @@ -35,9 +35,10 @@ export interface VectorscopeOptions { multiband?: boolean dataSource?: VectorscopeDataSource frameScheduler?: FrameScheduler + nativeAnalyzer?: VectorscopeNativeAnalyzer | null } -type ResolvedVectorscopeOptions = Required> +type ResolvedVectorscopeOptions = Required> const defaultOptions: ResolvedVectorscopeOptions = { lineColor: '#00ffff', @@ -74,6 +75,7 @@ export class Vectorscope { private staticLayerCtx: CanvasRenderingContext2D private options: ResolvedVectorscopeOptions private dataSource: VectorscopeDataSource + private nativeAnalyzer: VectorscopeNativeAnalyzer | null private frameLoop: VisualizerFrameLoop private nativeInitialized = false private lastSampleRate = 0 @@ -94,9 +96,10 @@ export class Vectorscope { if (!ctx) throw new Error('Could not get 2D context') this.ctx = ctx - const { dataSource, frameScheduler, ...optionOverrides } = options + const { dataSource, frameScheduler, nativeAnalyzer, ...optionOverrides } = options this.options = { ...defaultOptions, ...optionOverrides } this.dataSource = dataSource ?? defaultVectorscopeDataSource + this.nativeAnalyzer = nativeAnalyzer === undefined ? nativeVectorscope : nativeAnalyzer this.frameLoop = new VisualizerFrameLoop({ frameScheduler, shouldRun: () => this.dataSource.isPlaying(), @@ -128,13 +131,13 @@ export class Vectorscope { } private initNative(): void { - if (isNativeAvailable() && !this.nativeInitialized) { + if (this.isNativeAvailable() && !this.nativeInitialized) { const sampleRate = this.dataSource.getSampleRate() this.lastSampleRate = sampleRate - nativeVectorscope.setSampleRate(sampleRate) + this.nativeAnalyzer?.setSampleRate(sampleRate) this.nativeInitialized = true console.log(`Vectorscope: Using native DSP (${sampleRate}Hz)`) - } else if (!isNativeAvailable()) { + } else if (!this.isNativeAvailable()) { console.log('Vectorscope: Using JavaScript fallback') } } @@ -143,16 +146,16 @@ export class Vectorscope { const currentRate = this.dataSource.getSampleRate() if (currentRate !== this.lastSampleRate && currentRate > 0) { this.lastSampleRate = currentRate - if (isNativeAvailable()) { - nativeVectorscope.setSampleRate(currentRate) + if (this.isNativeAvailable()) { + this.nativeAnalyzer?.setSampleRate(currentRate) } this.splitter.configure(currentRate) } } private resetDisplay(): void { - if (isNativeAvailable()) { - nativeVectorscope.reset() + if (this.isNativeAvailable()) { + this.nativeAnalyzer?.reset() } this.splitter.reset() this.multibandBuffer.reset() @@ -161,11 +164,27 @@ export class Vectorscope { } setOptions(options: Partial): void { - const { dataSource, frameScheduler: _frameScheduler, ...optionUpdates } = options - this.options = { ...this.options, ...optionUpdates } + const { dataSource, frameScheduler: _frameScheduler, nativeAnalyzer, ...optionUpdates } = options + const nextOptions: ResolvedVectorscopeOptions = { ...this.options, ...optionUpdates } + const multibandChanged = nextOptions.multiband !== this.options.multiband + const modeChanged = nextOptions.mode !== this.options.mode + this.options = nextOptions + let shouldResetDisplay = false + if (nativeAnalyzer !== undefined && nativeAnalyzer !== this.nativeAnalyzer) { + this.nativeAnalyzer = nativeAnalyzer + this.nativeInitialized = false + this.initNative() + shouldResetDisplay = true + } if (dataSource && dataSource !== this.dataSource) { this.dataSource = dataSource this.subscribeToSessionChanges() + shouldResetDisplay = true + } + if (multibandChanged || modeChanged) { + shouldResetDisplay = true + } + if (shouldResetDisplay) { this.resetDisplay() } this.staticLayerKey = '' @@ -224,11 +243,13 @@ export class Vectorscope { const pendingSamples = this.dataSource.getPendingVectorscopeSamples() if (options.multiband) { - this.drawMultibandPoints(offscreenCtx, pendingSamples, centerX, centerY, scale) - } else if (isNativeAvailable()) { + if (!this.drawNativeMultibandPoints(offscreenCtx, pendingSamples, centerX, centerY, scale)) { + this.drawMultibandPoints(offscreenCtx, pendingSamples, centerX, centerY, scale) + } + } else if (this.isNativeAvailable()) { if (pendingSamples.length > 0) { const { left, right } = this.concatStereoChunks(pendingSamples) - nativeVectorscope.pushSamples(left, right) + this.nativeAnalyzer?.pushSamples(left, right) } const count = this.fillNativePoints(options.displayPoints) @@ -243,6 +264,17 @@ export class Vectorscope { ctx.drawImage(offscreenCanvas, 0, 0) } + private isNativeAvailable(): boolean { + return Boolean(this.nativeAnalyzer) && this.nativeAnalyzer?.isAvailable?.() !== false + } + + private isNativeMultibandAvailable(): boolean { + return this.isNativeAvailable() + && Boolean(this.nativeAnalyzer?.pushMultibandSamples) + && Boolean(this.nativeAnalyzer?.getMultibandPoints) + && this.nativeAnalyzer?.isMultibandAvailable?.() !== false + } + private renderStaticLayer(): void { this.ensureStaticLayer() this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height) @@ -401,6 +433,66 @@ export class Vectorscope { ctx.globalAlpha = 1.0 } + private drawNativeMultibandPoints( + ctx: CanvasRenderingContext2D, + pendingSamples: { left: Float32Array; right: Float32Array }[], + centerX: number, + centerY: number, + scale: number + ): boolean { + if (!this.isNativeMultibandAvailable()) { + return false + } + + if (pendingSamples.length > 0) { + const { left, right } = this.concatStereoChunks(pendingSamples) + this.nativeAnalyzer?.pushMultibandSamples?.(left, right) + } + + const result = this.nativeAnalyzer?.getMultibandPoints?.(this.options.displayPoints) + if (!result) { + return false + } + + const count = Math.min(result.count, Math.floor(result.data.length / 6), this.options.displayPoints) + if (count === 0) { + return true + } + + const mode = this.options.mode + const dpr = window.devicePixelRatio || 1 + const dotSize = this.options.lineWidth * dpr + const segments = 8 + const pointsPerSegment = Math.ceil(count / segments) + const bandOffsets: Record<(typeof BAND_ORDER)[number], [number, number]> = { + low: [0, 1], + mid: [2, 3], + high: [4, 5], + } + + for (let seg = 0; seg < segments; seg++) { + const startIdx = seg * pointsPerSegment + const endIdx = Math.min((seg + 1) * pointsPerSegment, count) + if (startIdx >= count) break + + const alpha = 0.15 + 0.85 * (seg / Math.max(segments - 1, 1)) + ctx.globalAlpha = alpha + + for (const band of BAND_ORDER) { + const [leftOffset, rightOffset] = bandOffsets[band] + ctx.fillStyle = this.options.bandColors[band] + + for (let i = startIdx; i < endIdx; i++) { + const offset = i * 6 + this.drawProjectedDot(ctx, result.data[offset + leftOffset], result.data[offset + rightOffset], mode, centerX, centerY, scale, dotSize) + } + } + } + + ctx.globalAlpha = 1.0 + return true + } + private ensureNativePointBuffers(displayPoints: number): void { if (this.nativePointX.length !== displayPoints) { this.nativePointX = new Float32Array(displayPoints) @@ -410,7 +502,7 @@ export class Vectorscope { private fillNativePoints(displayPoints: number): number { this.ensureNativePointBuffers(displayPoints) - return nativeVectorscope.fillPoints(this.nativePointX, this.nativePointY) + return this.nativeAnalyzer?.fillPoints(this.nativePointX, this.nativePointY) ?? 0 } private ensureMultibandScratch(leftLength: number, rightLength: number): MultibandChunk { @@ -489,8 +581,8 @@ export class Vectorscope { this.unsubscribeSessionChange = null } - if (isNativeAvailable()) { - nativeVectorscope.reset() + if (this.isNativeAvailable()) { + this.nativeAnalyzer?.reset() } } } diff --git a/src/renderer/visualizers/Waveform.ts b/src/renderer/visualizers/Waveform.ts index 915eb2b..a433917 100644 --- a/src/renderer/visualizers/Waveform.ts +++ b/src/renderer/visualizers/Waveform.ts @@ -1,4 +1,5 @@ import { audioRouter } from '../audio/AudioRouter' +import { waveform as nativeWaveform, type WaveformNativeAnalyzer } from '../audio/native' import { resolveColorToRgb } from '../utils/color' import { defaultVisualizerSessionSource, type VisualizerSessionSource } from './dataSource' import { FrameScheduler } from './frameScheduler' @@ -36,9 +37,10 @@ export interface WaveformOptions { multiband?: boolean dataSource?: WaveformDataSource frameScheduler?: FrameScheduler + nativeAnalyzer?: WaveformNativeAnalyzer | null } -type ResolvedWaveformOptions = Required> +type ResolvedWaveformOptions = Required> const defaultOptions: ResolvedWaveformOptions = { backgroundColor: 'transparent', @@ -74,6 +76,7 @@ export class Waveform { private ctx: CanvasRenderingContext2D private options: ResolvedWaveformOptions private dataSource: WaveformDataSource + private nativeAnalyzer: WaveformNativeAnalyzer | null private frameLoop: VisualizerFrameLoop private waterfallCanvas: HTMLCanvasElement @@ -105,7 +108,7 @@ export class Waveform { this.ctx = ctx this.ctx.imageSmoothingEnabled = false - const { dataSource, frameScheduler, ...optionOverrides } = options + const { dataSource, frameScheduler, nativeAnalyzer, ...optionOverrides } = options this.options = { ...defaultOptions, ...optionOverrides, @@ -114,6 +117,7 @@ export class Waveform { multiband: optionOverrides.multiband ?? defaultOptions.multiband, } this.dataSource = dataSource ?? defaultWaveformDataSource + this.nativeAnalyzer = nativeAnalyzer === undefined ? nativeWaveform : nativeAnalyzer this.frameLoop = new VisualizerFrameLoop({ frameScheduler, shouldRun: () => this.dataSource.isPlaying(), @@ -150,9 +154,18 @@ export class Waveform { this.waterfallCtx.clearRect(0, 0, this.waterfallCanvas.width, this.waterfallCanvas.height) this.columnAccumulatorPos = 0 this.splitter.reset() + this.nativeAnalyzer?.reset() + this.configureNativeAnalyzer() this.invalidate() } + private configureNativeAnalyzer(): void { + if (this.nativeAnalyzer?.isAvailable?.() === false) { + return + } + this.nativeAnalyzer?.configure(this.lastSampleRate || Math.max(1, this.dataSource.getSampleRate()), this.samplesPerColumn) + } + private recomputeSamplesPerColumn(): void { const sampleRate = Math.max(1, this.dataSource.getSampleRate()) const pixelsPerSecond = BASE_PIXELS_PER_SECOND * this.options.scrollSpeed @@ -171,10 +184,11 @@ export class Waveform { } this.lastSampleRate = sampleRate this.splitter.configure(sampleRate) + this.configureNativeAnalyzer() } setOptions(options: Partial): void { - const { dataSource, frameScheduler: _frameScheduler, ...optionUpdates } = options + const { dataSource, frameScheduler: _frameScheduler, nativeAnalyzer, ...optionUpdates } = options const nextOptions: ResolvedWaveformOptions = { ...this.options, ...optionUpdates, @@ -187,9 +201,16 @@ export class Waveform { const multibandChanged = nextOptions.multiband !== this.options.multiband const modeChanged = nextOptions.mode !== this.options.mode const dataSourceChanged = Boolean(dataSource && dataSource !== this.dataSource) + const nativeAnalyzerChanged = nativeAnalyzer !== undefined && nativeAnalyzer !== this.nativeAnalyzer this.options = nextOptions + if (nativeAnalyzerChanged) { + this.nativeAnalyzer = nativeAnalyzer + this.nativeAnalyzer?.reset() + this.configureNativeAnalyzer() + } + if (dataSourceChanged && dataSource) { this.dataSource = dataSource this.subscribeToSessionChanges() @@ -201,10 +222,12 @@ export class Waveform { if (multibandChanged || modeChanged) { this.splitter.reset() + this.nativeAnalyzer?.reset() + this.configureNativeAnalyzer() } this.staticLayerKey = '' - if (dataSourceChanged || speedChanged || multibandChanged || modeChanged) { + if (dataSourceChanged || speedChanged || multibandChanged || modeChanged || nativeAnalyzerChanged) { this.resetDisplay() } @@ -248,11 +271,8 @@ export class Waveform { midBandSamples: Float32Array, highBandSamples: Float32Array, ): [number, number, number] { - const lowBand = this.toBandColorTuple(this.options.bandColors.low) - const midBand = this.toBandColorTuple(this.options.bandColors.mid) - const highBand = this.toBandColorTuple(this.options.bandColors.high) const n = this.columnAccumulatorPos - if (n === 0) return midBand + if (n === 0) return this.toBandColorTuple(this.options.bandColors.mid) let lowSum = 0 let midSum = 0 @@ -266,9 +286,21 @@ export class Waveform { highSum += high * high } - const lowRms = Math.sqrt(lowSum / n) - const midRms = Math.sqrt(midSum / n) - const highRms = Math.sqrt(highSum / n) + return this.computeBandColorFromRms( + Math.sqrt(lowSum / n), + Math.sqrt(midSum / n), + Math.sqrt(highSum / n), + ) + } + + private computeBandColorFromRms( + lowRms: number, + midRms: number, + highRms: number, + ): [number, number, number] { + const lowBand = this.toBandColorTuple(this.options.bandColors.low) + const midBand = this.toBandColorTuple(this.options.bandColors.mid) + const highBand = this.toBandColorTuple(this.options.bandColors.high) const total = lowRms + midRms + highRms if (total < 1e-10) return midBand @@ -311,6 +343,15 @@ export class Waveform { ] } + private resolveNativeColumnColor(lowRms: number, midRms: number, highRms: number): [number, number, number] { + if (this.options.multiband) { + return this.computeBandColorFromRms(lowRms, midRms, highRms) + } + + const lineColor = resolveColorToRgb(this.options.lineColor) + return [lineColor.r, lineColor.g, lineColor.b] + } + private toBandColorTuple(color: string): [number, number, number] { const { r, g, b } = resolveColorToRgb(color) return [r, g, b] @@ -454,6 +495,12 @@ export class Waveform { } private processMonoChunk(chunk: Float32Array, width: number, height: number): void { + if (this.useNativeMultiband()) { + if (this.processNativeMonoChunk(chunk, width, height)) { + return + } + } + let lowBand: Float32Array | null = null let midBand: Float32Array | null = null let highBand: Float32Array | null = null @@ -493,6 +540,12 @@ export class Waveform { const leftSamples = chunk.left.length === length ? chunk.left : chunk.left.subarray(0, length) const rightSamples = chunk.right.length === length ? chunk.right : chunk.right.subarray(0, length) + if (this.useNativeMultiband()) { + if (this.processNativeStereoChunk(leftSamples, rightSamples, width, height)) { + return + } + } + let lowLeft: Float32Array | null = null let midLeft: Float32Array | null = null let highLeft: Float32Array | null = null @@ -537,6 +590,47 @@ export class Waveform { } } + private useNativeMultiband(): boolean { + return this.options.multiband && Boolean(this.nativeAnalyzer) && this.nativeAnalyzer?.isAvailable?.() !== false + } + + private processNativeMonoChunk(chunk: Float32Array, width: number, height: number): boolean { + const summaries = this.nativeAnalyzer?.processMono(chunk) + if (!summaries) { + return false + } + + const stride = 5 + const columnCount = Math.floor(summaries.length / stride) + for (let column = 0; column < columnCount; column += 1) { + const offset = column * stride + const color = this.resolveNativeColumnColor(summaries[offset + 2], summaries[offset + 3], summaries[offset + 4]) + this.shiftWaterfall() + this.paintColumn(summaries[offset], summaries[offset + 1], width, 0, height, color) + } + return true + } + + private processNativeStereoChunk(leftSamples: Float32Array, rightSamples: Float32Array, width: number, height: number): boolean { + const summaries = this.nativeAnalyzer?.processStereo(leftSamples, rightSamples) + if (!summaries) { + return false + } + + const stride = 10 + const laneHeight = height / 2 + const columnCount = Math.floor(summaries.length / stride) + for (let column = 0; column < columnCount; column += 1) { + const offset = column * stride + const leftColor = this.resolveNativeColumnColor(summaries[offset + 2], summaries[offset + 3], summaries[offset + 4]) + const rightColor = this.resolveNativeColumnColor(summaries[offset + 7], summaries[offset + 8], summaries[offset + 9]) + this.shiftWaterfall() + this.paintColumn(summaries[offset], summaries[offset + 1], width, 0, laneHeight, leftColor) + this.paintColumn(summaries[offset + 5], summaries[offset + 6], width, laneHeight, laneHeight, rightColor) + } + return true + } + private ensureMultibandScratch(length: number): MultibandChunk { if (this.multibandScratch.low.left.length < length) { this.multibandScratch = createMultibandChunk(length) @@ -613,6 +707,7 @@ export class Waveform { dispose(): void { this.stop() this.frameLoop.dispose() + this.nativeAnalyzer?.reset() if (this.unsubscribeSessionChange) { this.unsubscribeSessionChange() this.unsubscribeSessionChange = null diff --git a/test/renderer-helpers.test.ts b/test/renderer-helpers.test.ts index d4522d9..a943e27 100644 --- a/test/renderer-helpers.test.ts +++ b/test/renderer-helpers.test.ts @@ -81,8 +81,10 @@ import type { SpectrogramNativeAnalyzer, SpectrogramNativeOptions, SpectrogramNativeResult, + VectorscopeNativeAnalyzer, VUMeterNativeAnalyzer, VUMeterNativeSnapshot, + WaveformNativeAnalyzer, } from '../src/renderer/audio/native' import { HEAT_LOW_DB, @@ -96,6 +98,7 @@ import { Oscilloscope } from '../src/renderer/visualizers/Oscilloscope' import { SpectrumAnalyzer, type SpectrumAnalyzerOptions } from '../src/renderer/visualizers/SpectrumAnalyzer' import { Spectrogram, type SpectrogramOptions } from '../src/renderer/visualizers/Spectrogram' import { Vectorscope } from '../src/renderer/visualizers/Vectorscope' +import { Waveform } from '../src/renderer/visualizers/Waveform' import { VUMeter, VU_NEEDLE_FACE_HEIGHT_CSS_PX, @@ -4029,6 +4032,260 @@ test('MultibandSplitter and MultibandBuffer reuse caller-owned buffers', () => { assert.notEqual(pointTarget.low.left[0], 0) }) +function createFakeWaveformNativeAnalyzer(options: { + available?: boolean + monoSummary?: Float32Array + stereoSummary?: Float32Array +} = {}): WaveformNativeAnalyzer & { + configs: Array<{ sampleRate: number; samplesPerColumn: number }> + monoPushes: Float32Array[] + stereoPushes: Array<{ left: Float32Array; right: Float32Array }> + resetCount: number +} { + return { + configs: [], + monoPushes: [], + stereoPushes: [], + resetCount: 0, + isAvailable: () => options.available ?? true, + configure(sampleRate, samplesPerColumn) { + this.configs.push({ sampleRate, samplesPerColumn }) + }, + processMono(samples) { + this.monoPushes.push(samples) + return options.monoSummary ?? new Float32Array(0) + }, + processStereo(left, right) { + this.stereoPushes.push({ left, right }) + return options.stereoSummary ?? new Float32Array(0) + }, + reset() { + this.resetCount += 1 + }, + } +} + +function createFakeVectorscopeNativeAnalyzer(options: { + multibandAvailable?: boolean + multibandData?: Float32Array + multibandCount?: number +} = {}): VectorscopeNativeAnalyzer & { + sampleRates: number[] + multibandPushes: Array<{ left: Float32Array; right: Float32Array }> + multibandRequests: number[] + resetCount: number +} { + const analyzer: VectorscopeNativeAnalyzer & { + sampleRates: number[] + multibandPushes: Array<{ left: Float32Array; right: Float32Array }> + multibandRequests: number[] + resetCount: number + } = { + sampleRates: [], + multibandPushes: [], + multibandRequests: [], + resetCount: 0, + isAvailable: () => true, + isMultibandAvailable: () => options.multibandAvailable ?? true, + setSampleRate(sampleRate) { + this.sampleRates.push(sampleRate) + }, + pushSamples() {}, + fillPoints() { + return 0 + }, + reset() { + this.resetCount += 1 + }, + } + + if (options.multibandAvailable !== false) { + analyzer.pushMultibandSamples = (left, right) => { + analyzer.multibandPushes.push({ left, right }) + } + analyzer.getMultibandPoints = (maxPoints) => { + analyzer.multibandRequests.push(maxPoints) + const data = options.multibandData ?? new Float32Array(0) + return { + data, + count: options.multibandCount ?? Math.floor(data.length / 6), + } + } + } + + return analyzer +} + +test('Waveform uses native multiband column summaries when available', () => { + const recorder = createFakeCanvasRecorder() + const dom = installFakeCanvasDom(() => createFakeCanvas(recorder)) + const nativeAnalyzer = createFakeWaveformNativeAnalyzer({ + monoSummary: new Float32Array([-0.5, 0.5, 1, 0, 0]), + }) + const dataSource = { + getPendingWaveformSamples: () => [new Float32Array([0.25])], + getPendingWaveformStereoSamples: () => [], + getSampleRate: () => 64, + isPlaying: () => true, + subscribeToSessionChanges: () => () => {}, + } + const waveform = new Waveform(createFakeCanvas(recorder), { + dataSource, + multiband: true, + nativeAnalyzer, + bandColors: { + low: '#ff0000', + mid: '#00ff00', + high: '#0000ff', + }, + }) + + try { + ;(waveform as unknown as { drawFrame: () => void }).drawFrame() + assert.equal(nativeAnalyzer.monoPushes.length, 1) + assert.equal(nativeAnalyzer.configs.at(-1)?.samplesPerColumn, 1) + assert.equal( + recorder.fillRects.some((rect) => rect.fillStyle === 'rgba(235, 20, 0, 0.72)'), + true, + ) + } finally { + waveform.dispose() + dom.restore() + } +}) + +test('Waveform falls back to JavaScript multiband when native is unavailable', () => { + const recorder = createFakeCanvasRecorder() + const dom = installFakeCanvasDom(() => createFakeCanvas(recorder)) + const nativeAnalyzer = createFakeWaveformNativeAnalyzer({ available: false }) + const dataSource = { + getPendingWaveformSamples: () => [new Float32Array([0.75])], + getPendingWaveformStereoSamples: () => [], + getSampleRate: () => 64, + isPlaying: () => true, + subscribeToSessionChanges: () => () => {}, + } + const waveform = new Waveform(createFakeCanvas(recorder), { + dataSource, + multiband: true, + nativeAnalyzer, + }) + + try { + ;(waveform as unknown as { drawFrame: () => void }).drawFrame() + assert.equal(nativeAnalyzer.monoPushes.length, 0) + assert.equal(recorder.fillRects.some((rect) => rect.width === 1), true) + } finally { + waveform.dispose() + dom.restore() + } +}) + +test('Waveform reconfigures and resets native multiband state on option changes', () => { + const dom = installFakeCanvasDom() + const nativeAnalyzer = createFakeWaveformNativeAnalyzer() + const dataSource = { + getPendingWaveformSamples: () => [], + getPendingWaveformStereoSamples: () => [], + getSampleRate: () => 128, + isPlaying: () => true, + subscribeToSessionChanges: () => () => {}, + } + const waveform = new Waveform(createFakeCanvas(), { + dataSource, + multiband: true, + nativeAnalyzer, + }) + + try { + nativeAnalyzer.configs.length = 0 + nativeAnalyzer.resetCount = 0 + waveform.setOptions({ scrollSpeed: 2 }) + assert.equal(nativeAnalyzer.configs.at(-1)?.samplesPerColumn, 1) + assert.equal(nativeAnalyzer.resetCount > 0, true) + + waveform.setOptions({ multiband: false }) + assert.equal(nativeAnalyzer.resetCount > 1, true) + } finally { + waveform.dispose() + dom.restore() + } +}) + +test('Vectorscope multiband uses native interleaved band points when available', () => { + const recorder = createFakeCanvasRecorder() + const dom = installFakeCanvasDom(() => createFakeCanvas(recorder)) + const nativeAnalyzer = createFakeVectorscopeNativeAnalyzer({ + multibandData: new Float32Array([0.1, 0.2, 0.2, 0.1, -0.1, 0.15]), + multibandCount: 1, + }) + const dataSource = { + getPendingVectorscopeSamples: () => [{ left: new Float32Array([0.2]), right: new Float32Array([0.1]) }], + getSampleRate: () => 48000, + isPlaying: () => true, + subscribeToSessionChanges: () => () => {}, + } + const vectorscope = new Vectorscope(createFakeCanvas(recorder), { + dataSource, + multiband: true, + showGrid: false, + displayPoints: 1, + nativeAnalyzer, + bandColors: { + low: '#110000', + mid: '#001100', + high: '#000011', + }, + }) + + try { + ;(vectorscope as unknown as { drawFrame: () => void }).drawFrame() + assert.equal(nativeAnalyzer.multibandPushes.length, 1) + assert.deepEqual(nativeAnalyzer.multibandRequests, [1]) + assert.equal(recorder.fillRects.some((rect) => rect.fillStyle === '#110000'), true) + assert.equal(recorder.fillRects.some((rect) => rect.fillStyle === '#001100'), true) + assert.equal(recorder.fillRects.some((rect) => rect.fillStyle === '#000011'), true) + } finally { + vectorscope.dispose() + dom.restore() + } +}) + +test('Vectorscope multiband falls back to JavaScript splitter when native multiband is missing', () => { + const recorder = createFakeCanvasRecorder() + const dom = installFakeCanvasDom(() => createFakeCanvas(recorder)) + const nativeAnalyzer = createFakeVectorscopeNativeAnalyzer({ multibandAvailable: false }) + const dataSource = { + getPendingVectorscopeSamples: () => [createSineStereoChunk(440, 48000, 8)], + getSampleRate: () => 48000, + isPlaying: () => true, + subscribeToSessionChanges: () => () => {}, + } + const vectorscope = new Vectorscope(createFakeCanvas(recorder), { + dataSource, + multiband: true, + showGrid: false, + displayPoints: 8, + nativeAnalyzer, + bandColors: { + low: '#210000', + mid: '#002100', + high: '#000021', + }, + }) + + try { + ;(vectorscope as unknown as { drawFrame: () => void }).drawFrame() + assert.equal(nativeAnalyzer.multibandPushes.length, 0) + assert.equal(recorder.fillRects.some((rect) => rect.fillStyle === '#210000'), true) + assert.equal(recorder.fillRects.some((rect) => rect.fillStyle === '#002100'), true) + assert.equal(recorder.fillRects.some((rect) => rect.fillStyle === '#000021'), true) + } finally { + vectorscope.dispose() + dom.restore() + } +}) + function createFakeVUMeterNativeAnalyzer( snapshotOverrides: Partial = {}, available = true,