From 9b22400747c0b64e0265d15982c6b2f7b7a3868f Mon Sep 17 00:00:00 2001 From: Boof2015 <75185879+Boof2015@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:42:43 -0400 Subject: [PATCH] significantly improve visual performance --- native/src/main.cpp | 21 ++ native/src/spectrum.cpp | 25 +- native/src/spectrum.h | 9 +- src/renderer/audio/AudioCapture.ts | 29 +- src/renderer/audio/AudioRouter.ts | 28 +- .../audio/NativeVisualizerTransport.ts | 231 ++++++++++++ src/renderer/audio/native/index.ts | 9 + src/renderer/audio/native/visualizer-dsp.d.ts | 2 + src/renderer/popouts/ScopePopoutDataSource.ts | 28 +- src/renderer/visualizers/Oscilloscope.ts | 5 +- src/renderer/visualizers/SpectrumAnalyzer.ts | 14 +- src/renderer/visualizers/Vectorscope.ts | 7 +- src/renderer/visualizers/dataSource.ts | 4 + test/audio-router.test.ts | 51 +++ test/renderer-helpers.test.ts | 335 ++++++++++++++++++ 15 files changed, 778 insertions(+), 20 deletions(-) create mode 100644 src/renderer/audio/NativeVisualizerTransport.ts diff --git a/native/src/main.cpp b/native/src/main.cpp index 4ca598c..716abbc 100644 --- a/native/src/main.cpp +++ b/native/src/main.cpp @@ -139,6 +139,25 @@ Napi::Value SpectrumSetSmoothing(const Napi::CallbackInfo& info) { return env.Undefined(); } +Napi::Value SpectrumPushSamples(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() < 1 || !info[0].IsTypedArray()) { + Napi::TypeError::New(env, "Expected Float32Array").ThrowAsJavaScriptException(); + return env.Null(); + } + Napi::Float32Array audioData = info[0].As(); + spectrum.pushSamples(audioData.Data(), audioData.ElementLength()); + return env.Undefined(); +} + +Napi::Value SpectrumGetMagnitudes(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + const auto& magnitudes = spectrum.getMagnitudes(); + Napi::Float32Array result = Napi::Float32Array::New(env, magnitudes.size()); + memcpy(result.Data(), magnitudes.data(), magnitudes.size() * sizeof(float)); + return result; +} + Napi::Value SpectrumProcess(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (info.Length() < 1 || !info[0].IsTypedArray()) { @@ -280,6 +299,8 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) { specExports.Set("getFFTSize", Napi::Function::New(env, SpectrumGetFFTSize)); specExports.Set("setSampleRate", Napi::Function::New(env, SpectrumSetSampleRate)); specExports.Set("setSmoothing", Napi::Function::New(env, SpectrumSetSmoothing)); + specExports.Set("pushSamples", Napi::Function::New(env, SpectrumPushSamples)); + specExports.Set("getMagnitudes", Napi::Function::New(env, SpectrumGetMagnitudes)); specExports.Set("process", Napi::Function::New(env, SpectrumProcess)); specExports.Set("binToFrequency", Napi::Function::New(env, SpectrumBinToFrequency)); specExports.Set("reset", Napi::Function::New(env, SpectrumReset)); diff --git a/native/src/spectrum.cpp b/native/src/spectrum.cpp index c3d0475..c86a1f4 100644 --- a/native/src/spectrum.cpp +++ b/native/src/spectrum.cpp @@ -55,7 +55,7 @@ void Spectrum::applyWindow(const float* input, float* output, size_t length) { } } -void Spectrum::pushSamples(const float* input, size_t length) { +void Spectrum::pushHistory(const float* input, size_t length) { if (length == 0 || fftSize_ == 0) { return; } @@ -73,13 +73,9 @@ void Spectrum::pushSamples(const float* input, size_t length) { bufferedSamples_ = std::min(fftSize_, bufferedSamples_ + length); } -const std::vector& Spectrum::process(const float* audioData, size_t length) { - if (audioData != nullptr && length > 0) { - pushSamples(audioData, length); - } - +void Spectrum::updateMagnitudes() { if (historyBuffer_.empty() || magnitudes_.empty()) { - return smoothedMagnitudes_; + return; } // Always analyze a full FFT frame from the rolling buffer. @@ -115,6 +111,21 @@ const std::vector& Spectrum::process(const float* audioData, size_t lengt smoothedMagnitudes_[i] = -100.0f; } } +} + +void Spectrum::pushSamples(const float* input, size_t length) { + if (input != nullptr && length > 0) { + pushHistory(input, length); + } + updateMagnitudes(); +} + +const std::vector& Spectrum::process(const float* audioData, size_t length) { + if (audioData != nullptr && length > 0) { + pushHistory(audioData, length); + } + + updateMagnitudes(); return smoothedMagnitudes_; } diff --git a/native/src/spectrum.h b/native/src/spectrum.h index 8d7322b..edb8911 100644 --- a/native/src/spectrum.h +++ b/native/src/spectrum.h @@ -16,6 +16,12 @@ public: void setSampleRate(float sampleRate); void setSmoothing(float smoothing); // 0.0 - 1.0 + // Feed new samples into the rolling history and update the latest magnitudes. + void pushSamples(const float* input, size_t length); + + // Read the latest smoothed magnitudes without mutating analyzer state. + const std::vector& getMagnitudes() const { return smoothedMagnitudes_; } + // Process audio and get spectrum data // Returns magnitude data (size = fftSize / 2) const std::vector& process(const float* audioData, size_t length); @@ -39,7 +45,8 @@ private: size_t bufferedSamples_; void applyWindow(const float* input, float* output, size_t length); - void pushSamples(const float* input, size_t length); + void pushHistory(const float* input, size_t length); + void updateMagnitudes(); }; } // namespace Visualizer diff --git a/src/renderer/audio/AudioCapture.ts b/src/renderer/audio/AudioCapture.ts index 91c8747..0a6d99b 100644 --- a/src/renderer/audio/AudioCapture.ts +++ b/src/renderer/audio/AudioCapture.ts @@ -5,6 +5,7 @@ */ import { audioRouter } from './AudioRouter' +import { nativeVisualizerTransport } from './NativeVisualizerTransport' import { applyInputGainToStereoSamples, inputGainDbToLinear } from './inputGain' import type { CaptureBackendKind, @@ -52,6 +53,17 @@ interface CaptureBackend { getStatus(): CaptureBackendStatus } +function isDocumentHidden(): boolean { + return typeof document !== 'undefined' && document.hidden === true +} + +export function resolveNativeCapturePollDelay(chunkCount: number): number { + if (isDocumentHidden()) { + return 16 + } + return chunkCount > 0 ? 0 : 2 +} + export interface CaptureManagerStatus { captureMode: CaptureMode backendPolicy: CaptureBackendPolicy @@ -366,7 +378,7 @@ class ElectronDeviceCaptureBackend implements CaptureBackend { } } -abstract class NativePolledCaptureBackend implements CaptureBackend { +export abstract class NativePolledCaptureBackend implements CaptureBackend { abstract readonly kind: CaptureBackendKind private readonly chunkListeners = new Set<(chunk: CaptureChunk) => void>() @@ -464,8 +476,10 @@ abstract class NativePolledCaptureBackend implements CaptureBackend { const poll = (): void => { if (!this.active) return + let chunkCount = 0 try { const result = this.getNativeCaptureModule()?.drain(32) as NativeCaptureDrainResult | undefined + chunkCount = result?.chunks.length ?? 0 if (result) { for (const chunk of result.chunks) { const routedChunk: CaptureChunk = { @@ -488,7 +502,7 @@ abstract class NativePolledCaptureBackend implements CaptureBackend { return } - this.pollTimer = window.setTimeout(poll, 4) + this.pollTimer = window.setTimeout(poll, resolveNativeCapturePollDelay(chunkCount)) } this.pollTimer = window.setTimeout(poll, 0) @@ -592,6 +606,11 @@ class AudioCapture { this.electronSystemBackend.subscribe((chunk) => this.handleChunk(this.electronSystemBackend.kind, chunk)) this.electronDeviceBackend.subscribe((chunk) => this.handleChunk(this.electronDeviceBackend.kind, chunk)) + + audioRouter.subscribeToDemandChanges((demand) => { + nativeVisualizerTransport.setDemand(demand) + }) + nativeVisualizerTransport.reset(audioRouter.getSessionState()) } subscribeStatus(listener: StatusListener): () => void { @@ -653,6 +672,7 @@ class AudioCapture { backendStatus.channelCount, backend.kind, ) + nativeVisualizerTransport.reset(audioRouter.getSessionState()) this.emitStatus() return } catch (error) { @@ -813,6 +833,7 @@ class AudioCapture { private async stopActiveCapture(): Promise { if (this.sessionId !== null) { audioRouter.endSession() + nativeVisualizerTransport.reset(audioRouter.getSessionState()) this.sessionId = null } @@ -838,6 +859,10 @@ class AudioCapture { capturedAt: chunk.capturedAt, sequence: chunk.sequence, }) + nativeVisualizerTransport.handleChunk(chunk.left, chunk.right, { + sessionId: this.sessionId, + channelCount: chunk.channelCount, + }) } setInputGain(db: number): void { diff --git a/src/renderer/audio/AudioRouter.ts b/src/renderer/audio/AudioRouter.ts index 30d54d5..4215d90 100644 --- a/src/renderer/audio/AudioRouter.ts +++ b/src/renderer/audio/AudioRouter.ts @@ -60,6 +60,8 @@ export interface AudioRouterDiagnostics { scopes: Record } +export type NormalizedVisualizerConsumerDemand = Required + interface AudioChunkMeta { sessionId?: number channelCount?: number @@ -187,7 +189,7 @@ class RollingLatencyWindow { } } -function createEmptyDemand(): VisualizerConsumerDemand { +function createEmptyDemand(): NormalizedVisualizerConsumerDemand { return { spectrum: false, oscilloscope: false, @@ -242,6 +244,7 @@ export class AudioRouter { private undemandedChunks = 0 private sessionListeners = new Set<(state: AudioSessionState) => void>() + private demandListeners = new Set<(demand: NormalizedVisualizerConsumerDemand) => void>() private emitSessionState(): void { const state = this.getSessionState() @@ -307,6 +310,18 @@ export class AudioRouter { } } + subscribeToDemandChanges(listener: (demand: NormalizedVisualizerConsumerDemand) => void): () => void { + this.demandListeners.add(listener) + listener(this.getActiveVisualizerDemand()) + return () => { + this.demandListeners.delete(listener) + } + } + + getActiveVisualizerDemand(): NormalizedVisualizerConsumerDemand { + return this.getActiveDemand() + } + setVisualizerConsumerDemand(consumerId: string, demand: VisualizerConsumerDemand): void { const normalized: VisualizerConsumerDemand = { spectrum: Boolean(demand.spectrum), @@ -326,11 +341,13 @@ export class AudioRouter { } this.pruneQueuesForDemand() + this.emitDemandState() } clearVisualizerConsumerDemand(consumerId: string): void { if (this.consumerDemand.delete(consumerId)) { this.pruneQueuesForDemand() + this.emitDemandState() } } @@ -491,7 +508,7 @@ export class AudioRouter { this.resetLatencyTrackers() } - private getActiveDemand(): VisualizerConsumerDemand { + private getActiveDemand(): NormalizedVisualizerConsumerDemand { const aggregated = createEmptyDemand() for (const demand of this.consumerDemand.values()) { for (const scope of SCOPE_KINDS) { @@ -503,6 +520,13 @@ export class AudioRouter { return aggregated } + private emitDemandState(): void { + const demand = this.getActiveVisualizerDemand() + for (const listener of this.demandListeners) { + listener(demand) + } + } + private pruneQueuesForDemand(): void { const activeDemand = this.getActiveDemand() for (const scope of SCOPE_KINDS) { diff --git a/src/renderer/audio/NativeVisualizerTransport.ts b/src/renderer/audio/NativeVisualizerTransport.ts new file mode 100644 index 0000000..247d59d --- /dev/null +++ b/src/renderer/audio/NativeVisualizerTransport.ts @@ -0,0 +1,231 @@ +import { oscilloscope, spectrum, vectorscope, isNativeAvailable } from './native' +import type { VisualizerConsumerDemand } from './AudioRouter' + +export interface NativeVisualizerTransportChunkMeta { + sessionId?: number + channelCount?: number +} + +export interface NativeVisualizerTransportSessionState { + sessionId: number + sampleRate: number + channelCount: number + capturing: boolean +} + +export interface NativeVisualizerTransportBridge { + isAvailable: () => boolean + oscilloscope: { + setSampleRate: (sampleRate: number) => void + pushSamples: (samples: Float32Array) => void + reset: () => void + } + spectrum: { + setSampleRate: (sampleRate: number) => void + pushSamples: (samples: Float32Array) => void + getMagnitudes: () => Float32Array | null + reset: () => void + } + vectorscope: { + setSampleRate: (sampleRate: number) => void + pushSamples: (left: Float32Array, right: Float32Array) => void + reset: () => void + } +} + +const EMPTY_DEMAND: Required = { + spectrum: false, + oscilloscope: false, + vectorscope: false, + spectrogram: false, + vumeter: false, + lufsmeter: false, + waveform: false, +} + +const defaultBridge: NativeVisualizerTransportBridge = { + isAvailable: () => isNativeAvailable(), + oscilloscope: { + setSampleRate: (sampleRate) => oscilloscope.setSampleRate(sampleRate), + pushSamples: (samples) => oscilloscope.pushSamples(samples), + reset: () => oscilloscope.reset(), + }, + spectrum: { + setSampleRate: (sampleRate) => spectrum.setSampleRate(sampleRate), + pushSamples: (samples) => spectrum.pushSamples(samples), + getMagnitudes: () => spectrum.getMagnitudes(), + reset: () => spectrum.reset(), + }, + vectorscope: { + setSampleRate: (sampleRate) => vectorscope.setSampleRate(sampleRate), + pushSamples: (left, right) => vectorscope.pushSamples(left, right), + reset: () => vectorscope.reset(), + }, +} + +function normalizeDemand(demand: VisualizerConsumerDemand): Required { + return { + spectrum: Boolean(demand.spectrum), + oscilloscope: Boolean(demand.oscilloscope), + vectorscope: Boolean(demand.vectorscope), + spectrogram: Boolean(demand.spectrogram), + vumeter: Boolean(demand.vumeter), + lufsmeter: Boolean(demand.lufsmeter), + waveform: Boolean(demand.waveform), + } +} + +export class NativeVisualizerTransport { + private readonly bridge: NativeVisualizerTransportBridge + private demand: Required = { ...EMPTY_DEMAND } + private sessionId = 0 + private sampleRate = 48000 + private capturing = false + private hasSpectrumData = false + + constructor(bridge: NativeVisualizerTransportBridge = defaultBridge) { + this.bridge = bridge + } + + setDemand(demand: VisualizerConsumerDemand): void { + const nextDemand = normalizeDemand(demand) + if ( + nextDemand.spectrum === this.demand.spectrum + && nextDemand.oscilloscope === this.demand.oscilloscope + && nextDemand.vectorscope === this.demand.vectorscope + && nextDemand.spectrogram === this.demand.spectrogram + && nextDemand.vumeter === this.demand.vumeter + && nextDemand.lufsmeter === this.demand.lufsmeter + && nextDemand.waveform === this.demand.waveform + ) { + return + } + + if (this.bridge.isAvailable()) { + if (this.demand.oscilloscope && !nextDemand.oscilloscope) { + this.bridge.oscilloscope.reset() + } + if (this.demand.spectrum && !nextDemand.spectrum) { + this.bridge.spectrum.reset() + this.hasSpectrumData = false + } + if (this.demand.vectorscope && !nextDemand.vectorscope) { + this.bridge.vectorscope.reset() + } + if (!this.demand.oscilloscope && nextDemand.oscilloscope) { + this.bridge.oscilloscope.setSampleRate(this.sampleRate) + } + if (!this.demand.spectrum && nextDemand.spectrum) { + this.bridge.spectrum.setSampleRate(this.sampleRate) + this.hasSpectrumData = false + } + if (!this.demand.vectorscope && nextDemand.vectorscope) { + this.bridge.vectorscope.setSampleRate(this.sampleRate) + } + } + + this.demand = nextDemand + } + + setSampleRate(sampleRate: number): void { + const nextSampleRate = Math.max(1, Math.floor(sampleRate) || 1) + if (nextSampleRate === this.sampleRate) { + return + } + + this.sampleRate = nextSampleRate + if (!this.bridge.isAvailable()) { + return + } + + if (this.demand.oscilloscope) { + this.bridge.oscilloscope.setSampleRate(nextSampleRate) + } + if (this.demand.spectrum) { + this.bridge.spectrum.setSampleRate(nextSampleRate) + } + if (this.demand.vectorscope) { + this.bridge.vectorscope.setSampleRate(nextSampleRate) + } + } + + handleChunk(left: Float32Array, right: Float32Array, meta: NativeVisualizerTransportChunkMeta = {}): void { + if (!this.capturing || !this.bridge.isAvailable()) { + return + } + + if (meta.sessionId !== undefined && meta.sessionId !== this.sessionId) { + return + } + + if (!this.demand.oscilloscope && !this.demand.spectrum && !this.demand.vectorscope) { + return + } + + const effectiveChannelCount = Math.max(1, Math.floor(meta.channelCount ?? 2) || 1) + const resolvedRight = effectiveChannelCount > 1 && right.length > 0 ? right : left + const length = Math.min(left.length, resolvedRight.length) + if (length === 0) { + return + } + + const leftSamples = left.length === length ? left : left.subarray(0, length) + const rightSamples = resolvedRight.length === length ? resolvedRight : resolvedRight.subarray(0, length) + + if (this.demand.oscilloscope) { + this.bridge.oscilloscope.pushSamples(leftSamples) + } + + if (this.demand.vectorscope) { + this.bridge.vectorscope.pushSamples(leftSamples, rightSamples) + } + + if (this.demand.spectrum) { + const mono = new Float32Array(length) + for (let index = 0; index < length; index += 1) { + mono[index] = (leftSamples[index] + rightSamples[index]) * 0.5 + } + this.bridge.spectrum.pushSamples(mono) + this.hasSpectrumData = true + } + } + + reset(sessionState: NativeVisualizerTransportSessionState): void { + this.sessionId = sessionState.sessionId + this.capturing = sessionState.capturing + this.sampleRate = Math.max(1, Math.floor(sessionState.sampleRate) || 1) + this.hasSpectrumData = false + + if (!this.bridge.isAvailable()) { + return + } + + this.bridge.oscilloscope.reset() + this.bridge.spectrum.reset() + this.bridge.vectorscope.reset() + + if (!sessionState.capturing) { + return + } + + if (this.demand.oscilloscope) { + this.bridge.oscilloscope.setSampleRate(this.sampleRate) + } + if (this.demand.spectrum) { + this.bridge.spectrum.setSampleRate(this.sampleRate) + } + if (this.demand.vectorscope) { + this.bridge.vectorscope.setSampleRate(this.sampleRate) + } + } + + getLatestSpectrumMagnitudes(): Float32Array | null { + if (!this.bridge.isAvailable() || !this.demand.spectrum || !this.hasSpectrumData) { + return null + } + + return this.bridge.spectrum.getMagnitudes() + } +} + +export const nativeVisualizerTransport = new NativeVisualizerTransport() diff --git a/src/renderer/audio/native/index.ts b/src/renderer/audio/native/index.ts index e97c379..a5f06ca 100644 --- a/src/renderer/audio/native/index.ts +++ b/src/renderer/audio/native/index.ts @@ -93,6 +93,15 @@ export const spectrum = { nativeModule?.spectrum.setSmoothing(smoothing) }, + pushSamples: (audioData: Float32Array): void => { + nativeModule?.spectrum.pushSamples(audioData) + }, + + getMagnitudes: (): Float32Array | null => { + if (!nativeModule) return null + return nativeModule.spectrum.getMagnitudes() + }, + process: (audioData: Float32Array): Float32Array | null => { if (!nativeModule) return null return nativeModule.spectrum.process(audioData) diff --git a/src/renderer/audio/native/visualizer-dsp.d.ts b/src/renderer/audio/native/visualizer-dsp.d.ts index 2dda2f2..3a4380b 100644 --- a/src/renderer/audio/native/visualizer-dsp.d.ts +++ b/src/renderer/audio/native/visualizer-dsp.d.ts @@ -49,6 +49,8 @@ export interface SpectrumModule { getFFTSize(): number; setSampleRate(sampleRate: number): void; setSmoothing(smoothing: number): void; + pushSamples(audioData: Float32Array): void; + getMagnitudes(): Float32Array; process(audioData: Float32Array): Float32Array; binToFrequency(bin: number): number; reset(): void; diff --git a/src/renderer/popouts/ScopePopoutDataSource.ts b/src/renderer/popouts/ScopePopoutDataSource.ts index 1a0e227..6a9c1ce 100644 --- a/src/renderer/popouts/ScopePopoutDataSource.ts +++ b/src/renderer/popouts/ScopePopoutDataSource.ts @@ -4,6 +4,7 @@ import type { ScopePopoutStereoBatch, } from '../../types/popout' import type { ScopeKind } from '../../types/scope' +import { NativeVisualizerTransport } from '../audio/NativeVisualizerTransport' import type { LUFSMeterDataSource } from '../visualizers/LUFSMeter' import type { OscilloscopeDataSource } from '../visualizers/Oscilloscope' import type { SpectrogramDataSource } from '../visualizers/Spectrogram' @@ -42,22 +43,43 @@ export class ScopePopoutDataSource implements AnyScopeDataSource { private stereoQueue: ScopePopoutStereoBatch = [] private sessionState: ScopePopoutSessionState = INITIAL_SESSION_STATE private readonly listeners = new Set<(state: ScopePopoutSessionState) => void>() + private readonly nativeVisualizerTransport = new NativeVisualizerTransport() - constructor(private readonly scopeKind: ScopeKind) {} + constructor(private readonly scopeKind: ScopeKind) { + this.nativeVisualizerTransport.setDemand({ + spectrum: scopeKind === 'spectrum', + oscilloscope: scopeKind === 'oscilloscope', + vectorscope: scopeKind === 'vectorscope', + }) + this.nativeVisualizerTransport.reset(this.sessionState) + } pushAudioBatch(batch: ScopePopoutAudioBatch): void { if (isStereoScope(this.scopeKind)) { if (!isStereoBatch(batch)) return this.stereoQueue.push(...batch) + for (const chunk of batch) { + this.nativeVisualizerTransport.handleChunk(chunk.left, chunk.right, { + sessionId: this.sessionState.sessionId, + channelCount: this.sessionState.channelCount, + }) + } return } if (isStereoBatch(batch)) return this.monoQueue.push(...batch) + for (const chunk of batch) { + this.nativeVisualizerTransport.handleChunk(chunk, chunk, { + sessionId: this.sessionState.sessionId, + channelCount: 1, + }) + } } setSessionState(nextState: ScopePopoutSessionState): void { this.sessionState = nextState + this.nativeVisualizerTransport.reset(nextState) if (!nextState.capturing) { this.monoQueue = [] this.stereoQueue = [] @@ -84,6 +106,10 @@ export class ScopePopoutDataSource implements AnyScopeDataSource { } } + getNativeVisualizerTransport(): NativeVisualizerTransport { + return this.nativeVisualizerTransport + } + getPendingSpectrumSamples(): Float32Array[] { const batch = this.monoQueue this.monoQueue = [] diff --git a/src/renderer/visualizers/Oscilloscope.ts b/src/renderer/visualizers/Oscilloscope.ts index 79b42a2..394adec 100644 --- a/src/renderer/visualizers/Oscilloscope.ts +++ b/src/renderer/visualizers/Oscilloscope.ts @@ -195,9 +195,12 @@ export class Oscilloscope { return } + const nativeTransport = this.dataSource.getNativeVisualizerTransport?.() ?? null const pendingSamples = this.dataSource.getPendingOscilloscopeSamples() for (const chunk of pendingSamples) { - nativeOscilloscope.pushSamples(chunk) + if (!nativeTransport) { + nativeOscilloscope.pushSamples(chunk) + } this.samplesReceived += chunk.length } diff --git a/src/renderer/visualizers/SpectrumAnalyzer.ts b/src/renderer/visualizers/SpectrumAnalyzer.ts index 1a56319..f844b9a 100644 --- a/src/renderer/visualizers/SpectrumAnalyzer.ts +++ b/src/renderer/visualizers/SpectrumAnalyzer.ts @@ -334,14 +334,20 @@ export class SpectrumAnalyzer { return } + const nativeTransport = this.dataSource.getNativeVisualizerTransport?.() ?? null const pendingSpectrum = this.dataSource.getPendingSpectrumSamples() - const monoData = this.mergePendingSpectrumChunks(pendingSpectrum) - if (!monoData) { - return + if (!nativeTransport) { + const monoData = this.mergePendingSpectrumChunks(pendingSpectrum) + if (monoData) { + nativeSpectrum.pushSamples(monoData) + } } - const frequencyData = nativeSpectrum.process(monoData) + const frequencyData = nativeTransport + ? nativeTransport.getLatestSpectrumMagnitudes() + : nativeSpectrum.getMagnitudes() if (!frequencyData) { + this.renderStaticLayer(minFrequency, maxFrequency) return } diff --git a/src/renderer/visualizers/Vectorscope.ts b/src/renderer/visualizers/Vectorscope.ts index 3880e4a..30bf7be 100644 --- a/src/renderer/visualizers/Vectorscope.ts +++ b/src/renderer/visualizers/Vectorscope.ts @@ -195,13 +195,16 @@ export class Vectorscope { offscreenCtx.fillRect(0, 0, width, height) offscreenCtx.globalCompositeOperation = 'source-over' + const nativeTransport = this.dataSource.getNativeVisualizerTransport?.() ?? null const pendingSamples = this.dataSource.getPendingVectorscopeSamples() if (options.multiband) { this.drawMultibandPoints(offscreenCtx, pendingSamples, centerX, centerY, scale) } else if (isNativeAvailable()) { - for (const chunk of pendingSamples) { - nativeVectorscope.pushSamples(chunk.left, chunk.right) + if (!nativeTransport) { + for (const chunk of pendingSamples) { + nativeVectorscope.pushSamples(chunk.left, chunk.right) + } } const pointsResult = nativeVectorscope.getPoints(options.displayPoints) diff --git a/src/renderer/visualizers/dataSource.ts b/src/renderer/visualizers/dataSource.ts index 8dfb308..cc6982b 100644 --- a/src/renderer/visualizers/dataSource.ts +++ b/src/renderer/visualizers/dataSource.ts @@ -1,14 +1,18 @@ import { audioRouter } from '../audio/AudioRouter' +import { nativeVisualizerTransport } from '../audio/NativeVisualizerTransport' import type { ScopePopoutSessionState } from '../../types/popout' +import type { NativeVisualizerTransport } from '../audio/NativeVisualizerTransport' export interface VisualizerSessionSource { getSampleRate: () => number isPlaying: () => boolean subscribeToSessionChanges: (listener: (state: ScopePopoutSessionState) => void) => () => void + getNativeVisualizerTransport?: () => NativeVisualizerTransport | null } export const defaultVisualizerSessionSource: VisualizerSessionSource = { getSampleRate: () => audioRouter.getSampleRate(), isPlaying: () => audioRouter.isCapturing(), subscribeToSessionChanges: (listener) => audioRouter.subscribeToSessionChanges((state) => listener(state)), + getNativeVisualizerTransport: () => nativeVisualizerTransport, } diff --git a/test/audio-router.test.ts b/test/audio-router.test.ts index 6ca8bae..c4a4468 100644 --- a/test/audio-router.test.ts +++ b/test/audio-router.test.ts @@ -88,3 +88,54 @@ test('drops stale-session chunks before they reach scope queues', () => { assert.equal(router.flushPendingVUMeterSamples().length, 0) assert.equal(router.getDiagnosticsSnapshot().staleSessionDrops, 1) }) + +test('publishes aggregated visualizer demand changes for downstream transports', () => { + const router = new AudioRouter() + const snapshots: Array> = [] + + const unsubscribe = router.subscribeToDemandChanges((demand) => { + snapshots.push({ ...demand }) + }) + + router.setVisualizerConsumerDemand('docked-strip', { spectrum: true, oscilloscope: true }) + router.setVisualizerConsumerDemand('popout:vectorscope', { vectorscope: true }) + router.clearVisualizerConsumerDemand('docked-strip') + unsubscribe() + + assert.deepEqual(snapshots[0], { + spectrum: false, + oscilloscope: false, + vectorscope: false, + spectrogram: false, + vumeter: false, + lufsmeter: false, + waveform: false, + }) + assert.deepEqual(snapshots[1], { + spectrum: true, + oscilloscope: true, + vectorscope: false, + spectrogram: false, + vumeter: false, + lufsmeter: false, + waveform: false, + }) + assert.deepEqual(snapshots[2], { + spectrum: true, + oscilloscope: true, + vectorscope: true, + spectrogram: false, + vumeter: false, + lufsmeter: false, + waveform: false, + }) + assert.deepEqual(snapshots[3], { + spectrum: false, + oscilloscope: false, + vectorscope: true, + spectrogram: false, + vumeter: false, + lufsmeter: false, + waveform: false, + }) +}) diff --git a/test/renderer-helpers.test.ts b/test/renderer-helpers.test.ts index 91e1a4f..116fae0 100644 --- a/test/renderer-helpers.test.ts +++ b/test/renderer-helpers.test.ts @@ -28,8 +28,17 @@ import { } from '../src/renderer/visualizers/vuMeterBallistics' import { FrameScheduler } from '../src/renderer/visualizers/frameScheduler' import { VisualizerFrameLoop } from '../src/renderer/visualizers/visualizerFrameLoop' +import { + NativeVisualizerTransport, + type NativeVisualizerTransportBridge, +} from '../src/renderer/audio/NativeVisualizerTransport' type WindowWithRaf = typeof globalThis & Pick +type WindowWithTimers = typeof globalThis & Pick & { + electronAPI: { + platform: string + } +} function installFakeAnimationFrame(): { pendingCount: () => number @@ -73,6 +82,76 @@ function installFakeAnimationFrame(): { } } +function installFakeTimeouts(hidden = false): { + pendingCount: () => number + nextDelay: () => number | null + runNext: () => void + restore: () => void + setHidden: (value: boolean) => void +} { + let nextTimerId = 1 + const timers = new Map void; delay: number }>() + const globalWithWindow = globalThis as typeof globalThis & { window?: WindowWithTimers; document?: Document } + const previousWindow = globalWithWindow.window + const previousDocument = globalWithWindow.document + const documentState = { hidden } + + globalWithWindow.window = { + ...globalThis, + electronAPI: { platform: 'darwin' }, + setTimeout(callback: TimerHandler, delay?: number): number { + const timerId = nextTimerId + nextTimerId += 1 + const run = typeof callback === 'function' + ? callback as () => void + : () => {} + timers.set(timerId, { + callback: run, + delay: typeof delay === 'number' ? delay : 0, + }) + return timerId + }, + clearTimeout(timerId: number): void { + timers.delete(timerId) + }, + } as WindowWithTimers + + globalWithWindow.document = documentState as Document + + return { + pendingCount: () => timers.size, + nextDelay: () => { + const nextTimer = timers.values().next().value + return nextTimer ? nextTimer.delay : null + }, + runNext(): void { + const next = timers.entries().next().value as [number, { callback: () => void; delay: number }] | undefined + if (!next) { + return + } + const [timerId, timer] = next + timers.delete(timerId) + timer.callback() + }, + restore(): void { + if (previousWindow === undefined) { + delete globalWithWindow.window + } else { + globalWithWindow.window = previousWindow + } + + if (previousDocument === undefined) { + delete globalWithWindow.document + } else { + globalWithWindow.document = previousDocument + } + }, + setHidden(value: boolean): void { + documentState.hidden = value + }, + } +} + function assertAlmostEqual(actual: number, expected: number, tolerance: number, message: string): void { assert.ok( Math.abs(actual - expected) <= tolerance, @@ -113,6 +192,77 @@ function createScopePopouts(poppedOutScopes: ScopeKind[] = []): ScopePopoutState }, {} as ScopePopoutStateMap) } +function createFakeTransportBridge(): { + bridge: NativeVisualizerTransportBridge + calls: { + oscilloscopePushes: Float32Array[] + vectorscopePushes: Array<{ left: Float32Array; right: Float32Array }> + spectrumPushes: Float32Array[] + oscilloscopeResets: number + vectorscopeResets: number + spectrumResets: number + oscilloscopeSampleRates: number[] + vectorscopeSampleRates: number[] + spectrumSampleRates: number[] + } +} { + let latestSpectrumMagnitudes: Float32Array | null = null + const calls = { + oscilloscopePushes: [] as Float32Array[], + vectorscopePushes: [] as Array<{ left: Float32Array; right: Float32Array }>, + spectrumPushes: [] as Float32Array[], + oscilloscopeResets: 0, + vectorscopeResets: 0, + spectrumResets: 0, + oscilloscopeSampleRates: [] as number[], + vectorscopeSampleRates: [] as number[], + spectrumSampleRates: [] as number[], + } + + return { + bridge: { + isAvailable: () => true, + oscilloscope: { + setSampleRate: (sampleRate) => { + calls.oscilloscopeSampleRates.push(sampleRate) + }, + pushSamples: (samples) => { + calls.oscilloscopePushes.push(samples) + }, + reset: () => { + calls.oscilloscopeResets += 1 + }, + }, + spectrum: { + setSampleRate: (sampleRate) => { + calls.spectrumSampleRates.push(sampleRate) + }, + pushSamples: (samples) => { + calls.spectrumPushes.push(samples) + latestSpectrumMagnitudes = new Float32Array([samples[0] ?? 0, samples[samples.length - 1] ?? 0]) + }, + getMagnitudes: () => latestSpectrumMagnitudes, + reset: () => { + calls.spectrumResets += 1 + latestSpectrumMagnitudes = null + }, + }, + vectorscope: { + setSampleRate: (sampleRate) => { + calls.vectorscopeSampleRates.push(sampleRate) + }, + pushSamples: (left, right) => { + calls.vectorscopePushes.push({ left, right }) + }, + reset: () => { + calls.vectorscopeResets += 1 + }, + }, + }, + calls, + } +} + test('parseColorToRgb handles hex, rgb, rgba, and percentage formats', () => { assert.deepEqual(parseColorToRgb('#38bdf8'), { r: 56, g: 189, b: 248 }) assert.deepEqual(parseColorToRgb('#3bf'), { r: 51, g: 187, b: 255 }) @@ -499,3 +649,188 @@ test('VUMeterBallistics lets the bar outrun the VU needle while peak hold remain assert.ok(afterDecay.peakLDb < beforeDecay.peakLDb) assert.ok(afterDecay.peakLDb > -3) }) + +test('NativeVisualizerTransport feeds native scope state from chunk arrival without a render tick', () => { + const { bridge, calls } = createFakeTransportBridge() + const transport = new NativeVisualizerTransport(bridge) + const left = new Float32Array([0.2, 0.4, 0.6]) + const right = new Float32Array([0.8, 0.6, 0.4]) + + transport.setDemand({ + spectrum: true, + oscilloscope: true, + vectorscope: true, + }) + transport.reset({ + sessionId: 1, + sampleRate: 48000, + channelCount: 2, + capturing: true, + }) + transport.handleChunk(left, right, { + sessionId: 1, + channelCount: 2, + }) + + assert.equal(calls.oscilloscopePushes.length, 1) + assert.equal(calls.vectorscopePushes.length, 1) + assert.equal(calls.spectrumPushes.length, 1) + assert.equal(calls.oscilloscopePushes[0], left) + assert.equal(calls.vectorscopePushes[0]?.left, left) + assert.equal(calls.vectorscopePushes[0]?.right, right) + assert.deepEqual(Array.from(calls.spectrumPushes[0]), [0.5, 0.5, 0.5]) + assert.deepEqual(Array.from(transport.getLatestSpectrumMagnitudes() ?? []), [0.5, 0.5]) +}) + +test('NativeVisualizerTransport resets cached state on session changes and sample-rate updates', () => { + const { bridge, calls } = createFakeTransportBridge() + const transport = new NativeVisualizerTransport(bridge) + + transport.setDemand({ spectrum: true, oscilloscope: true, vectorscope: true }) + transport.reset({ + sessionId: 7, + sampleRate: 48000, + channelCount: 2, + capturing: true, + }) + transport.handleChunk(new Float32Array([1, 1]), new Float32Array([1, 1]), { + sessionId: 7, + channelCount: 2, + }) + assert.deepEqual(Array.from(transport.getLatestSpectrumMagnitudes() ?? []), [1, 1]) + + transport.setSampleRate(96000) + transport.reset({ + sessionId: 8, + sampleRate: 96000, + channelCount: 2, + capturing: true, + }) + + assert.equal(calls.oscilloscopeSampleRates.at(-1), 96000) + assert.equal(calls.spectrumSampleRates.at(-1), 96000) + assert.equal(calls.vectorscopeSampleRates.at(-1), 96000) + assert.equal(calls.oscilloscopeResets >= 2, true) + assert.equal(calls.spectrumResets >= 2, true) + assert.equal(calls.vectorscopeResets >= 2, true) + assert.equal(transport.getLatestSpectrumMagnitudes(), null) +}) + +test('NativeVisualizerTransport stops feeding scopes when demand is removed', () => { + const { bridge, calls } = createFakeTransportBridge() + const transport = new NativeVisualizerTransport(bridge) + + transport.setDemand({ spectrum: true }) + transport.reset({ + sessionId: 2, + sampleRate: 48000, + channelCount: 2, + capturing: true, + }) + transport.handleChunk(new Float32Array([0.25]), new Float32Array([0.75]), { + sessionId: 2, + channelCount: 2, + }) + assert.equal(calls.spectrumPushes.length, 1) + + transport.setDemand({}) + transport.handleChunk(new Float32Array([0.5]), new Float32Array([0.5]), { + sessionId: 2, + channelCount: 2, + }) + + assert.equal(calls.spectrumPushes.length, 1) + assert.equal(calls.spectrumResets >= 1, true) + assert.equal(transport.getLatestSpectrumMagnitudes(), null) +}) + +test('NativePolledCaptureBackend schedules immediate, backoff, and hidden-document polls and cancels on stop', async () => { + const timers = installFakeTimeouts() + + try { + const { NativePolledCaptureBackend } = await import('../src/renderer/audio/AudioCapture') + + const drainResults = [ + { + chunks: [{ + left: new Float32Array([0.1, 0.2]), + right: new Float32Array([0.3, 0.4]), + channelCount: 2, + capturedAtMilliseconds: 5, + sequence: 1, + }], + overwriteCount: 0, + queueDepth: 0, + }, + { + chunks: [], + overwriteCount: 0, + queueDepth: 0, + }, + { + chunks: [], + overwriteCount: 0, + queueDepth: 0, + }, + ] + + const nativeModule = { + getSupport: () => ({ available: true, reason: null }), + listOutputDevices: () => [], + start: () => ({ + sampleRate: 48000, + channelCount: 2, + deviceId: 'device', + deviceLabel: 'Device', + }), + stop: () => {}, + drain: () => drainResults.shift() ?? { + chunks: [], + overwriteCount: 0, + queueDepth: 0, + }, + nowMilliseconds: () => 0, + } + + class TestNativeBackend extends NativePolledCaptureBackend { + readonly kind = 'native-macos' as const + + protected getNativeCaptureModule() { + return nativeModule + } + + protected getBackendLabel(): string { + return 'Test Native' + } + } + + const backend = new TestNativeBackend({ + kind: 'native-macos', + available: true, + reason: null, + }) + const receivedSequences: number[] = [] + backend.subscribe((chunk) => { + receivedSequences.push(chunk.sequence) + }) + + await backend.start() + assert.equal(timers.nextDelay(), 0) + + timers.runNext() + assert.deepEqual(receivedSequences, [1]) + assert.equal(timers.nextDelay(), 0) + + timers.runNext() + assert.equal(timers.nextDelay(), 2) + + timers.setHidden(true) + timers.runNext() + assert.equal(timers.nextDelay(), 16) + + await backend.stop() + assert.equal(timers.pendingCount(), 0) + } finally { + timers.restore() + } +})