significantly improve visual performance

src/renderer/audio/AudioCapture.ts

@@ -5,6 +5,7 @@
  */
 
 import { audioRouter } from './AudioRouter'
+import { nativeVisualizerTransport } from './NativeVisualizerTransport'
 import { applyInputGainToStereoSamples, inputGainDbToLinear } from './inputGain'
 import type {
   CaptureBackendKind,
@@ -52,6 +53,17 @@ interface CaptureBackend {
   getStatus(): CaptureBackendStatus
 }
 
+function isDocumentHidden(): boolean {
+  return typeof document !== 'undefined' && document.hidden === true
+}
+
+export function resolveNativeCapturePollDelay(chunkCount: number): number {
+  if (isDocumentHidden()) {
+    return 16
+  }
+  return chunkCount > 0 ? 0 : 2
+}
+
 export interface CaptureManagerStatus {
   captureMode: CaptureMode
   backendPolicy: CaptureBackendPolicy
@@ -366,7 +378,7 @@ class ElectronDeviceCaptureBackend implements CaptureBackend {
   }
 }
 
-abstract class NativePolledCaptureBackend implements CaptureBackend {
+export abstract class NativePolledCaptureBackend implements CaptureBackend {
   abstract readonly kind: CaptureBackendKind
 
   private readonly chunkListeners = new Set<(chunk: CaptureChunk) => void>()
@@ -464,8 +476,10 @@ abstract class NativePolledCaptureBackend implements CaptureBackend {
     const poll = (): void => {
       if (!this.active) return
 
+      let chunkCount = 0
       try {
         const result = this.getNativeCaptureModule()?.drain(32) as NativeCaptureDrainResult | undefined
+        chunkCount = result?.chunks.length ?? 0
         if (result) {
           for (const chunk of result.chunks) {
             const routedChunk: CaptureChunk = {
@@ -488,7 +502,7 @@ abstract class NativePolledCaptureBackend implements CaptureBackend {
         return
       }
 
-      this.pollTimer = window.setTimeout(poll, 4)
+      this.pollTimer = window.setTimeout(poll, resolveNativeCapturePollDelay(chunkCount))
     }
 
     this.pollTimer = window.setTimeout(poll, 0)
@@ -592,6 +606,11 @@ class AudioCapture {
 
     this.electronSystemBackend.subscribe((chunk) => this.handleChunk(this.electronSystemBackend.kind, chunk))
     this.electronDeviceBackend.subscribe((chunk) => this.handleChunk(this.electronDeviceBackend.kind, chunk))
+
+    audioRouter.subscribeToDemandChanges((demand) => {
+      nativeVisualizerTransport.setDemand(demand)
+    })
+    nativeVisualizerTransport.reset(audioRouter.getSessionState())
   }
 
   subscribeStatus(listener: StatusListener): () => void {
@@ -653,6 +672,7 @@ class AudioCapture {
           backendStatus.channelCount,
           backend.kind,
         )
+        nativeVisualizerTransport.reset(audioRouter.getSessionState())
         this.emitStatus()
         return
       } catch (error) {
@@ -813,6 +833,7 @@ class AudioCapture {
   private async stopActiveCapture(): Promise<void> {
     if (this.sessionId !== null) {
       audioRouter.endSession()
+      nativeVisualizerTransport.reset(audioRouter.getSessionState())
       this.sessionId = null
     }
 
@@ -838,6 +859,10 @@ class AudioCapture {
       capturedAt: chunk.capturedAt,
       sequence: chunk.sequence,
     })
+    nativeVisualizerTransport.handleChunk(chunk.left, chunk.right, {
+      sessionId: this.sessionId,
+      channelCount: chunk.channelCount,
+    })
   }
 
   setInputGain(db: number): void {

src/renderer/audio/AudioRouter.ts

@@ -60,6 +60,8 @@ export interface AudioRouterDiagnostics {
   scopes: Record<ScopeKind, AudioRouterScopeDiagnostics>
 }
 
+export type NormalizedVisualizerConsumerDemand = Required<VisualizerConsumerDemand>
+
 interface AudioChunkMeta {
   sessionId?: number
   channelCount?: number
@@ -187,7 +189,7 @@ class RollingLatencyWindow {
   }
 }
 
-function createEmptyDemand(): VisualizerConsumerDemand {
+function createEmptyDemand(): NormalizedVisualizerConsumerDemand {
   return {
     spectrum: false,
     oscilloscope: false,
@@ -242,6 +244,7 @@ export class AudioRouter {
   private undemandedChunks = 0
 
   private sessionListeners = new Set<(state: AudioSessionState) => void>()
+  private demandListeners = new Set<(demand: NormalizedVisualizerConsumerDemand) => void>()
 
   private emitSessionState(): void {
     const state = this.getSessionState()
@@ -307,6 +310,18 @@ export class AudioRouter {
     }
   }
 
+  subscribeToDemandChanges(listener: (demand: NormalizedVisualizerConsumerDemand) => void): () => void {
+    this.demandListeners.add(listener)
+    listener(this.getActiveVisualizerDemand())
+    return () => {
+      this.demandListeners.delete(listener)
+    }
+  }
+
+  getActiveVisualizerDemand(): NormalizedVisualizerConsumerDemand {
+    return this.getActiveDemand()
+  }
+
   setVisualizerConsumerDemand(consumerId: string, demand: VisualizerConsumerDemand): void {
     const normalized: VisualizerConsumerDemand = {
       spectrum: Boolean(demand.spectrum),
@@ -326,11 +341,13 @@ export class AudioRouter {
     }
 
     this.pruneQueuesForDemand()
+    this.emitDemandState()
   }
 
   clearVisualizerConsumerDemand(consumerId: string): void {
     if (this.consumerDemand.delete(consumerId)) {
       this.pruneQueuesForDemand()
+      this.emitDemandState()
     }
   }
 
@@ -491,7 +508,7 @@ export class AudioRouter {
     this.resetLatencyTrackers()
   }
 
-  private getActiveDemand(): VisualizerConsumerDemand {
+  private getActiveDemand(): NormalizedVisualizerConsumerDemand {
     const aggregated = createEmptyDemand()
     for (const demand of this.consumerDemand.values()) {
       for (const scope of SCOPE_KINDS) {
@@ -503,6 +520,13 @@ export class AudioRouter {
     return aggregated
   }
 
+  private emitDemandState(): void {
+    const demand = this.getActiveVisualizerDemand()
+    for (const listener of this.demandListeners) {
+      listener(demand)
+    }
+  }
+
   private pruneQueuesForDemand(): void {
     const activeDemand = this.getActiveDemand()
     for (const scope of SCOPE_KINDS) {

src/renderer/audio/NativeVisualizerTransport.ts

@@ -0,0 +1,231 @@
+import { oscilloscope, spectrum, vectorscope, isNativeAvailable } from './native'
+import type { VisualizerConsumerDemand } from './AudioRouter'
+
+export interface NativeVisualizerTransportChunkMeta {
+  sessionId?: number
+  channelCount?: number
+}
+
+export interface NativeVisualizerTransportSessionState {
+  sessionId: number
+  sampleRate: number
+  channelCount: number
+  capturing: boolean
+}
+
+export interface NativeVisualizerTransportBridge {
+  isAvailable: () => boolean
+  oscilloscope: {
+    setSampleRate: (sampleRate: number) => void
+    pushSamples: (samples: Float32Array) => void
+    reset: () => void
+  }
+  spectrum: {
+    setSampleRate: (sampleRate: number) => void
+    pushSamples: (samples: Float32Array) => void
+    getMagnitudes: () => Float32Array | null
+    reset: () => void
+  }
+  vectorscope: {
+    setSampleRate: (sampleRate: number) => void
+    pushSamples: (left: Float32Array, right: Float32Array) => void
+    reset: () => void
+  }
+}
+
+const EMPTY_DEMAND: Required<VisualizerConsumerDemand> = {
+  spectrum: false,
+  oscilloscope: false,
+  vectorscope: false,
+  spectrogram: false,
+  vumeter: false,
+  lufsmeter: false,
+  waveform: false,
+}
+
+const defaultBridge: NativeVisualizerTransportBridge = {
+  isAvailable: () => isNativeAvailable(),
+  oscilloscope: {
+    setSampleRate: (sampleRate) => oscilloscope.setSampleRate(sampleRate),
+    pushSamples: (samples) => oscilloscope.pushSamples(samples),
+    reset: () => oscilloscope.reset(),
+  },
+  spectrum: {
+    setSampleRate: (sampleRate) => spectrum.setSampleRate(sampleRate),
+    pushSamples: (samples) => spectrum.pushSamples(samples),
+    getMagnitudes: () => spectrum.getMagnitudes(),
+    reset: () => spectrum.reset(),
+  },
+  vectorscope: {
+    setSampleRate: (sampleRate) => vectorscope.setSampleRate(sampleRate),
+    pushSamples: (left, right) => vectorscope.pushSamples(left, right),
+    reset: () => vectorscope.reset(),
+  },
+}
+
+function normalizeDemand(demand: VisualizerConsumerDemand): Required<VisualizerConsumerDemand> {
+  return {
+    spectrum: Boolean(demand.spectrum),
+    oscilloscope: Boolean(demand.oscilloscope),
+    vectorscope: Boolean(demand.vectorscope),
+    spectrogram: Boolean(demand.spectrogram),
+    vumeter: Boolean(demand.vumeter),
+    lufsmeter: Boolean(demand.lufsmeter),
+    waveform: Boolean(demand.waveform),
+  }
+}
+
+export class NativeVisualizerTransport {
+  private readonly bridge: NativeVisualizerTransportBridge
+  private demand: Required<VisualizerConsumerDemand> = { ...EMPTY_DEMAND }
+  private sessionId = 0
+  private sampleRate = 48000
+  private capturing = false
+  private hasSpectrumData = false
+
+  constructor(bridge: NativeVisualizerTransportBridge = defaultBridge) {
+    this.bridge = bridge
+  }
+
+  setDemand(demand: VisualizerConsumerDemand): void {
+    const nextDemand = normalizeDemand(demand)
+    if (
+      nextDemand.spectrum === this.demand.spectrum
+      && nextDemand.oscilloscope === this.demand.oscilloscope
+      && nextDemand.vectorscope === this.demand.vectorscope
+      && nextDemand.spectrogram === this.demand.spectrogram
+      && nextDemand.vumeter === this.demand.vumeter
+      && nextDemand.lufsmeter === this.demand.lufsmeter
+      && nextDemand.waveform === this.demand.waveform
+    ) {
+      return
+    }
+
+    if (this.bridge.isAvailable()) {
+      if (this.demand.oscilloscope && !nextDemand.oscilloscope) {
+        this.bridge.oscilloscope.reset()
+      }
+      if (this.demand.spectrum && !nextDemand.spectrum) {
+        this.bridge.spectrum.reset()
+        this.hasSpectrumData = false
+      }
+      if (this.demand.vectorscope && !nextDemand.vectorscope) {
+        this.bridge.vectorscope.reset()
+      }
+      if (!this.demand.oscilloscope && nextDemand.oscilloscope) {
+        this.bridge.oscilloscope.setSampleRate(this.sampleRate)
+      }
+      if (!this.demand.spectrum && nextDemand.spectrum) {
+        this.bridge.spectrum.setSampleRate(this.sampleRate)
+        this.hasSpectrumData = false
+      }
+      if (!this.demand.vectorscope && nextDemand.vectorscope) {
+        this.bridge.vectorscope.setSampleRate(this.sampleRate)
+      }
+    }
+
+    this.demand = nextDemand
+  }
+
+  setSampleRate(sampleRate: number): void {
+    const nextSampleRate = Math.max(1, Math.floor(sampleRate) || 1)
+    if (nextSampleRate === this.sampleRate) {
+      return
+    }
+
+    this.sampleRate = nextSampleRate
+    if (!this.bridge.isAvailable()) {
+      return
+    }
+
+    if (this.demand.oscilloscope) {
+      this.bridge.oscilloscope.setSampleRate(nextSampleRate)
+    }
+    if (this.demand.spectrum) {
+      this.bridge.spectrum.setSampleRate(nextSampleRate)
+    }
+    if (this.demand.vectorscope) {
+      this.bridge.vectorscope.setSampleRate(nextSampleRate)
+    }
+  }
+
+  handleChunk(left: Float32Array, right: Float32Array, meta: NativeVisualizerTransportChunkMeta = {}): void {
+    if (!this.capturing || !this.bridge.isAvailable()) {
+      return
+    }
+
+    if (meta.sessionId !== undefined && meta.sessionId !== this.sessionId) {
+      return
+    }
+
+    if (!this.demand.oscilloscope && !this.demand.spectrum && !this.demand.vectorscope) {
+      return
+    }
+
+    const effectiveChannelCount = Math.max(1, Math.floor(meta.channelCount ?? 2) || 1)
+    const resolvedRight = effectiveChannelCount > 1 && right.length > 0 ? right : left
+    const length = Math.min(left.length, resolvedRight.length)
+    if (length === 0) {
+      return
+    }
+
+    const leftSamples = left.length === length ? left : left.subarray(0, length)
+    const rightSamples = resolvedRight.length === length ? resolvedRight : resolvedRight.subarray(0, length)
+
+    if (this.demand.oscilloscope) {
+      this.bridge.oscilloscope.pushSamples(leftSamples)
+    }
+
+    if (this.demand.vectorscope) {
+      this.bridge.vectorscope.pushSamples(leftSamples, rightSamples)
+    }
+
+    if (this.demand.spectrum) {
+      const mono = new Float32Array(length)
+      for (let index = 0; index < length; index += 1) {
+        mono[index] = (leftSamples[index] + rightSamples[index]) * 0.5
+      }
+      this.bridge.spectrum.pushSamples(mono)
+      this.hasSpectrumData = true
+    }
+  }
+
+  reset(sessionState: NativeVisualizerTransportSessionState): void {
+    this.sessionId = sessionState.sessionId
+    this.capturing = sessionState.capturing
+    this.sampleRate = Math.max(1, Math.floor(sessionState.sampleRate) || 1)
+    this.hasSpectrumData = false
+
+    if (!this.bridge.isAvailable()) {
+      return
+    }
+
+    this.bridge.oscilloscope.reset()
+    this.bridge.spectrum.reset()
+    this.bridge.vectorscope.reset()
+
+    if (!sessionState.capturing) {
+      return
+    }
+
+    if (this.demand.oscilloscope) {
+      this.bridge.oscilloscope.setSampleRate(this.sampleRate)
+    }
+    if (this.demand.spectrum) {
+      this.bridge.spectrum.setSampleRate(this.sampleRate)
+    }
+    if (this.demand.vectorscope) {
+      this.bridge.vectorscope.setSampleRate(this.sampleRate)
+    }
+  }
+
+  getLatestSpectrumMagnitudes(): Float32Array | null {
+    if (!this.bridge.isAvailable() || !this.demand.spectrum || !this.hasSpectrumData) {
+      return null
+    }
+
+    return this.bridge.spectrum.getMagnitudes()
+  }
+}
+
+export const nativeVisualizerTransport = new NativeVisualizerTransport()

src/renderer/audio/native/index.ts

@@ -93,6 +93,15 @@ export const spectrum = {
     nativeModule?.spectrum.setSmoothing(smoothing)
   },
 
+  pushSamples: (audioData: Float32Array): void => {
+    nativeModule?.spectrum.pushSamples(audioData)
+  },
+
+  getMagnitudes: (): Float32Array | null => {
+    if (!nativeModule) return null
+    return nativeModule.spectrum.getMagnitudes()
+  },
+
   process: (audioData: Float32Array): Float32Array | null => {
     if (!nativeModule) return null
     return nativeModule.spectrum.process(audioData)

src/renderer/audio/native/visualizer-dsp.d.ts

@@ -49,6 +49,8 @@ export interface SpectrumModule {
   getFFTSize(): number;
   setSampleRate(sampleRate: number): void;
   setSmoothing(smoothing: number): void;
+  pushSamples(audioData: Float32Array): void;
+  getMagnitudes(): Float32Array;
   process(audioData: Float32Array): Float32Array;
   binToFrequency(bin: number): number;
   reset(): void;

src/renderer/popouts/ScopePopoutDataSource.ts

@@ -4,6 +4,7 @@ import type {
   ScopePopoutStereoBatch,
 } from '../../types/popout'
 import type { ScopeKind } from '../../types/scope'
+import { NativeVisualizerTransport } from '../audio/NativeVisualizerTransport'
 import type { LUFSMeterDataSource } from '../visualizers/LUFSMeter'
 import type { OscilloscopeDataSource } from '../visualizers/Oscilloscope'
 import type { SpectrogramDataSource } from '../visualizers/Spectrogram'
@@ -42,22 +43,43 @@ export class ScopePopoutDataSource implements AnyScopeDataSource {
   private stereoQueue: ScopePopoutStereoBatch = []
   private sessionState: ScopePopoutSessionState = INITIAL_SESSION_STATE
   private readonly listeners = new Set<(state: ScopePopoutSessionState) => void>()
+  private readonly nativeVisualizerTransport = new NativeVisualizerTransport()
 
-  constructor(private readonly scopeKind: ScopeKind) {}
+  constructor(private readonly scopeKind: ScopeKind) {
+    this.nativeVisualizerTransport.setDemand({
+      spectrum: scopeKind === 'spectrum',
+      oscilloscope: scopeKind === 'oscilloscope',
+      vectorscope: scopeKind === 'vectorscope',
+    })
+    this.nativeVisualizerTransport.reset(this.sessionState)
+  }
 
   pushAudioBatch(batch: ScopePopoutAudioBatch): void {
     if (isStereoScope(this.scopeKind)) {
       if (!isStereoBatch(batch)) return
       this.stereoQueue.push(...batch)
+      for (const chunk of batch) {
+        this.nativeVisualizerTransport.handleChunk(chunk.left, chunk.right, {
+          sessionId: this.sessionState.sessionId,
+          channelCount: this.sessionState.channelCount,
+        })
+      }
       return
     }
 
     if (isStereoBatch(batch)) return
     this.monoQueue.push(...batch)
+    for (const chunk of batch) {
+      this.nativeVisualizerTransport.handleChunk(chunk, chunk, {
+        sessionId: this.sessionState.sessionId,
+        channelCount: 1,
+      })
+    }
   }
 
   setSessionState(nextState: ScopePopoutSessionState): void {
     this.sessionState = nextState
+    this.nativeVisualizerTransport.reset(nextState)
     if (!nextState.capturing) {
       this.monoQueue = []
       this.stereoQueue = []
@@ -84,6 +106,10 @@ export class ScopePopoutDataSource implements AnyScopeDataSource {
     }
   }
 
+  getNativeVisualizerTransport(): NativeVisualizerTransport {
+    return this.nativeVisualizerTransport
+  }
+
   getPendingSpectrumSamples(): Float32Array[] {
     const batch = this.monoQueue
     this.monoQueue = []

src/renderer/visualizers/Oscilloscope.ts

@@ -195,9 +195,12 @@ export class Oscilloscope {
       return
     }
 
+    const nativeTransport = this.dataSource.getNativeVisualizerTransport?.() ?? null
     const pendingSamples = this.dataSource.getPendingOscilloscopeSamples()
     for (const chunk of pendingSamples) {
-      nativeOscilloscope.pushSamples(chunk)
+      if (!nativeTransport) {
+        nativeOscilloscope.pushSamples(chunk)
+      }
       this.samplesReceived += chunk.length
     }
 

src/renderer/visualizers/SpectrumAnalyzer.ts

@@ -334,14 +334,20 @@ export class SpectrumAnalyzer {
       return
     }
 
+    const nativeTransport = this.dataSource.getNativeVisualizerTransport?.() ?? null
     const pendingSpectrum = this.dataSource.getPendingSpectrumSamples()
-    const monoData = this.mergePendingSpectrumChunks(pendingSpectrum)
-    if (!monoData) {
-      return
+    if (!nativeTransport) {
+      const monoData = this.mergePendingSpectrumChunks(pendingSpectrum)
+      if (monoData) {
+        nativeSpectrum.pushSamples(monoData)
+      }
     }
 
-    const frequencyData = nativeSpectrum.process(monoData)
+    const frequencyData = nativeTransport
+      ? nativeTransport.getLatestSpectrumMagnitudes()
+      : nativeSpectrum.getMagnitudes()
     if (!frequencyData) {
+      this.renderStaticLayer(minFrequency, maxFrequency)
       return
     }
 

src/renderer/visualizers/Vectorscope.ts

@@ -195,13 +195,16 @@ export class Vectorscope {
     offscreenCtx.fillRect(0, 0, width, height)
     offscreenCtx.globalCompositeOperation = 'source-over'
 
+    const nativeTransport = this.dataSource.getNativeVisualizerTransport?.() ?? null
     const pendingSamples = this.dataSource.getPendingVectorscopeSamples()
 
     if (options.multiband) {
       this.drawMultibandPoints(offscreenCtx, pendingSamples, centerX, centerY, scale)
     } else if (isNativeAvailable()) {
-      for (const chunk of pendingSamples) {
-        nativeVectorscope.pushSamples(chunk.left, chunk.right)
+      if (!nativeTransport) {
+        for (const chunk of pendingSamples) {
+          nativeVectorscope.pushSamples(chunk.left, chunk.right)
+        }
       }
 
       const pointsResult = nativeVectorscope.getPoints(options.displayPoints)

src/renderer/visualizers/dataSource.ts

@@ -1,14 +1,18 @@
 import { audioRouter } from '../audio/AudioRouter'
+import { nativeVisualizerTransport } from '../audio/NativeVisualizerTransport'
 import type { ScopePopoutSessionState } from '../../types/popout'
+import type { NativeVisualizerTransport } from '../audio/NativeVisualizerTransport'
 
 export interface VisualizerSessionSource {
   getSampleRate: () => number
   isPlaying: () => boolean
   subscribeToSessionChanges: (listener: (state: ScopePopoutSessionState) => void) => () => void
+  getNativeVisualizerTransport?: () => NativeVisualizerTransport | null
 }
 
 export const defaultVisualizerSessionSource: VisualizerSessionSource = {
   getSampleRate: () => audioRouter.getSampleRate(),
   isPlaying: () => audioRouter.isCapturing(),
   subscribeToSessionChanges: (listener) => audioRouter.subscribeToSessionChanges((state) => listener(state)),
+  getNativeVisualizerTransport: () => nativeVisualizerTransport,
 }