From 78d8be23a30602b88d9e0336f2170b4047d45b9b Mon Sep 17 00:00:00 2001 From: Boof2015 <75185879+Boof2015@users.noreply.github.com> Date: Sat, 21 Mar 2026 18:38:21 -0400 Subject: [PATCH] significantly improve latency --- package.json | 1 + scripts/run-audio-router-tests.mjs | 44 ++ src/main/index.ts | 46 ++ src/preload/index.ts | 2 + src/renderer/App.tsx | 5 +- src/renderer/audio/AudioCapture.ts | 622 ++++++++++++++++++---- src/renderer/audio/AudioRouter.ts | 512 ++++++++++++++---- src/renderer/components/SettingsPanel.tsx | 78 ++- src/renderer/components/Strip.tsx | 24 +- src/renderer/env.d.ts | 2 + src/renderer/stores/audioStore.ts | 74 ++- src/renderer/styles/globals.css | 6 + src/renderer/visualizers/VUMeter.ts | 8 + src/types/capture.ts | 29 + test/audio-router.test.ts | 90 ++++ 15 files changed, 1323 insertions(+), 220 deletions(-) create mode 100644 scripts/run-audio-router-tests.mjs create mode 100644 src/types/capture.ts create mode 100644 test/audio-router.test.ts diff --git a/package.json b/package.json index bd5635b..32d8a0f 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "build": "electron-vite build", "preview": "electron-vite preview", "typecheck": "tsc --noEmit", + "test:audio-router": "node scripts/run-audio-router-tests.mjs", "build:native": "cd native && node-gyp rebuild", "rebuild:native": "node -e \"const e=require('electron/package.json').version;const a=process.arch;const{execSync}=require('child_process');execSync('node-gyp rebuild --target='+e+' --arch='+a+' --dist-url=https://electronjs.org/headers',{stdio:'inherit',cwd:'native'})\"", "postinstall": "npm run rebuild:native || echo 'Native build failed, will use JS fallback'", diff --git a/scripts/run-audio-router-tests.mjs b/scripts/run-audio-router-tests.mjs new file mode 100644 index 0000000..8f978e2 --- /dev/null +++ b/scripts/run-audio-router-tests.mjs @@ -0,0 +1,44 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' +import { spawn } from 'node:child_process' +import { build } from 'esbuild' + +const rootDir = dirname(dirname(fileURLToPath(import.meta.url))) +const tempDir = await mkdtemp(join(tmpdir(), 'prism-audio-router-tests-')) +const bundledTestPath = join(tempDir, 'audio-router.test.mjs') +const entryPoint = join(rootDir, 'test', 'audio-router.test.ts') + +let exitCode = 1 + +try { + await build({ + entryPoints: [entryPoint], + outfile: bundledTestPath, + bundle: true, + platform: 'node', + format: 'esm', + target: 'node23', + sourcemap: 'inline', + }) + + exitCode = await new Promise((resolve) => { + const child = spawn(process.execPath, ['--test', bundledTestPath], { + stdio: 'inherit', + cwd: rootDir, + }) + + child.on('exit', (code) => { + resolve(code ?? 1) + }) + + child.on('error', () => { + resolve(1) + }) + }) +} finally { + await rm(tempDir, { recursive: true, force: true }) +} + +process.exit(exitCode) diff --git a/src/main/index.ts b/src/main/index.ts index a0c6230..34a630e 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -1,5 +1,6 @@ import { app, BrowserWindow, desktopCapturer, ipcMain, session } from 'electron' import { join } from 'path' +import type { CaptureBackendSupport, CaptureBackendSupportEntry } from '../types/capture' let mainWindow: BrowserWindow | null = null let currentSettingsHeight = 0 @@ -45,6 +46,47 @@ function createWindow(): void { } } +function getNativeCaptureSupportEntry(): CaptureBackendSupportEntry { + if (process.platform === 'darwin') { + return { + kind: 'native-macos', + available: false, + reason: 'Native macOS system audio capture is not implemented in this build.', + } + } + + if (process.platform === 'win32') { + return { + kind: 'native-windows', + available: false, + reason: 'Native Windows WASAPI loopback capture is not implemented in this build.', + } + } + + return { + kind: 'native-linux', + available: false, + reason: 'Native Linux monitor capture is not implemented in this build.', + } +} + +function getCaptureBackendSupport(): CaptureBackendSupport { + return { + policyOptions: ['auto', 'native', 'electron'], + nativeBackend: getNativeCaptureSupportEntry(), + electronSystem: { + kind: 'electron-system', + available: true, + reason: null, + }, + electronDevice: { + kind: 'electron-device', + available: true, + reason: null, + }, + } +} + // Auto-grant media (microphone) permission for audio capture function setupPermissions(): void { session.defaultSession.setPermissionRequestHandler((_webContents, permission, callback) => { @@ -82,6 +124,10 @@ function setupIPC(): void { return sources.map((s) => ({ id: s.id, name: s.name })) }) + ipcMain.handle('capture:get-backend-support', () => { + return getCaptureBackendSupport() + }) + ipcMain.on('window:expand-settings', (_event, panelHeight: number) => { if (!mainWindow) return const [width, height] = mainWindow.getSize() diff --git a/src/preload/index.ts b/src/preload/index.ts index 3edc7ed..06661a5 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -1,4 +1,5 @@ import { contextBridge, ipcRenderer } from 'electron' +import type { CaptureBackendSupport } from '../types/capture' // Expose Electron API to renderer contextBridge.exposeInMainWorld('electronAPI', { @@ -8,6 +9,7 @@ contextBridge.exposeInMainWorld('electronAPI', { toggleAlwaysOnTop: () => ipcRenderer.send('window:toggle-always-on-top'), isAlwaysOnTop: () => ipcRenderer.invoke('window:is-always-on-top'), getDesktopSources: () => ipcRenderer.invoke('audio:get-desktop-sources') as Promise<{ id: string; name: string }[]>, + getCaptureBackendSupport: () => ipcRenderer.invoke('capture:get-backend-support') as Promise, expandSettings: (panelHeight: number) => ipcRenderer.send('window:expand-settings', panelHeight), collapseSettings: (panelHeight: number) => ipcRenderer.send('window:collapse-settings', panelHeight), setSettingsHeight: (panelHeight: number) => ipcRenderer.send('window:set-settings-height', panelHeight), diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index 88cd1bd..c319124 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -19,7 +19,10 @@ export default function App(): JSX.Element { // Auto-capture on launch useEffect(() => { - useAudioStore.getState().startCapture() + const { isCapturing, captureStatus, startCapture } = useAudioStore.getState() + if (!isCapturing && captureStatus !== 'connecting') { + void startCapture() + } }, []) useEffect(() => { diff --git a/src/renderer/audio/AudioCapture.ts b/src/renderer/audio/AudioCapture.ts index 553bdc9..088a1ae 100644 --- a/src/renderer/audio/AudioCapture.ts +++ b/src/renderer/audio/AudioCapture.ts @@ -1,42 +1,251 @@ /** - * AudioCapture — captures system audio output via Electron's desktopCapturer, - * with fallback to getUserMedia for virtual audio devices (BlackHole, etc). - * Feeds captured samples into AudioRouter for distribution to visualizers. + * AudioCapture — backend manager for Prism's live capture pipeline. + * Stage 1 keeps Chromium capture as the working backend, while exposing + * native-backend policy and support plumbing for future low-latency paths. */ import { audioRouter } from './AudioRouter' +import type { + CaptureBackendKind, + CaptureBackendPolicy, + CaptureBackendSupport, + CaptureBackendSupportEntry, + CaptureMode, + CaptureSourceDescriptor, +} from '../../types/capture' -export type CaptureMode = 'system' | 'device' +export type { CaptureMode } from '../../types/capture' -class AudioCapture { +interface CaptureChunk { + left: Float32Array + right: Float32Array + channelCount: number + capturedAt: number + sequence: number +} + +interface CaptureBackendStatus { + kind: CaptureBackendKind + active: boolean + available: boolean + reason: string | null + sampleRate: number + channelCount: number +} + +interface CaptureBackendStartRequest { + deviceId?: string +} + +interface CaptureBackend { + readonly kind: CaptureBackendKind + start(request?: CaptureBackendStartRequest): Promise + stop(): Promise + listSources(): Promise + subscribe(listener: (chunk: CaptureChunk) => void): () => void + getStatus(): CaptureBackendStatus +} + +export interface CaptureManagerStatus { + captureMode: CaptureMode + backendPolicy: CaptureBackendPolicy + activeBackendKind: CaptureBackendKind | null + activeBackendReason: string | null + backendSupport: CaptureBackendSupport | null + sampleRate: number + channelCount: number + isCapturing: boolean +} + +type StatusListener = (status: CaptureManagerStatus) => void + +const DEFAULT_BACKEND_POLICY: CaptureBackendPolicy = 'auto' + +const DEFAULT_BACKEND_SUPPORT: CaptureBackendSupport = { + policyOptions: ['auto', 'native', 'electron'], + nativeBackend: { + kind: window.electronAPI.platform === 'darwin' + ? 'native-macos' + : window.electronAPI.platform === 'win32' + ? 'native-windows' + : 'native-linux', + available: false, + reason: 'Native system audio capture is not implemented in this build.', + }, + electronSystem: { + kind: 'electron-system', + available: true, + reason: null, + }, + electronDevice: { + kind: 'electron-device', + available: true, + reason: null, + }, +} + +function toDeviceSourceDescriptor(device: MediaDeviceInfo): CaptureSourceDescriptor { + return { + id: device.deviceId, + label: device.label || `Input ${device.deviceId.slice(0, 8)}`, + kind: 'device', + } +} + +class ElectronCaptureRuntime { private audioContext: AudioContext | null = null private stream: MediaStream | null = null private sourceNode: MediaStreamAudioSourceNode | null = null private workletNode: AudioWorkletNode | null = null - private selectedDeviceId: string | null = null - private captureMode: CaptureMode = 'system' - private sessionId: number | null = null + private workletLoaded = false + private chunkListeners = new Set<(chunk: CaptureChunk) => void>() + private active = false + private currentConfigKey: string | null = null + private sequence = 0 + private sampleRate = 48000 + private channelCount = 2 - /** - * Start capturing system audio output via desktopCapturer (ScreenCaptureKit on macOS 13+). - * This captures all system audio without needing BlackHole or any virtual device. - */ - async startSystemAudio(): Promise { - this.stop() + subscribe(listener: (chunk: CaptureChunk) => void): () => void { + this.chunkListeners.add(listener) + return () => { + this.chunkListeners.delete(listener) + } + } - // Get a screen source ID from the main process + async startSystem(): Promise { + await this.start({ mode: 'system' }) + } + + async startDevice(deviceId?: string): Promise { + await this.start({ mode: 'device', deviceId }) + } + + async stop(): Promise { + this.active = false + if (this.audioContext && this.audioContext.state === 'running') { + await this.audioContext.suspend() + } + } + + async listSystemSources(): Promise { + const sources = await window.electronAPI.getDesktopSources() + return sources.map((source) => ({ + id: source.id, + label: source.name, + kind: 'system', + })) + } + + async listDeviceSources(): Promise { + const devices = await navigator.mediaDevices.enumerateDevices() + return devices + .filter((device) => device.kind === 'audioinput') + .map((device) => toDeviceSourceDescriptor(device)) + } + + getStatus(kind: CaptureBackendKind, reason: string | null = null): CaptureBackendStatus { + return { + kind, + active: this.active, + available: true, + reason, + sampleRate: this.sampleRate, + channelCount: this.channelCount, + } + } + + private async start(config: { mode: CaptureMode; deviceId?: string }): Promise { + const configKey = `${config.mode}:${config.deviceId ?? ''}` + await this.ensureContext() + + if (this.currentConfigKey !== configKey || !this.stream || !this.sourceNode) { + const nextStream = config.mode === 'system' + ? await this.requestSystemStream() + : await this.requestDeviceStream(config.deviceId) + this.attachStream(nextStream, configKey) + } + + this.sequence = 0 + this.active = true + if (this.audioContext && this.audioContext.state !== 'running') { + await this.audioContext.resume() + } + } + + private async ensureContext(): Promise { + if (!this.audioContext) { + this.audioContext = new AudioContext({ latencyHint: 'interactive' }) + this.sampleRate = Math.max(1, Math.floor(this.audioContext.sampleRate)) + } + + if (!this.workletLoaded) { + await this.audioContext.audioWorklet.addModule('./capture-worklet.js') + this.workletLoaded = true + } + + if (!this.workletNode) { + this.workletNode = new AudioWorkletNode(this.audioContext, 'capture-processor', { + numberOfInputs: 1, + numberOfOutputs: 0, + channelCount: 2, + }) + + this.workletNode.port.onmessage = (event: MessageEvent<{ + left: Float32Array + right: Float32Array + channelCount?: number + }>) => { + if (!this.active) return + + const chunk: CaptureChunk = { + left: event.data.left, + right: event.data.right, + channelCount: Math.max(1, Math.floor(event.data.channelCount ?? this.channelCount) || 1), + capturedAt: performance.now(), + sequence: ++this.sequence, + } + + for (const listener of this.chunkListeners) { + listener(chunk) + } + } + } + } + + private attachStream(stream: MediaStream, configKey: string): void { + if (!this.audioContext || !this.workletNode) return + + if (this.sourceNode) { + this.sourceNode.disconnect() + this.sourceNode = null + } + + if (this.stream && this.stream !== stream) { + this.stream.getTracks().forEach((track) => track.stop()) + } + + this.stream = stream + this.sourceNode = this.audioContext.createMediaStreamSource(stream) + this.sourceNode.connect(this.workletNode) + + const audioTrack = stream.getAudioTracks()[0] ?? null + const trackSettings = audioTrack?.getSettings() + + this.channelCount = Math.max( + 1, + Math.floor(trackSettings?.channelCount ?? this.sourceNode.channelCount ?? 2), + ) + this.sampleRate = Math.max(1, Math.floor(this.audioContext.sampleRate)) + this.currentConfigKey = configKey + } + + private async requestSystemStream(): Promise { const sources = await window.electronAPI.getDesktopSources() if (!sources.length) { throw new Error('No desktop sources available for system audio capture') } - // Create AudioContext - this.audioContext = new AudioContext() - await this.audioContext.audioWorklet.addModule('./capture-worklet.js') - - // Request system audio via desktop capturer — must include video (Chromium requirement), - // but we immediately discard the video track - this.stream = await navigator.mediaDevices.getUserMedia({ + const stream = await navigator.mediaDevices.getUserMedia({ audio: { mandatory: { chromeMediaSource: 'desktop', @@ -50,28 +259,14 @@ class AudioCapture { } as unknown as MediaTrackConstraints, }) - // Drop the video track immediately — we only need audio - this.stream.getVideoTracks().forEach((track) => track.stop()) - - this.captureMode = 'system' - this.wireUpStream() + stream.getVideoTracks().forEach((track) => track.stop()) + return stream } - /** - * Start capturing from a specific audio input device (e.g. BlackHole, microphone). - * Fallback for when system audio capture isn't available. - */ - async startDevice(deviceId?: string): Promise { - this.stop() - - const targetDeviceId = deviceId ?? this.selectedDeviceId - - this.audioContext = new AudioContext() - await this.audioContext.audioWorklet.addModule('./capture-worklet.js') - + private async requestDeviceStream(deviceId?: string): Promise { const constraints: MediaStreamConstraints = { audio: { - ...(targetDeviceId ? { deviceId: { exact: targetDeviceId } } : {}), + ...(deviceId ? { deviceId: { exact: deviceId } } : {}), echoCancellation: false, noiseSuppression: false, autoGainControl: false, @@ -79,104 +274,219 @@ class AudioCapture { } as MediaTrackConstraints, } - this.stream = await navigator.mediaDevices.getUserMedia(constraints) + return navigator.mediaDevices.getUserMedia(constraints) + } +} - this.captureMode = 'device' - this.wireUpStream() +class ElectronSystemCaptureBackend implements CaptureBackend { + readonly kind = 'electron-system' as const - if (targetDeviceId) { - this.selectedDeviceId = targetDeviceId + constructor(private readonly runtime: ElectronCaptureRuntime) {} + + async start(): Promise { + await this.runtime.startSystem() + } + + async stop(): Promise { + await this.runtime.stop() + } + + async listSources(): Promise { + return this.runtime.listSystemSources() + } + + subscribe(listener: (chunk: CaptureChunk) => void): () => void { + return this.runtime.subscribe(listener) + } + + getStatus(): CaptureBackendStatus { + return this.runtime.getStatus(this.kind) + } +} + +class ElectronDeviceCaptureBackend implements CaptureBackend { + readonly kind = 'electron-device' as const + private lastDeviceId: string | undefined + + constructor(private readonly runtime: ElectronCaptureRuntime) {} + + async start(request?: CaptureBackendStartRequest): Promise { + this.lastDeviceId = request?.deviceId + await this.runtime.startDevice(request?.deviceId) + } + + async stop(): Promise { + await this.runtime.stop() + } + + async listSources(): Promise { + return this.runtime.listDeviceSources() + } + + subscribe(listener: (chunk: CaptureChunk) => void): () => void { + return this.runtime.subscribe(listener) + } + + getStatus(): CaptureBackendStatus { + return this.runtime.getStatus(this.kind, this.lastDeviceId ? null : null) + } +} + +class NativeUnavailableCaptureBackend implements CaptureBackend { + readonly kind: CaptureBackendKind + private readonly reason: string | null + + constructor(private readonly supportEntry: CaptureBackendSupportEntry) { + this.kind = supportEntry.kind + this.reason = supportEntry.reason + } + + async start(): Promise { + throw new Error(this.reason ?? 'Native system audio capture is unavailable.') + } + + async stop(): Promise { + // No-op stub until native capture backends are implemented. + } + + async listSources(): Promise { + return [] + } + + subscribe(): () => void { + return () => {} + } + + getStatus(): CaptureBackendStatus { + return { + kind: this.kind, + active: false, + available: this.supportEntry.available, + reason: this.reason, + sampleRate: 48000, + channelCount: 2, + } + } +} + +class AudioCapture { + private readonly electronRuntime = new ElectronCaptureRuntime() + private readonly electronSystemBackend: CaptureBackend + private readonly electronDeviceBackend: CaptureBackend + + private backendSupport: CaptureBackendSupport | null = null + private backendSupportPromise: Promise | null = null + private nativeBackend: CaptureBackend | null = null + private activeBackend: CaptureBackend | null = null + + private selectedDeviceId: string | null = null + private captureMode: CaptureMode = 'system' + private backendPolicy: CaptureBackendPolicy = DEFAULT_BACKEND_POLICY + private activeBackendReason: string | null = null + private sessionId: number | null = null + private statusListeners = new Set() + + constructor() { + this.electronSystemBackend = new ElectronSystemCaptureBackend(this.electronRuntime) + this.electronDeviceBackend = new ElectronDeviceCaptureBackend(this.electronRuntime) + + this.electronSystemBackend.subscribe((chunk) => this.handleChunk(this.electronSystemBackend.kind, chunk)) + this.electronDeviceBackend.subscribe((chunk) => this.handleChunk(this.electronDeviceBackend.kind, chunk)) + } + + subscribeStatus(listener: StatusListener): () => void { + this.statusListeners.add(listener) + listener(this.getStatus()) + return () => { + this.statusListeners.delete(listener) } } - /** - * Start capture — uses system audio by default, falls back to device capture. - */ + async refreshBackendSupport(): Promise { + this.backendSupportPromise = null + return this.ensureBackendSupport() + } + + async startSystemAudio(): Promise { + this.captureMode = 'system' + await this.start() + } + + async startDevice(deviceId?: string): Promise { + this.captureMode = 'device' + if (deviceId) { + this.selectedDeviceId = deviceId + } + await this.start() + } + async start(deviceId?: string): Promise { if (deviceId) { - return this.startDevice(deviceId) + this.selectedDeviceId = deviceId + this.captureMode = 'device' } - try { - await this.startSystemAudio() - } catch (err) { - console.warn('System audio capture failed, falling back to device capture:', err) - await this.startDevice(deviceId) - } - } + const support = await this.ensureBackendSupport() + const requestedMode = this.captureMode + const requestedDeviceId = requestedMode === 'device' ? this.selectedDeviceId ?? undefined : undefined + const candidateBackends = this.resolveCandidateBackends(support, requestedMode) - private wireUpStream(): void { - if (!this.audioContext || !this.stream) return + let lastError: Error | null = null + let nativeFallbackReason: string | null = null - this.sourceNode = this.audioContext.createMediaStreamSource(this.stream) - const audioTrack = this.stream.getAudioTracks()[0] ?? null - const trackSettings = audioTrack?.getSettings() - const channelCount = Math.max( - 1, - Math.floor(trackSettings?.channelCount ?? this.sourceNode.channelCount ?? 2) - ) - const sampleRate = Math.max( - 1, - Math.floor(trackSettings?.sampleRate ?? this.audioContext.sampleRate) - ) - const sessionId = audioRouter.beginSession(sampleRate, channelCount) - this.sessionId = sessionId - - this.workletNode = new AudioWorkletNode(this.audioContext, 'capture-processor', { - numberOfInputs: 1, - numberOfOutputs: 0, - channelCount: 2, - }) - - this.workletNode.port.onmessage = (event: MessageEvent<{ - left: Float32Array - right: Float32Array - channelCount?: number - }>) => { - audioRouter.ingestChunk(event.data.left, event.data.right, { - sessionId, - channelCount: event.data.channelCount ?? channelCount, - }) + for (const backend of candidateBackends) { + try { + await backend.start({ deviceId: requestedDeviceId }) + this.activeBackend = backend + this.activeBackendReason = nativeFallbackReason + const backendStatus = backend.getStatus() + this.sessionId = audioRouter.beginSession( + backendStatus.sampleRate, + backendStatus.channelCount, + backend.kind, + ) + this.emitStatus() + return + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown capture backend failure' + lastError = error instanceof Error ? error : new Error(message) + if (backend.kind.startsWith('native-')) { + nativeFallbackReason = message + } + } } - this.sourceNode.connect(this.workletNode) - console.log( - `AudioCapture: session ${sessionId} started (${sampleRate}Hz, ${channelCount}ch, mode=${this.captureMode})` - ) + this.activeBackendReason = nativeFallbackReason + this.emitStatus() + throw lastError ?? new Error('No capture backend succeeded.') } stop(): void { if (this.sessionId !== null) { - console.log(`AudioCapture: ending session ${this.sessionId}`) audioRouter.endSession() this.sessionId = null } - if (this.workletNode) { - this.workletNode.disconnect() - this.workletNode.port.onmessage = null - this.workletNode = null + if (this.activeBackend) { + void this.activeBackend.stop() } - if (this.sourceNode) { - this.sourceNode.disconnect() - this.sourceNode = null + this.emitStatus() + } + + async listSources(mode: CaptureMode = this.captureMode): Promise { + await this.ensureBackendSupport() + if (mode === 'device') { + return this.electronDeviceBackend.listSources() } - if (this.stream) { - this.stream.getTracks().forEach((track) => track.stop()) - this.stream = null - } - - if (this.audioContext) { - this.audioContext.close() - this.audioContext = null - } + const activeSystemBackend = this.resolveCandidateBackends(this.backendSupport ?? DEFAULT_BACKEND_SUPPORT, 'system')[0] + return activeSystemBackend.listSources() } async listDevices(): Promise { const devices = await navigator.mediaDevices.enumerateDevices() - return devices.filter((d) => d.kind === 'audioinput') + return devices.filter((device) => device.kind === 'audioinput') } getSelectedDeviceId(): string | null { @@ -185,14 +495,106 @@ class AudioCapture { setSelectedDeviceId(id: string | null): void { this.selectedDeviceId = id + this.emitStatus() } getCaptureMode(): CaptureMode { return this.captureMode } + setCaptureMode(mode: CaptureMode): void { + this.captureMode = mode + this.emitStatus() + } + + getBackendPolicy(): CaptureBackendPolicy { + return this.backendPolicy + } + + setBackendPolicy(policy: CaptureBackendPolicy): void { + this.backendPolicy = policy + this.emitStatus() + } + getSampleRate(): number { - return this.audioContext?.sampleRate ?? 48000 + return this.activeBackend?.getStatus().sampleRate ?? 48000 + } + + getStatus(): CaptureManagerStatus { + const backendStatus = this.activeBackend?.getStatus() + return { + captureMode: this.captureMode, + backendPolicy: this.backendPolicy, + activeBackendKind: this.activeBackend?.kind ?? null, + activeBackendReason: this.activeBackendReason, + backendSupport: this.backendSupport, + sampleRate: backendStatus?.sampleRate ?? 48000, + channelCount: backendStatus?.channelCount ?? 2, + isCapturing: Boolean(this.activeBackend?.getStatus().active && this.sessionId !== null), + } + } + + private async ensureBackendSupport(): Promise { + if (this.backendSupport) { + return this.backendSupport + } + + if (!this.backendSupportPromise) { + this.backendSupportPromise = window.electronAPI.getCaptureBackendSupport() + .catch(() => DEFAULT_BACKEND_SUPPORT) + .then((support) => { + this.backendSupport = support + this.nativeBackend = new NativeUnavailableCaptureBackend(support.nativeBackend) + this.emitStatus() + return support + }) + } + + return this.backendSupportPromise + } + + private resolveCandidateBackends( + support: CaptureBackendSupport, + mode: CaptureMode, + ): CaptureBackend[] { + if (mode === 'device') { + return [this.electronDeviceBackend] + } + + const nativeBackend = this.nativeBackend ?? new NativeUnavailableCaptureBackend(support.nativeBackend) + + switch (this.backendPolicy) { + case 'electron': + this.activeBackendReason = null + return [this.electronSystemBackend] + case 'native': + case 'auto': + if (support.nativeBackend.available) { + return [nativeBackend, this.electronSystemBackend] + } + this.activeBackendReason = support.nativeBackend.reason + return [this.electronSystemBackend] + } + } + + private handleChunk(originKind: CaptureBackendKind, chunk: CaptureChunk): void { + if (!this.activeBackend || this.activeBackend.kind !== originKind || this.sessionId === null) { + return + } + + audioRouter.ingestChunk(chunk.left, chunk.right, { + sessionId: this.sessionId, + channelCount: chunk.channelCount, + capturedAt: chunk.capturedAt, + sequence: chunk.sequence, + }) + } + + private emitStatus(): void { + const status = this.getStatus() + for (const listener of this.statusListeners) { + listener(status) + } } } diff --git a/src/renderer/audio/AudioRouter.ts b/src/renderer/audio/AudioRouter.ts index a09356d..30d54d5 100644 --- a/src/renderer/audio/AudioRouter.ts +++ b/src/renderer/audio/AudioRouter.ts @@ -1,37 +1,246 @@ /** - * AudioRouter — distributes captured audio samples to per-scope pending buffers. - * Pattern extracted from Astra's AudioEngine (lines 196-202, 470-561, 2997-3043). + * AudioRouter — demand-aware audio chunk routing for Prism's visualizers. + * Captured worklet chunks are stored in fixed-capacity rings so hidden scopes + * do not accumulate backlog and queue overflow never reallocates. */ +import { SCOPE_KINDS, type ScopeKind } from '../../types/scope' +import type { CaptureBackendKind } from '../../types/capture' + const MAX_PENDING_CHUNKS = 20 const MAX_PENDING_SPECTRUM_CHUNKS = 96 const MAX_PENDING_VECTORSCOPE_CHUNKS = 20 +const LATENCY_SAMPLE_WINDOW = 240 + +const SCOPE_RING_CAPACITY: Record = { + spectrum: MAX_PENDING_SPECTRUM_CHUNKS, + oscilloscope: MAX_PENDING_CHUNKS, + vectorscope: MAX_PENDING_VECTORSCOPE_CHUNKS, + spectrogram: MAX_PENDING_SPECTRUM_CHUNKS, + vumeter: MAX_PENDING_VECTORSCOPE_CHUNKS, + lufsmeter: MAX_PENDING_SPECTRUM_CHUNKS, + waveform: MAX_PENDING_SPECTRUM_CHUNKS, +} export interface AudioSessionState { sessionId: number sampleRate: number channelCount: number capturing: boolean + backendKind: CaptureBackendKind | null +} + +export interface VisualizerConsumerDemand { + spectrum?: boolean + oscilloscope?: boolean + vectorscope?: boolean + spectrogram?: boolean + vumeter?: boolean + lufsmeter?: boolean + waveform?: boolean +} + +export interface AudioRouterScopeDiagnostics { + lastCaptureToScopeMs: number | null + p95CaptureToScopeMs: number | null + drainedChunks: number + overwriteCount: number + queuedChunks: number + lastSequence: number | null +} + +export interface AudioRouterDiagnostics { + updatedAt: number + activeDemand: VisualizerConsumerDemand + overallP95CaptureToScopeMs: number | null + totalOverwriteCount: number + notCapturingDrops: number + staleSessionDrops: number + undemandedChunks: number + scopes: Record } interface AudioChunkMeta { sessionId?: number channelCount?: number + capturedAt?: number + sequence?: number } -class AudioRouter { - private pendingOscilloscopeSamples: Float32Array[] = [] - private pendingSpectrumSamples: Float32Array[] = [] - private pendingSpectrogramSamples: Float32Array[] = [] - private pendingVectorscopeSamples: { left: Float32Array; right: Float32Array }[] = [] - private pendingVUMeterSamples: { left: Float32Array; right: Float32Array }[] = [] - private pendingLUFSMeterSamples: { left: Float32Array; right: Float32Array }[] = [] - private pendingWaveformSamples: Float32Array[] = [] +interface MonoChunkRecord { + samples: Float32Array + capturedAt: number + sequence: number +} + +interface StereoChunkRecord { + left: Float32Array + right: Float32Array + capturedAt: number + sequence: number +} + +interface ScopeLatencyTracker { + lastCaptureToScopeMs: number | null + drainedChunks: number + lastSequence: number | null + latencyWindow: RollingLatencyWindow +} + +type ScopeRingMap = { + spectrum: FixedChunkRing + oscilloscope: FixedChunkRing + vectorscope: FixedChunkRing + spectrogram: FixedChunkRing + vumeter: FixedChunkRing + lufsmeter: FixedChunkRing + waveform: FixedChunkRing +} + +class FixedChunkRing { + private readonly buffer: Array + private start = 0 + private size = 0 + private overwriteCount = 0 + + constructor(private readonly capacity: number) { + this.buffer = new Array(Math.max(1, capacity)) + } + + push(item: T): void { + if (this.capacity <= 0) return + + if (this.size < this.capacity) { + const index = (this.start + this.size) % this.capacity + this.buffer[index] = item + this.size += 1 + return + } + + this.buffer[this.start] = item + this.start = (this.start + 1) % this.capacity + this.overwriteCount += 1 + } + + drain(): T[] { + if (this.size === 0) return [] + + const drained = new Array(this.size) + for (let index = 0; index < this.size; index += 1) { + const bufferIndex = (this.start + index) % this.capacity + const item = this.buffer[bufferIndex] + if (item !== undefined) { + drained[index] = item + } + this.buffer[bufferIndex] = undefined + } + + this.start = 0 + this.size = 0 + return drained.filter((item): item is T => item !== undefined) + } + + clear(): void { + if (this.size === 0) return + for (let index = 0; index < this.size; index += 1) { + const bufferIndex = (this.start + index) % this.capacity + this.buffer[bufferIndex] = undefined + } + this.start = 0 + this.size = 0 + } + + getSize(): number { + return this.size + } + + getOverwriteCount(): number { + return this.overwriteCount + } +} + +class RollingLatencyWindow { + private readonly values: number[] + private cursor = 0 + private count = 0 + + constructor(size: number) { + this.values = new Array(Math.max(1, size)) + } + + push(value: number): void { + this.values[this.cursor] = value + this.cursor = (this.cursor + 1) % this.values.length + this.count = Math.min(this.count + 1, this.values.length) + } + + getP95(): number | null { + if (this.count === 0) return null + const sorted = this.values.slice(0, this.count).sort((left, right) => left - right) + const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * 0.95) - 1)) + return sorted[index] ?? null + } + + reset(): void { + this.cursor = 0 + this.count = 0 + } +} + +function createEmptyDemand(): VisualizerConsumerDemand { + return { + spectrum: false, + oscilloscope: false, + vectorscope: false, + spectrogram: false, + vumeter: false, + lufsmeter: false, + waveform: false, + } +} + +function createScopeLatencyTracker(): ScopeLatencyTracker { + return { + lastCaptureToScopeMs: null, + drainedChunks: 0, + lastSequence: null, + latencyWindow: new RollingLatencyWindow(LATENCY_SAMPLE_WINDOW), + } +} + +export class AudioRouter { + private readonly rings: ScopeRingMap = { + spectrum: new FixedChunkRing(SCOPE_RING_CAPACITY.spectrum), + oscilloscope: new FixedChunkRing(SCOPE_RING_CAPACITY.oscilloscope), + vectorscope: new FixedChunkRing(SCOPE_RING_CAPACITY.vectorscope), + spectrogram: new FixedChunkRing(SCOPE_RING_CAPACITY.spectrogram), + vumeter: new FixedChunkRing(SCOPE_RING_CAPACITY.vumeter), + lufsmeter: new FixedChunkRing(SCOPE_RING_CAPACITY.lufsmeter), + waveform: new FixedChunkRing(SCOPE_RING_CAPACITY.waveform), + } + + private readonly scopeLatency: Record = { + spectrum: createScopeLatencyTracker(), + oscilloscope: createScopeLatencyTracker(), + vectorscope: createScopeLatencyTracker(), + spectrogram: createScopeLatencyTracker(), + vumeter: createScopeLatencyTracker(), + lufsmeter: createScopeLatencyTracker(), + waveform: createScopeLatencyTracker(), + } + + private readonly consumerDemand = new Map() private _sampleRate = 48000 private _capturing = false private _channelCount = 2 private _sessionId = 0 + private _backendKind: CaptureBackendKind | null = null + + private notCapturingDrops = 0 + private staleSessionDrops = 0 + private undemandedChunks = 0 + private sessionListeners = new Set<(state: AudioSessionState) => void>() private emitSessionState(): void { @@ -67,14 +276,16 @@ class AudioRouter { sampleRate: this._sampleRate, channelCount: this._channelCount, capturing: this._capturing, + backendKind: this._backendKind, } } - beginSession(sampleRate: number, channelCount: number): number { + beginSession(sampleRate: number, channelCount: number, backendKind: CaptureBackendKind | null = null): number { this._sessionId += 1 this._sampleRate = sampleRate this._channelCount = Math.max(1, Math.floor(channelCount) || 1) this._capturing = true + this._backendKind = backendKind this.reset() this.emitSessionState() return this._sessionId @@ -83,6 +294,7 @@ class AudioRouter { endSession(): void { this._sessionId += 1 this._capturing = false + this._backendKind = null this.reset() this.emitSessionState() } @@ -95,139 +307,243 @@ class AudioRouter { } } + setVisualizerConsumerDemand(consumerId: string, demand: VisualizerConsumerDemand): void { + const normalized: VisualizerConsumerDemand = { + spectrum: Boolean(demand.spectrum), + oscilloscope: Boolean(demand.oscilloscope), + vectorscope: Boolean(demand.vectorscope), + spectrogram: Boolean(demand.spectrogram), + vumeter: Boolean(demand.vumeter), + lufsmeter: Boolean(demand.lufsmeter), + waveform: Boolean(demand.waveform), + } + + const hasAnyDemand = Object.values(normalized).some(Boolean) + if (hasAnyDemand) { + this.consumerDemand.set(consumerId, normalized) + } else { + this.consumerDemand.delete(consumerId) + } + + this.pruneQueuesForDemand() + } + + clearVisualizerConsumerDemand(consumerId: string): void { + if (this.consumerDemand.delete(consumerId)) { + this.pruneQueuesForDemand() + } + } + ingestChunk(left: Float32Array, right: Float32Array, meta: AudioChunkMeta = {}): void { - if (!this._capturing) return - if (meta.sessionId !== undefined && meta.sessionId !== this._sessionId) return + if (!this._capturing) { + this.notCapturingDrops += 1 + return + } + + if (meta.sessionId !== undefined && meta.sessionId !== this._sessionId) { + this.staleSessionDrops += 1 + return + } const effectiveChannelCount = Math.max(1, Math.floor(meta.channelCount ?? this._channelCount) || 1) this._channelCount = effectiveChannelCount - const resolvedRight = effectiveChannelCount > 1 && right.length > 0 ? right : left - // Compute mono + const resolvedRight = effectiveChannelCount > 1 && right.length > 0 ? right : left const len = Math.min(left.length, resolvedRight.length) if (len === 0) return - const mono = new Float32Array(len) - for (let i = 0; i < len; i++) { - mono[i] = (left[i] + resolvedRight[i]) / 2 + const activeDemand = this.getActiveDemand() + const needsMono = Boolean(activeDemand.spectrum || activeDemand.spectrogram) + const needsStereo = Boolean(activeDemand.vectorscope || activeDemand.vumeter || activeDemand.lufsmeter) + const needsLeft = Boolean(activeDemand.oscilloscope || activeDemand.waveform) + + if (!needsMono && !needsStereo && !needsLeft) { + this.undemandedChunks += 1 + return } - // Oscilloscope — uses left channel - if (this.pendingOscilloscopeSamples.length >= MAX_PENDING_CHUNKS) { - this.pendingOscilloscopeSamples = this.pendingOscilloscopeSamples.slice( - -Math.floor(MAX_PENDING_CHUNKS / 2) - ) - } - this.pendingOscilloscopeSamples.push(left.slice(0, len)) + const capturedAt = meta.capturedAt ?? performance.now() + const sequence = meta.sequence ?? 0 + const leftSamples = left.length === len ? left : left.subarray(0, len) + const rightSamples = resolvedRight.length === len ? resolvedRight : resolvedRight.subarray(0, len) - // Spectrum — uses mono - if (this.pendingSpectrumSamples.length >= MAX_PENDING_SPECTRUM_CHUNKS) { - this.pendingSpectrumSamples = this.pendingSpectrumSamples.slice( - -Math.floor(MAX_PENDING_SPECTRUM_CHUNKS / 2) - ) + let mono: Float32Array | null = null + if (needsMono) { + mono = new Float32Array(len) + for (let index = 0; index < len; index += 1) { + mono[index] = (leftSamples[index] + rightSamples[index]) * 0.5 + } } - this.pendingSpectrumSamples.push(mono) - // Spectrogram — uses mono - if (this.pendingSpectrogramSamples.length >= MAX_PENDING_SPECTRUM_CHUNKS) { - this.pendingSpectrogramSamples = this.pendingSpectrogramSamples.slice( - -Math.floor(MAX_PENDING_SPECTRUM_CHUNKS / 2) - ) + if (activeDemand.oscilloscope) { + this.rings.oscilloscope.push({ samples: leftSamples, capturedAt, sequence }) } - this.pendingSpectrogramSamples.push(mono) - // Vectorscope — uses stereo - if (this.pendingVectorscopeSamples.length >= MAX_PENDING_VECTORSCOPE_CHUNKS) { - this.pendingVectorscopeSamples = this.pendingVectorscopeSamples.slice( - -Math.floor(MAX_PENDING_VECTORSCOPE_CHUNKS / 2) - ) + if (activeDemand.spectrum && mono) { + this.rings.spectrum.push({ samples: mono, capturedAt, sequence }) } - this.pendingVectorscopeSamples.push({ - left: left.slice(0, len), - right: resolvedRight.slice(0, len), - }) - // VU Meter — uses stereo - if (this.pendingVUMeterSamples.length >= MAX_PENDING_VECTORSCOPE_CHUNKS) { - this.pendingVUMeterSamples = this.pendingVUMeterSamples.slice( - -Math.floor(MAX_PENDING_VECTORSCOPE_CHUNKS / 2) - ) + if (activeDemand.spectrogram && mono) { + this.rings.spectrogram.push({ samples: mono, capturedAt, sequence }) } - this.pendingVUMeterSamples.push({ - left: left.slice(0, len), - right: resolvedRight.slice(0, len), - }) - // LUFS Meter — uses stereo - if (this.pendingLUFSMeterSamples.length >= MAX_PENDING_SPECTRUM_CHUNKS) { - this.pendingLUFSMeterSamples = this.pendingLUFSMeterSamples.slice( - -Math.floor(MAX_PENDING_SPECTRUM_CHUNKS / 2) - ) + if (activeDemand.vectorscope) { + this.rings.vectorscope.push({ left: leftSamples, right: rightSamples, capturedAt, sequence }) } - this.pendingLUFSMeterSamples.push({ - left: left.slice(0, len), - right: resolvedRight.slice(0, len), - }) - // Waveform — uses left channel - if (this.pendingWaveformSamples.length >= MAX_PENDING_SPECTRUM_CHUNKS) { - this.pendingWaveformSamples = this.pendingWaveformSamples.slice( - -Math.floor(MAX_PENDING_SPECTRUM_CHUNKS / 2) - ) + if (activeDemand.vumeter) { + this.rings.vumeter.push({ left: leftSamples, right: rightSamples, capturedAt, sequence }) + } + + if (activeDemand.lufsmeter) { + this.rings.lufsmeter.push({ left: leftSamples, right: rightSamples, capturedAt, sequence }) + } + + if (activeDemand.waveform) { + this.rings.waveform.push({ samples: leftSamples, capturedAt, sequence }) } - this.pendingWaveformSamples.push(new Float32Array(left)) } flushPendingOscilloscopeSamples(): Float32Array[] { - const samples = this.pendingOscilloscopeSamples - this.pendingOscilloscopeSamples = [] - return samples + const records = this.rings.oscilloscope.drain() + this.recordScopeDrain('oscilloscope', records) + return records.map((record) => record.samples) } flushPendingSpectrumSamples(): Float32Array[] { - const samples = this.pendingSpectrumSamples - this.pendingSpectrumSamples = [] - return samples + const records = this.rings.spectrum.drain() + this.recordScopeDrain('spectrum', records) + return records.map((record) => record.samples) } flushPendingSpectrogramSamples(): Float32Array[] { - const samples = this.pendingSpectrogramSamples - this.pendingSpectrogramSamples = [] - return samples + const records = this.rings.spectrogram.drain() + this.recordScopeDrain('spectrogram', records) + return records.map((record) => record.samples) } flushPendingVectorscopeSamples(): { left: Float32Array; right: Float32Array }[] { - const samples = this.pendingVectorscopeSamples - this.pendingVectorscopeSamples = [] - return samples + const records = this.rings.vectorscope.drain() + this.recordScopeDrain('vectorscope', records) + return records.map((record) => ({ left: record.left, right: record.right })) } flushPendingVUMeterSamples(): { left: Float32Array; right: Float32Array }[] { - const samples = this.pendingVUMeterSamples - this.pendingVUMeterSamples = [] - return samples + const records = this.rings.vumeter.drain() + this.recordScopeDrain('vumeter', records) + return records.map((record) => ({ left: record.left, right: record.right })) } flushPendingLUFSMeterSamples(): { left: Float32Array; right: Float32Array }[] { - const samples = this.pendingLUFSMeterSamples - this.pendingLUFSMeterSamples = [] - return samples + const records = this.rings.lufsmeter.drain() + this.recordScopeDrain('lufsmeter', records) + return records.map((record) => ({ left: record.left, right: record.right })) } flushPendingWaveformSamples(): Float32Array[] { - const samples = this.pendingWaveformSamples - this.pendingWaveformSamples = [] - return samples + const records = this.rings.waveform.drain() + this.recordScopeDrain('waveform', records) + return records.map((record) => record.samples) + } + + getDiagnosticsSnapshot(): AudioRouterDiagnostics { + const activeDemand = this.getActiveDemand() + let totalOverwriteCount = 0 + let overallP95CaptureToScopeMs: number | null = null + + const scopes = SCOPE_KINDS.reduce>((result, scope) => { + const scopeTracker = this.scopeLatency[scope] + const overwriteCount = this.getRing(scope).getOverwriteCount() + totalOverwriteCount += overwriteCount + + const p95CaptureToScopeMs = scopeTracker.latencyWindow.getP95() + if (p95CaptureToScopeMs !== null && activeDemand[scope]) { + overallP95CaptureToScopeMs = overallP95CaptureToScopeMs === null + ? p95CaptureToScopeMs + : Math.max(overallP95CaptureToScopeMs, p95CaptureToScopeMs) + } + + result[scope] = { + lastCaptureToScopeMs: scopeTracker.lastCaptureToScopeMs, + p95CaptureToScopeMs, + drainedChunks: scopeTracker.drainedChunks, + overwriteCount, + queuedChunks: this.getRing(scope).getSize(), + lastSequence: scopeTracker.lastSequence, + } + return result + }, {} as Record) + + return { + updatedAt: performance.now(), + activeDemand, + overallP95CaptureToScopeMs, + totalOverwriteCount, + notCapturingDrops: this.notCapturingDrops, + staleSessionDrops: this.staleSessionDrops, + undemandedChunks: this.undemandedChunks, + scopes, + } } reset(): void { - this.pendingOscilloscopeSamples = [] - this.pendingSpectrumSamples = [] - this.pendingSpectrogramSamples = [] - this.pendingVectorscopeSamples = [] - this.pendingVUMeterSamples = [] - this.pendingLUFSMeterSamples = [] - this.pendingWaveformSamples = [] + this.clearAllRings() + this.resetLatencyTrackers() + } + + private getActiveDemand(): VisualizerConsumerDemand { + const aggregated = createEmptyDemand() + for (const demand of this.consumerDemand.values()) { + for (const scope of SCOPE_KINDS) { + if (demand[scope]) { + aggregated[scope] = true + } + } + } + return aggregated + } + + private pruneQueuesForDemand(): void { + const activeDemand = this.getActiveDemand() + for (const scope of SCOPE_KINDS) { + if (!activeDemand[scope]) { + this.getRing(scope).clear() + } + } + } + + private clearAllRings(): void { + for (const scope of SCOPE_KINDS) { + this.getRing(scope).clear() + } + } + + private resetLatencyTrackers(): void { + for (const scope of SCOPE_KINDS) { + const tracker = this.scopeLatency[scope] + tracker.lastCaptureToScopeMs = null + tracker.drainedChunks = 0 + tracker.lastSequence = null + tracker.latencyWindow.reset() + } + } + + private recordScopeDrain(scope: ScopeKind, records: Array): void { + if (records.length === 0) return + + const tracker = this.scopeLatency[scope] + const now = performance.now() + for (const record of records) { + const captureToScopeMs = Math.max(0, now - record.capturedAt) + tracker.lastCaptureToScopeMs = captureToScopeMs + tracker.latencyWindow.push(captureToScopeMs) + tracker.lastSequence = record.sequence + } + tracker.drainedChunks += records.length + } + + private getRing(scope: ScopeKind): FixedChunkRing | FixedChunkRing { + return this.rings[scope] } } diff --git a/src/renderer/components/SettingsPanel.tsx b/src/renderer/components/SettingsPanel.tsx index 053b148..8c4f1ae 100644 --- a/src/renderer/components/SettingsPanel.tsx +++ b/src/renderer/components/SettingsPanel.tsx @@ -1,9 +1,11 @@ -import { useEffect, useMemo, useRef, type CSSProperties, type JSX, type ReactNode } from 'react' +import { useEffect, useMemo, useRef, useState, type CSSProperties, type JSX, type ReactNode } from 'react' import { useAudioStore } from '../stores/audioStore' import { useSettingsStore, type ScopeSettings } from '../stores/settingsStore' import { useThemeStore, PRESETS, PRESET_IDS } from '../stores/themeStore' import type { ScopeKind } from '../../types/scope' import { buildAnalyzerGridTemplateColumns } from '../analyzerLayout' +import { audioRouter, type AudioRouterDiagnostics } from '../audio/AudioRouter' +import type { CaptureBackendKind, CaptureBackendPolicy } from '../../types/capture' const SCOPE_LABELS: Record = { spectrum: 'Spectrum', @@ -15,6 +17,23 @@ const SCOPE_LABELS: Record = { waveform: 'Waveform', } +function captureBackendLabel(kind: CaptureBackendKind | null): string { + switch (kind) { + case 'electron-system': + return 'Electron System' + case 'electron-device': + return 'Electron Device' + case 'native-macos': + return 'Native macOS' + case 'native-windows': + return 'Native Windows' + case 'native-linux': + return 'Native Linux' + default: + return 'None' + } +} + function vectorscopeModeLabel(mode: ScopeSettings['vectorscope']['mode']): string { switch (mode) { case 'lissajous': @@ -480,17 +499,25 @@ export default function SettingsPanel({ onClose, onHeightChange }: SettingsPanel devices, selectedDeviceId, captureMode, + capturePolicy, + activeBackendKind, + activeBackendReason, isCapturing, captureStatus, captureError, refreshDevices, + refreshBackendSupport, selectDevice, setCaptureMode, + setCapturePolicy, startCapture, } = useAudioStore() const { scopeSettings, updateScopeSettings, hiddenScopes, scopeOrder, widthWeights } = useSettingsStore() const { presetId, accent, setPreset, setCustomAccent, customAccent } = useThemeStore() const panelRef = useRef(null) + const [routerDiagnostics, setRouterDiagnostics] = useState( + () => audioRouter.getDiagnosticsSnapshot(), + ) const visibleScopes = scopeOrder.filter((kind) => !hiddenScopes.has(kind)) const scopeTrackStyle = useMemo(() => { @@ -500,8 +527,18 @@ export default function SettingsPanel({ onClose, onHeightChange }: SettingsPanel }, [visibleScopes, widthWeights]) useEffect(() => { - void refreshDevices() - }, [refreshDevices]) + void Promise.all([refreshDevices(), refreshBackendSupport()]) + }, [refreshBackendSupport, refreshDevices]) + + useEffect(() => { + const intervalId = window.setInterval(() => { + setRouterDiagnostics(audioRouter.getDiagnosticsSnapshot()) + }, 250) + + return () => { + window.clearInterval(intervalId) + } + }, []) useEffect(() => { const panel = panelRef.current @@ -538,6 +575,10 @@ export default function SettingsPanel({ onClose, onHeightChange }: SettingsPanel await startCapture() } + const handlePolicyChange = async (value: string): Promise => { + await setCapturePolicy(value as CaptureBackendPolicy) + } + const indicatorLabel = isCapturing ? 'Capturing' : captureStatus === 'connecting' @@ -546,6 +587,10 @@ export default function SettingsPanel({ onClose, onHeightChange }: SettingsPanel ? 'Capture Failed' : 'Idle' + const latencyLabel = routerDiagnostics.overallP95CaptureToScopeMs === null + ? 'Waiting for samples' + : `${routerDiagnostics.overallP95CaptureToScopeMs.toFixed(1)} ms p95` + return (
@@ -572,11 +617,38 @@ export default function SettingsPanel({ onClose, onHeightChange }: SettingsPanel + +
{indicatorLabel}
+
+ Active backend: {captureBackendLabel(activeBackendKind)} +
+ +
+ Latency probe: {latencyLabel} · overwrites {routerDiagnostics.totalOverwriteCount} · stale drops {routerDiagnostics.staleSessionDrops} +
+ + {activeBackendReason ? ( +
{activeBackendReason}
+ ) : null} + {captureError ? (
{captureError}
) : null} diff --git a/src/renderer/components/Strip.tsx b/src/renderer/components/Strip.tsx index 41b95e7..a79df52 100644 --- a/src/renderer/components/Strip.tsx +++ b/src/renderer/components/Strip.tsx @@ -4,6 +4,7 @@ import { useThemeStore } from '../stores/themeStore' import type { ScopeKind } from '../../types/scope' import ScopeModule from './ScopeModule' import { buildAnalyzerGridTemplateColumns } from '../analyzerLayout' +import { audioRouter } from '../audio/AudioRouter' export default function Strip(): JSX.Element { const scopeOrder = useSettingsStore((s) => s.scopeOrder) @@ -15,7 +16,11 @@ export default function Strip(): JSX.Element { const gridRef = useRef(null) const scopeRefs = useRef>>({}) const [handleOffsets, setHandleOffsets] = useState([]) - const visibleScopes = scopeOrder.filter((k) => !hiddenScopes.has(k)) + const visibleScopes = useMemo( + () => scopeOrder.filter((k) => !hiddenScopes.has(k)), + [hiddenScopes, scopeOrder], + ) + const visibleScopeKey = useMemo(() => visibleScopes.join('|'), [visibleScopes]) const gridTemplateColumns = useMemo(() => { return buildAnalyzerGridTemplateColumns(visibleScopes, widthWeights) }, [visibleScopes, widthWeights]) @@ -40,6 +45,23 @@ export default function Strip(): JSX.Element { setHandleOffsets(nextOffsets) }, [visibleScopes]) + useEffect(() => { + const visibleScopeSet = new Set(visibleScopes) + audioRouter.setVisualizerConsumerDemand('docked-strip', { + spectrum: visibleScopeSet.has('spectrum'), + oscilloscope: visibleScopeSet.has('oscilloscope'), + vectorscope: visibleScopeSet.has('vectorscope'), + spectrogram: visibleScopeSet.has('spectrogram'), + vumeter: visibleScopeSet.has('vumeter'), + lufsmeter: visibleScopeSet.has('lufsmeter'), + waveform: visibleScopeSet.has('waveform'), + }) + + return () => { + audioRouter.clearVisualizerConsumerDemand('docked-strip') + } + }, [visibleScopeKey, visibleScopes]) + useEffect(() => { const strip = stripRef.current if (!strip) return diff --git a/src/renderer/env.d.ts b/src/renderer/env.d.ts index df87053..a5131ce 100644 --- a/src/renderer/env.d.ts +++ b/src/renderer/env.d.ts @@ -1,6 +1,7 @@ /// import type { VisualizerDSP } from './audio/native/visualizer-dsp' +import type { CaptureBackendSupport } from '../types/capture' declare global { interface Window { @@ -12,6 +13,7 @@ declare global { toggleAlwaysOnTop: () => void isAlwaysOnTop: () => Promise getDesktopSources: () => Promise<{ id: string; name: string }[]> + getCaptureBackendSupport: () => Promise expandSettings: (panelHeight: number) => void collapseSettings: (panelHeight: number) => void setSettingsHeight: (panelHeight: number) => void diff --git a/src/renderer/stores/audioStore.ts b/src/renderer/stores/audioStore.ts index ea9f667..313e66e 100644 --- a/src/renderer/stores/audioStore.ts +++ b/src/renderer/stores/audioStore.ts @@ -1,59 +1,112 @@ import { create } from 'zustand' -import { audioCapture, type CaptureMode } from '../audio/AudioCapture' +import { audioCapture, type CaptureManagerStatus } from '../audio/AudioCapture' +import type { + CaptureBackendKind, + CaptureBackendPolicy, + CaptureBackendSupport, + CaptureMode, +} from '../../types/capture' interface AudioState { devices: MediaDeviceInfo[] selectedDeviceId: string | null captureMode: CaptureMode + capturePolicy: CaptureBackendPolicy + activeBackendKind: CaptureBackendKind | null + activeBackendReason: string | null + backendSupport: CaptureBackendSupport | null isCapturing: boolean captureStatus: 'idle' | 'connecting' | 'capturing' | 'error' captureError: string | null sampleRate: number + channelCount: number refreshDevices: () => Promise + refreshBackendSupport: () => Promise selectDevice: (deviceId: string) => Promise setCaptureMode: (mode: CaptureMode) => void + setCapturePolicy: (policy: CaptureBackendPolicy) => Promise startCapture: () => Promise stopCapture: () => void } +function applyCaptureStatus(status: CaptureManagerStatus): Partial { + return { + captureMode: status.captureMode, + capturePolicy: status.backendPolicy, + activeBackendKind: status.activeBackendKind, + activeBackendReason: status.activeBackendReason, + backendSupport: status.backendSupport, + sampleRate: status.sampleRate, + channelCount: status.channelCount, + isCapturing: status.isCapturing, + } +} + export const useAudioStore = create((set, get) => ({ devices: [], selectedDeviceId: null, captureMode: 'system', + capturePolicy: 'auto', + activeBackendKind: null, + activeBackendReason: null, + backendSupport: null, isCapturing: false, captureStatus: 'idle', captureError: null, sampleRate: 48000, + channelCount: 2, refreshDevices: async () => { const devices = await audioCapture.listDevices() set({ devices }) }, + refreshBackendSupport: async () => { + const backendSupport = await audioCapture.refreshBackendSupport() + set({ backendSupport }) + }, + selectDevice: async (deviceId: string) => { - set({ selectedDeviceId: deviceId, captureMode: 'device' }) audioCapture.setSelectedDeviceId(deviceId) + audioCapture.setCaptureMode('device') + set({ selectedDeviceId: deviceId, captureMode: 'device' }) }, setCaptureMode: (mode: CaptureMode) => { + audioCapture.setCaptureMode(mode) set({ captureMode: mode }) }, + setCapturePolicy: async (policy: CaptureBackendPolicy) => { + audioCapture.setBackendPolicy(policy) + set({ capturePolicy: policy }) + await get().refreshBackendSupport() + + const { isCapturing, captureMode } = get() + if (isCapturing && captureMode === 'system') { + await get().startCapture() + } + }, + startCapture: async () => { set({ captureStatus: 'connecting', captureError: null }) try { - const { captureMode, selectedDeviceId } = get() + const { captureMode, selectedDeviceId, capturePolicy } = get() + audioCapture.setCaptureMode(captureMode) + audioCapture.setBackendPolicy(capturePolicy) + await get().refreshBackendSupport() + if (captureMode === 'system') { - await audioCapture.start() + await audioCapture.startSystemAudio() } else { await audioCapture.startDevice(selectedDeviceId ?? undefined) } + + const status = audioCapture.getStatus() set({ - isCapturing: true, + ...applyCaptureStatus(status), captureStatus: 'capturing', captureError: null, - sampleRate: audioCapture.getSampleRate(), - captureMode: audioCapture.getCaptureMode(), }) } catch (err) { console.error('Failed to start audio capture:', err) @@ -71,3 +124,10 @@ export const useAudioStore = create((set, get) => ({ set({ isCapturing: false, captureStatus: 'idle', captureError: null }) }, })) + +audioCapture.subscribeStatus((status) => { + useAudioStore.setState((state) => ({ + ...state, + ...applyCaptureStatus(status), + })) +}) diff --git a/src/renderer/styles/globals.css b/src/renderer/styles/globals.css index a42668a..99a36cc 100644 --- a/src/renderer/styles/globals.css +++ b/src/renderer/styles/globals.css @@ -520,6 +520,12 @@ select { line-height: 1.4; } +.settings-info-text { + color: var(--text-tertiary); + font-size: 11px; + line-height: 1.4; +} + .settings-theme-swatches { display: flex; flex-wrap: wrap; diff --git a/src/renderer/visualizers/VUMeter.ts b/src/renderer/visualizers/VUMeter.ts index 1ab86fd..afca5d1 100644 --- a/src/renderer/visualizers/VUMeter.ts +++ b/src/renderer/visualizers/VUMeter.ts @@ -65,6 +65,7 @@ export class VUMeter { private dataSource: VUMeterDataSource private animationId: number | null = null private isRunning = false + private unsubscribeSessionChange: (() => void) | null = null // Meter state private rmsL = METER_MIN_DB @@ -84,6 +85,9 @@ export class VUMeter { const { dataSource, ...optionOverrides } = options this.options = { ...defaultOptions, ...optionOverrides } this.dataSource = dataSource ?? defaultVUMeterDataSource + this.unsubscribeSessionChange = audioRouter.subscribeToSessionChanges(() => { + this.resetMeters() + }) } private resetMeters(): void { @@ -606,5 +610,9 @@ export class VUMeter { dispose(): void { this.stop() + if (this.unsubscribeSessionChange) { + this.unsubscribeSessionChange() + this.unsubscribeSessionChange = null + } } } diff --git a/src/types/capture.ts b/src/types/capture.ts new file mode 100644 index 0000000..5dd9264 --- /dev/null +++ b/src/types/capture.ts @@ -0,0 +1,29 @@ +export type CaptureMode = 'system' | 'device' + +export type CaptureBackendPolicy = 'auto' | 'native' | 'electron' + +export type CaptureBackendKind = + | 'electron-system' + | 'electron-device' + | 'native-macos' + | 'native-windows' + | 'native-linux' + +export interface CaptureSourceDescriptor { + id: string + label: string + kind: CaptureMode +} + +export interface CaptureBackendSupportEntry { + kind: CaptureBackendKind + available: boolean + reason: string | null +} + +export interface CaptureBackendSupport { + policyOptions: CaptureBackendPolicy[] + nativeBackend: CaptureBackendSupportEntry + electronSystem: CaptureBackendSupportEntry + electronDevice: CaptureBackendSupportEntry +} diff --git a/test/audio-router.test.ts b/test/audio-router.test.ts new file mode 100644 index 0000000..6ca8bae --- /dev/null +++ b/test/audio-router.test.ts @@ -0,0 +1,90 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { AudioRouter } from '../src/renderer/audio/AudioRouter' + +function createChunk(value: number, length = 4): Float32Array { + return new Float32Array(Array.from({ length }, () => value)) +} + +test('routes chunks only to demanded scopes and prunes queues when demand is removed', () => { + const router = new AudioRouter() + const sessionId = router.beginSession(48000, 2, 'electron-system') + + router.ingestChunk(createChunk(1), createChunk(1), { + sessionId, + channelCount: 2, + sequence: 1, + capturedAt: performance.now() - 5, + }) + assert.equal(router.flushPendingSpectrumSamples().length, 0) + assert.equal(router.getDiagnosticsSnapshot().undemandedChunks, 1) + + router.setVisualizerConsumerDemand('test-consumer', { spectrum: true, waveform: true }) + router.ingestChunk(createChunk(2), createChunk(4), { + sessionId, + channelCount: 2, + sequence: 2, + capturedAt: performance.now() - 5, + }) + + const spectrumChunks = router.flushPendingSpectrumSamples() + const waveformChunks = router.flushPendingWaveformSamples() + const oscilloscopeChunks = router.flushPendingOscilloscopeSamples() + + assert.equal(spectrumChunks.length, 1) + assert.equal(waveformChunks.length, 1) + assert.equal(oscilloscopeChunks.length, 0) + assert.deepEqual(Array.from(spectrumChunks[0]), [3, 3, 3, 3]) + assert.deepEqual(Array.from(waveformChunks[0]), [2, 2, 2, 2]) + + router.ingestChunk(createChunk(7), createChunk(9), { + sessionId, + channelCount: 2, + sequence: 3, + capturedAt: performance.now() - 5, + }) + router.clearVisualizerConsumerDemand('test-consumer') + + assert.equal(router.flushPendingSpectrumSamples().length, 0) + assert.equal(router.flushPendingWaveformSamples().length, 0) +}) + +test('keeps the newest chunks when a fixed-capacity ring overflows', () => { + const router = new AudioRouter() + const sessionId = router.beginSession(48000, 2, 'electron-system') + router.setVisualizerConsumerDemand('test-consumer', { oscilloscope: true }) + + for (let sequence = 1; sequence <= 25; sequence += 1) { + router.ingestChunk(createChunk(sequence), createChunk(sequence), { + sessionId, + channelCount: 2, + sequence, + capturedAt: performance.now() - 2, + }) + } + + const oscilloscopeChunks = router.flushPendingOscilloscopeSamples() + assert.equal(oscilloscopeChunks.length, 20) + assert.equal(oscilloscopeChunks[0]?.[0], 6) + assert.equal(oscilloscopeChunks[19]?.[0], 25) + + const diagnostics = router.getDiagnosticsSnapshot() + assert.equal(diagnostics.scopes.oscilloscope.overwriteCount, 5) + assert.ok((diagnostics.scopes.oscilloscope.p95CaptureToScopeMs ?? 0) >= 0) +}) + +test('drops stale-session chunks before they reach scope queues', () => { + const router = new AudioRouter() + const sessionId = router.beginSession(48000, 1, 'electron-device') + router.setVisualizerConsumerDemand('test-consumer', { vumeter: true }) + + router.ingestChunk(createChunk(1), createChunk(1), { + sessionId: sessionId + 1, + channelCount: 1, + sequence: 1, + capturedAt: performance.now() - 1, + }) + + assert.equal(router.flushPendingVUMeterSamples().length, 0) + assert.equal(router.getDiagnosticsSnapshot().staleSessionDrops, 1) +})