From 0763ba0294f15d79165a3d8791897a89d9578327 Mon Sep 17 00:00:00 2001 From: Boof2015 <75185879+Boof2015@users.noreply.github.com> Date: Fri, 28 Aug 2026 02:53:08 -0400 Subject: [PATCH] faster waveform generation --- .../AstraLibraryScannerModule.kt | 268 ++++++------------ modules/astra-library-scanner/index.ts | 15 +- src/app/settings/troubleshooting.tsx | 17 +- src/audio/applyNormalization.ts | 6 + src/audio/trackAnalysis.ts | 131 ++++++--- src/audio/useNormalizationSync.ts | 19 ++ src/components/WaveformSeekBar.tsx | 41 +-- src/scope/waveform.ts | 69 +---- src/scope/waveformMath.test.mts | 82 ++---- src/scope/waveformMath.ts | 32 +-- 10 files changed, 252 insertions(+), 428 deletions(-) diff --git a/modules/astra-library-scanner/android/src/main/java/expo/modules/astralibraryscanner/AstraLibraryScannerModule.kt b/modules/astra-library-scanner/android/src/main/java/expo/modules/astralibraryscanner/AstraLibraryScannerModule.kt index 4bf784d..10fe9f9 100644 --- a/modules/astra-library-scanner/android/src/main/java/expo/modules/astralibraryscanner/AstraLibraryScannerModule.kt +++ b/modules/astra-library-scanner/android/src/main/java/expo/modules/astralibraryscanner/AstraLibraryScannerModule.kt @@ -64,6 +64,8 @@ import kotlin.math.roundToInt private const val LIBRARY_SCAN_LOG_TAG = "AstraLibraryScan" +private fun nanosToDoubleMs(nanos: Long): Double = nanos / 1_000_000.0 + private data class MetadataStageTiming( val androidMetadataNanos: Long, val multiArtistNanos: Long, @@ -170,6 +172,24 @@ class AudioAnalysis : Record { @Field var mime: String? = null /** Whether the loudness meter rode along on this pass. */ @Field var withLoudness: Boolean = false + /** Time spent waiting for one of the two shared native analysis lanes. */ + @Field var queueWaitMs: Double = 0.0 + /** Extractor discovery + codec creation/configuration/start. */ + @Field var setupMs: Double? = null + /** Active-pass time until the first non-empty PCM output buffer. */ + @Field var firstPcmMs: Double? = null + /** Active-pass time when the first progressive waveform could be emitted. */ + @Field var firstProgressMs: Double? = null + /** Codec-start through end-of-stream, including the selected PCM workload. */ + @Field var decodeToEosMs: Double? = null + /** Final RMS normalization and/or LUFS gate resolution. */ + @Field var finalizeMs: Double? = null + /** Handler/codec/extractor teardown after the result was finalized. */ + @Field var cleanupMs: Double? = null + /** Whole native request including semaphore wait and teardown. */ + @Field var endToEndMs: Double? = null + @Field var sampleRate: Int? = null + @Field var channelCount: Int? = null } /** ReplayGain tags read from the container (no audio decode). Null = tag absent. */ @@ -293,12 +313,26 @@ class AstraLibraryScannerModule : Module() { AsyncFunction("analyzeTrack") Coroutine { uri: String, bins: Int, withLoudness: Boolean -> val flag = AtomicBoolean(false) activeAnalyses[uri] = flag + val requestedNanos = System.nanoTime() try { waveformSemaphore.withPermit { + val queueWaitMs = nanosToDoubleMs(System.nanoTime() - requestedNanos) // Cancelled while queued behind another decode — don't start at all. - if (flag.get()) AudioAnalysis().apply { cancelled = true } + if (flag.get()) AudioAnalysis().apply { + cancelled = true + this.queueWaitMs = queueWaitMs + endToEndMs = nanosToDoubleMs(System.nanoTime() - requestedNanos) + } else withContext(Dispatchers.IO) { - runAnalysis(uri, if (bins > 0) bins else 512, withLoudness, flag) + runAnalysis( + uriStr = uri, + bins = if (bins > 0) bins else 512, + withLoudness = withLoudness, + cancelFlag = flag, + ).also { analysis -> + analysis.queueWaitMs = queueWaitMs + analysis.endToEndMs = nanosToDoubleMs(System.nanoTime() - requestedNanos) + } } } } finally { @@ -312,15 +346,6 @@ class AstraLibraryScannerModule : Module() { activeAnalyses[uri]?.set(true) } - // Fast waveform preview for first paint: sparse short-window decode across - // the file. The JS side shows this immediately as the coarse full-width shape - // that the progressive accurate pass then fills over; only analyzeTrack persists. - AsyncFunction("extractWaveformPreview") Coroutine { uri: String, bins: Int -> - waveformSemaphore.withPermit { - withContext(Dispatchers.IO) { decodeWaveformPreview(uri, if (bins > 0) bins else 96) } - } - } - // ReplayGain tags (M4): reads container metadata only (no PCM decode), so it is // cheap and lets us normalize a tagged library without the slow loudness decode. AsyncFunction("readReplayGain") Coroutine { uri: String -> @@ -914,6 +939,8 @@ class AstraLibraryScannerModule : Module() { var codec: MediaCodec? = null var handlerThread: HandlerThread? = null val startNanos = System.nanoTime() + var codecStartNanos = 0L + var finalizedNanos = 0L try { extractor.setDataSource(context, uri, null) @@ -934,15 +961,19 @@ class AstraLibraryScannerModule : Module() { val sampleRate = if (format.containsKey(MediaFormat.KEY_SAMPLE_RATE)) format.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100 + result.sampleRate = sampleRate val durationUs = if (format.containsKey(MediaFormat.KEY_DURATION)) format.getLong(MediaFormat.KEY_DURATION) else 0L result.durationMs = durationUs / 1000.0 val totalFrames = max(1L, (durationUs / 1_000_000.0 * sampleRate).toLong()) - val acc = AnalyzeAccumulator(bins, totalFrames, withLoudness) - acc.sampleRate = sampleRate - acc.channelCount = + val initialChannels = if (format.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) else 2 + result.channelCount = initialChannels + val acc = AnalyzeAccumulator(bins, totalFrames, withLoudness).also { + it.sampleRate = sampleRate + it.channelCount = initialChannels + } val decoder = createAnalysisDecoder(mime) codec = decoder @@ -992,6 +1023,9 @@ class AstraLibraryScannerModule : Module() { return } if (info.size > 0) { + if (result.firstPcmMs == null) { + result.firstPcmMs = nanosToDoubleMs(System.nanoTime() - startNanos) + } val out = mc.getOutputBuffer(index) if (out != null) { out.position(info.offset) @@ -1007,12 +1041,22 @@ class AstraLibraryScannerModule : Module() { // in at the end. Skipped on the EOS buffer — the promise carries the final, // globally-normalized result a moment later. val now = System.nanoTime() - if (!eos && acc.filledBins > lastEmitBin && now - lastEmitNanos >= progressEmitNanos) { + if ( + !eos && + acc.filledBins > lastEmitBin && + now - lastEmitNanos >= progressEmitNanos + ) { lastEmitNanos = now lastEmitBin = acc.filledBins + if (result.firstProgressMs == null) { + result.firstProgressMs = nanosToDoubleMs(now - startNanos) + } emitWaveformProgress(uriStr, acc, lastEmitBin) } - if (eos) done.complete(Unit) + if (eos) { + result.decodeToEosMs = nanosToDoubleMs(now - codecStartNanos) + done.complete(Unit) + } } catch (_: Throwable) { done.complete(Unit) } @@ -1020,10 +1064,14 @@ class AstraLibraryScannerModule : Module() { override fun onOutputFormatChanged(mc: MediaCodec, fmt: MediaFormat) { if (fmt.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) { - acc.channelCount = fmt.getInteger(MediaFormat.KEY_CHANNEL_COUNT) + val channels = fmt.getInteger(MediaFormat.KEY_CHANNEL_COUNT) + result.channelCount = channels + acc.channelCount = channels } if (fmt.containsKey(MediaFormat.KEY_SAMPLE_RATE)) { - acc.sampleRate = fmt.getInteger(MediaFormat.KEY_SAMPLE_RATE) + val rate = fmt.getInteger(MediaFormat.KEY_SAMPLE_RATE) + result.sampleRate = rate + acc.sampleRate = rate } if (fmt.containsKey(MediaFormat.KEY_PCM_ENCODING)) { acc.pcmFloat = @@ -1039,7 +1087,9 @@ class AstraLibraryScannerModule : Module() { ) codec.configure(format, null, null, 0) + codecStartNanos = System.nanoTime() codec.start() + result.setupMs = nanosToDoubleMs(System.nanoTime() - startNanos) if (withTimeoutOrNull(analysisTimeoutMs) { done.await() } == null) { // Timed out: peaks are partial, so treat it as a cancellation rather than caching @@ -1048,12 +1098,15 @@ class AstraLibraryScannerModule : Module() { } if (result.cancelled) return result + val finalizeStartNanos = System.nanoTime() result.peaks = acc.finalPeaks() if (withLoudness) { result.lufs = acc.loudness result.peak = acc.samplePeak } - val decodeMs = (System.nanoTime() - startNanos) / 1_000_000.0 + finalizedNanos = System.nanoTime() + result.finalizeMs = nanosToDoubleMs(finalizedNanos - finalizeStartNanos) + val decodeMs = nanosToDoubleMs(finalizedNanos - startNanos) result.decodeMs = decodeMs val dur = result.durationMs if (dur != null && dur > 0 && decodeMs > 0) result.realtimeFactor = dur / decodeMs @@ -1061,6 +1114,7 @@ class AstraLibraryScannerModule : Module() { } catch (_: Throwable) { return result } finally { + val cleanupStartNanos = System.nanoTime() // Order matters: stopping the codec while a callback is mid-flight on the handler // thread can crash. Quit the looper and wait for the in-flight callback to drain // first, THEN tear the codec down. @@ -1071,6 +1125,12 @@ class AstraLibraryScannerModule : Module() { try { codec?.stop() } catch (_: Throwable) {} try { codec?.release() } catch (_: Throwable) {} try { extractor.release() } catch (_: Throwable) {} + val finishedNanos = System.nanoTime() + result.cleanupMs = nanosToDoubleMs(finishedNanos - cleanupStartNanos) + if (result.decodeMs == null && finalizedNanos == 0L) { + result.decodeMs = nanosToDoubleMs(cleanupStartNanos - startNanos) + } + result.endToEndMs = result.queueWaitMs + nanosToDoubleMs(finishedNanos - startNanos) } } @@ -1097,7 +1157,9 @@ class AstraLibraryScannerModule : Module() { // realtime playback, not bulk decode, and the instance is a scarce global resource shared // with the track that is actually playing — so prefer a software decoder and fall back to // the platform's default pick. - private fun createAnalysisDecoder(mime: String): MediaCodec { + private fun createAnalysisDecoder( + mime: String, + ): MediaCodec { try { val info = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.firstOrNull { c -> !c.isEncoder && @@ -1244,172 +1306,6 @@ class AstraLibraryScannerModule : Module() { } } - private data class PcmEnergy( - val sumSquares: Double, - val sampleCount: Long, - val frameCount: Long - ) - - // Sparse preview waveform: seek to a bounded number of points, decode a very - // short audio window at each point, and normalize those RMS samples. This is - // intentionally approximate; runAnalysis remains the accurate cache fill. - private fun decodeWaveformPreview(uriStr: String, bins: Int): FloatArray { - val context = requireContext() - val previewBins = bins.coerceIn(16, 128) - val uri = Uri.parse(uriStr) - val extractor = MediaExtractor() - var codec: MediaCodec? = null - try { - extractor.setDataSource(context, uri, null) - - var trackFormat: MediaFormat? = null - var trackIndex = -1 - for (i in 0 until extractor.trackCount) { - val f = extractor.getTrackFormat(i) - if (f.getString(MediaFormat.KEY_MIME)?.startsWith("audio/") == true) { - trackFormat = f; trackIndex = i; break - } - } - val format = trackFormat ?: return FloatArray(0) - extractor.selectTrack(trackIndex) - - val mime = format.getString(MediaFormat.KEY_MIME) ?: return FloatArray(0) - val durationUs = - if (format.containsKey(MediaFormat.KEY_DURATION)) format.getLong(MediaFormat.KEY_DURATION) else 0L - if (durationUs <= 0L) return FloatArray(0) - - val sampleRate = - if (format.containsKey(MediaFormat.KEY_SAMPLE_RATE)) format.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100 - var channelCount = - if (format.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) else 2 - var pcmFloat = false - val windowFrames = max(512L, (sampleRate * 0.018).roundToInt().toLong()) - val peaks = FloatArray(previewBins) - - codec = MediaCodec.createDecoderByType(mime) - codec.configure(format, null, null, 0) - codec.start() - val info = MediaCodec.BufferInfo() - - for (bin in 0 until previewBins) { - val targetUs = ((durationUs.toDouble() * bin) / previewBins).toLong() - .coerceIn(0L, max(0L, durationUs - 1)) - try { - extractor.seekTo(targetUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC) - codec.flush() - } catch (_: Throwable) { - continue - } - - var sawInputEOS = false - var frames = 0L - var sumSquares = 0.0 - var sampleCount = 0L - var safety = 0 - - while (frames < windowFrames && safety++ < 180) { - if (!sawInputEOS) { - val inIndex = codec.dequeueInputBuffer(2_000) - if (inIndex >= 0) { - val inBuf = codec.getInputBuffer(inIndex)!! - val size = extractor.readSampleData(inBuf, 0) - if (size < 0) { - codec.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM) - sawInputEOS = true - } else { - codec.queueInputBuffer(inIndex, 0, size, extractor.sampleTime, 0) - extractor.advance() - } - } - } - - when (val outIndex = codec.dequeueOutputBuffer(info, 2_000)) { - MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { - val nf = codec.outputFormat - if (nf.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) channelCount = nf.getInteger(MediaFormat.KEY_CHANNEL_COUNT) - if (nf.containsKey(MediaFormat.KEY_PCM_ENCODING)) { - pcmFloat = nf.getInteger(MediaFormat.KEY_PCM_ENCODING) == AudioFormat.ENCODING_PCM_FLOAT - } - } - MediaCodec.INFO_TRY_AGAIN_LATER -> { - if (sawInputEOS) break - } - else -> if (outIndex >= 0) { - if (info.size > 0) { - val out = codec.getOutputBuffer(outIndex)!! - out.position(info.offset) - out.limit(info.offset + info.size) - out.order(ByteOrder.nativeOrder()) - val energy = collectEnergy(out, pcmFloat, channelCount, windowFrames - frames) - frames += energy.frameCount - sumSquares += energy.sumSquares - sampleCount += energy.sampleCount - } - val ended = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 - codec.releaseOutputBuffer(outIndex, false) - if (ended) break - } - } - } - - if (sampleCount > 0) { - peaks[bin] = sqrt(sumSquares / sampleCount).toFloat() - } - } - - var globalMax = 0f - for (value in peaks) if (value > globalMax) globalMax = value - if (globalMax > 0f) { - for (i in peaks.indices) peaks[i] /= globalMax - } - return peaks - } catch (_: Throwable) { - return FloatArray(0) - } finally { - try { codec?.stop() } catch (_: Throwable) {} - try { codec?.release() } catch (_: Throwable) {} - try { extractor.release() } catch (_: Throwable) {} - } - } - - private fun collectEnergy( - out: java.nio.ByteBuffer, - pcmFloat: Boolean, - channelCount: Int, - maxFrames: Long - ): PcmEnergy { - if (maxFrames <= 0) return PcmEnergy(0.0, 0L, 0L) - var sumSquares = 0.0 - var sampleCount = 0L - var frameCount = 0L - if (pcmFloat) { - val fb = out.asFloatBuffer() - while (fb.hasRemaining() && frameCount < maxFrames) { - var c = 0 - while (c < channelCount && fb.hasRemaining()) { - val s = fb.get().toDouble() - sumSquares += s * s - sampleCount++ - c++ - } - frameCount++ - } - } else { - val sb = out.asShortBuffer() - while (sb.hasRemaining() && frameCount < maxFrames) { - var c = 0 - while (c < channelCount && sb.hasRemaining()) { - val s = sb.get() / 32768.0 - sumSquares += s * s - sampleCount++ - c++ - } - frameCount++ - } - } - return PcmEnergy(sumSquares, sampleCount, frameCount) - } - // Gated integrated K-weighted loudness per ITU-R BS.1770 + absolute sample peak. // Two cascaded biquads (high-shelf pre-filter + RLB high-pass) per channel with // pyloudnorm-reference coefficients (so the -0.691 offset holds), accumulated into diff --git a/modules/astra-library-scanner/index.ts b/modules/astra-library-scanner/index.ts index 9f26027..e8e4b29 100644 --- a/modules/astra-library-scanner/index.ts +++ b/modules/astra-library-scanner/index.ts @@ -130,6 +130,16 @@ export interface TrackAnalysis { decoderName: string | null; mime: string | null; withLoudness: boolean; + queueWaitMs: number; + setupMs: number | null; + firstPcmMs: number | null; + firstProgressMs: number | null; + decodeToEosMs: number | null; + finalizeMs: number | null; + cleanupMs: number | null; + endToEndMs: number | null; + sampleRate: number | null; + channelCount: number | null; } declare class AstraLibraryScannerModuleType extends NativeModule { @@ -155,11 +165,6 @@ declare class AstraLibraryScannerModuleType extends NativeModule; - /** - * Decode short windows across the file and return approximate RMS peaks for - * immediate seek-bar paint. Cheap preview only; callers should not persist it. - */ - extractWaveformPreview(uri: string, bins: number): Promise; /** * Read ReplayGain track/album gain (dB) + peak (linear) from container tags * (ID3 TXXX / Vorbis comments / MP4 freeform) without decoding audio. All fields diff --git a/src/app/settings/troubleshooting.tsx b/src/app/settings/troubleshooting.tsx index c5a29d2..1e6f080 100644 --- a/src/app/settings/troubleshooting.tsx +++ b/src/app/settings/troubleshooting.tsx @@ -220,17 +220,16 @@ function AnalysisTimingPanel() { {timings.map((timing) => ( - {timing.kind === 'preview' - ? `preview · ${Math.round(timing.decodeMs)}ms` - : `${(timing.mime ?? 'audio/?').replace('audio/', '')} · ${Math.round(timing.decodeMs)}ms` + - (timing.realtimeFactor ? ` · ${Math.round(timing.realtimeFactor)}× realtime` : '')} + {`${(timing.mime ?? 'audio/?').replace('audio/', '')} · ${Math.round(timing.decodeMs)}ms` + + (timing.realtimeFactor ? ` · ${Math.round(timing.realtimeFactor)}× realtime` : '')} - {timing.kind === 'preview' - ? 'sparse first-paint pass' - : `${timing.decoderName ?? 'unknown decoder'}${ - timing.withLoudness ? ' · loudness folded in' : '' - }`} + {`${timing.decoderName ?? 'unknown decoder'}${ + timing.withLoudness ? ' · loudness folded in' : '' + } · wait ${Math.round(timing.nativeQueueWaitMs ?? 0)}ms`} + + + prep {Math.round(timing.preparationMs ?? 0)} · setup {Math.round(timing.setupMs ?? 0)} · first PCM {Math.round(timing.firstPcmMs ?? 0)} · first fill {timing.firstProgressEndToEndMs == null ? '—' : Math.round(timing.firstProgressEndToEndMs)} · persist {Math.round(timing.persistenceMs ?? 0)} · total {Math.round(timing.endToEndMs)}ms ))} diff --git a/src/audio/applyNormalization.ts b/src/audio/applyNormalization.ts index 78a0fa6..0d591ea 100644 --- a/src/audio/applyNormalization.ts +++ b/src/audio/applyNormalization.ts @@ -43,6 +43,12 @@ export async function applyNormalizationForActiveTrack(): Promise { await useAudioSettingsStore.getState().load(); const settings = useAudioSettingsStore.getState().asNormalizationSettings(); + if (!settings.enabled) { + setNormalizationGainNative(1); + setTrackGainNative(url, 1); + activateTrackGainNative(url); + return; + } let facts = EMPTY_FACTS; try { diff --git a/src/audio/trackAnalysis.ts b/src/audio/trackAnalysis.ts index bb57a04..b75c88f 100644 --- a/src/audio/trackAnalysis.ts +++ b/src/audio/trackAnalysis.ts @@ -95,22 +95,30 @@ function start(path: string, wantPeaks: boolean): Promise { } async function run(path: string, wantPeaks: boolean): Promise { + const startedAt = Date.now(); const generation = cacheGate.capture(); + const lookupStartedAt = Date.now(); const [row, cachedPeaks] = await Promise.all([ AstraLibraryData.getTrackLoudness([path]) .then((rows) => rows[0] ?? null) .catch(() => null), AstraLibraryData.getWaveform(path).catch(() => null), ]); + const cacheLookupMs = Date.now() - lookupStartedAt; let facts = factsFromRow(row); let peaks = cachedPeaks && cachedPeaks.length > 0 ? Float32Array.from(cachedPeaks) : null; + const waveformCacheHit = peaks !== null; + const loudnessCacheHit = facts.loudnessLufs !== null; + let replayGainMs = 0; + const settings = useAudioSettingsStore.getState().asNormalizationSettings(); // ReplayGain tags: container-only, no decode. Decoupled from loudness so a track measured // before ReplayGain was enabled still picks up its tags; rg_scanned stays unset on failure // so it retries next touch. - if (!row || row.rg_scanned !== 1) { + if (settings.enabled && (!row || row.rg_scanned !== 1)) { + const replayGainStartedAt = Date.now(); try { const rg = await AstraLibraryScanner.readReplayGain(path); await AstraLibraryData.setTrackReplayGain( @@ -129,27 +137,41 @@ async function run(path: string, wantPeaks: boolean): Promise 0) { peaks = Float32Array.from(analysis.peaks); await persistPeaks(path, peaks, generation); @@ -158,6 +180,15 @@ async function run(path: string, wantPeaks: boolean): Promise {}); facts = { ...facts, loudnessLufs: analysis.lufs, samplePeak: analysis.peak }; } + recordTiming(path, analysis, { + cacheLookupMs, + replayGainMs, + preparationMs, + persistenceMs: Date.now() - persistenceStartedAt, + endToEndMs: Date.now() - startedAt, + waveformCacheHit, + loudnessCacheHit, + }); return { peaks, facts }; } @@ -200,15 +231,6 @@ export function activeAnalysisPaths(): string[] { return Array.from(inflight.keys()); } -/** - * Whether a decode for this path is already under way — i.e. progress events are about to - * start arriving, so a second "fast preview" decode would only land late and cause a - * visible rescale rather than buying a faster first paint. - */ -export function isAnalysisRunning(path: string): boolean { - return inflight.has(path); -} - /** Drops cached waveform rows and stops in-flight decodes from writing them back. */ export async function clearWaveformCache(): Promise { inflight.clear(); @@ -224,8 +246,7 @@ export async function clearWaveformCache(): Promise { export interface AnalysisTiming { path: string; - /** 'preview' entries are the sparse first-paint decode, 'analysis' the real pass. */ - kind: 'analysis' | 'preview'; + kind: 'analysis'; decodeMs: number; durationMs: number | null; /** durationMs / decodeMs — how many times faster than realtime the decode ran. */ @@ -233,9 +254,35 @@ export interface AnalysisTiming { decoderName: string | null; mime: string | null; withLoudness: boolean; + cancelled: boolean; + cacheLookupMs: number | null; + replayGainMs: number | null; + preparationMs: number | null; + nativeQueueWaitMs: number | null; + setupMs: number | null; + firstPcmMs: number | null; + firstProgressMs: number | null; + firstProgressEndToEndMs: number | null; + decodeToEosMs: number | null; + finalizeMs: number | null; + persistenceMs: number | null; + cleanupMs: number | null; + endToEndMs: number; + waveformCacheHit: boolean | null; + loudnessCacheHit: boolean | null; at: number; } +interface AnalysisJsTiming { + cacheLookupMs: number; + replayGainMs: number; + preparationMs: number; + persistenceMs: number; + endToEndMs: number; + waveformCacheHit: boolean; + loudnessCacheHit: boolean; +} + const MAX_TIMINGS = 20; const recentTimings: AnalysisTiming[] = []; @@ -244,43 +291,45 @@ function push(timing: AnalysisTiming): void { if (recentTimings.length > MAX_TIMINGS) recentTimings.length = MAX_TIMINGS; } -/** - * Record how long the sparse preview decode took, measured end to end from JS (so it - * includes the wait for a native permit — which is the number that decides whether the - * preview can still beat the real decode's first progress event to the screen). - */ -export function recordPreviewTiming(path: string, elapsedMs: number): void { - push({ - path, - kind: 'preview', - decodeMs: elapsedMs, - durationMs: null, - realtimeFactor: null, - decoderName: null, - mime: null, - withLoudness: false, - at: Date.now(), - }); - if (__DEV__) console.log(`[analysis] preview ${elapsedMs.toFixed(0)}ms`); -} - -function recordTiming(path: string, analysis: TrackAnalysis): void { - if (analysis.cancelled || analysis.decodeMs == null) return; +function recordTiming(path: string, analysis: TrackAnalysis, js: AnalysisJsTiming): void { + if (analysis.decodeMs == null && analysis.endToEndMs == null) return; + const nativeQueueWaitMs = analysis.queueWaitMs ?? null; + const firstProgressMs = analysis.firstProgressMs ?? null; push({ path, kind: 'analysis', - decodeMs: analysis.decodeMs, + decodeMs: analysis.decodeMs ?? analysis.endToEndMs ?? 0, durationMs: analysis.durationMs, realtimeFactor: analysis.realtimeFactor, decoderName: analysis.decoderName, mime: analysis.mime, withLoudness: analysis.withLoudness, + cancelled: analysis.cancelled, + cacheLookupMs: js.cacheLookupMs, + replayGainMs: js.replayGainMs, + preparationMs: js.preparationMs, + nativeQueueWaitMs, + setupMs: analysis.setupMs, + firstPcmMs: analysis.firstPcmMs, + firstProgressMs, + firstProgressEndToEndMs: + firstProgressMs == null + ? null + : js.preparationMs + (nativeQueueWaitMs ?? 0) + firstProgressMs, + decodeToEosMs: analysis.decodeToEosMs, + finalizeMs: analysis.finalizeMs, + persistenceMs: js.persistenceMs, + cleanupMs: analysis.cleanupMs, + endToEndMs: js.endToEndMs, + waveformCacheHit: js.waveformCacheHit, + loudnessCacheHit: js.loudnessCacheHit, at: Date.now(), }); - if (__DEV__) { + if (__DEV__ && !analysis.cancelled) { const rt = analysis.realtimeFactor; + const decodeMs = analysis.decodeMs ?? analysis.endToEndMs ?? 0; console.log( - `[analysis] ${analysis.mime ?? '?'} ${analysis.decodeMs.toFixed(0)}ms` + + `[analysis] ${analysis.mime ?? '?'} ${decodeMs.toFixed(0)}ms` + `${rt ? ` (${rt.toFixed(0)}x realtime)` : ''}` + ` via ${analysis.decoderName ?? '?'}${analysis.withLoudness ? ' +loudness' : ''}` ); diff --git a/src/audio/useNormalizationSync.ts b/src/audio/useNormalizationSync.ts index bababff..dceefda 100644 --- a/src/audio/useNormalizationSync.ts +++ b/src/audio/useNormalizationSync.ts @@ -57,6 +57,16 @@ export function useNormalizationSync(): void { return; } + // Disabled means unity immediately and no metadata/decode work. The seek bar owns + // waveform-on-demand independently, so normalization must not precompute it here. + if (!settings.enabled) { + setNormalizationGainNative(1); + setTrackGainNative(path, 1); + activateTrackGainNative(path); + useScopeStore.getState().setOscGain(DEFAULT_OSC_GAIN); + return; + } + // Remote tracks have no local file to decode and no synced loudness/RG facts, // so normalization is unity. Skip analysis (it would try to download the stream). if (current?.sourceType && current.sourceType !== 'local') { @@ -111,6 +121,15 @@ export function useNormalizationSync(): void { const { tracks, activeIndex } = useQueueStore.getState(); if (activeIndex < 0) return; const settings = useAudioSettingsStore.getState().asNormalizationSettings(); + if (!settings.enabled) { + // A settings change may land while decode-ahead is already running. Preserve the + // current track (the visible seek bar may be awaiting it), but release queue work. + const currentPath = usePlayerStore.getState().currentTrack?.path; + for (const path of activeAnalysisPaths()) { + if (path !== currentPath) cancelTrackAnalysis(path); + } + return; + } // The set of tracks worth spending a decode on right now. Everything else that is // still decoding has been skipped past. diff --git a/src/components/WaveformSeekBar.tsx b/src/components/WaveformSeekBar.tsx index 928339e..2b4eb4a 100644 --- a/src/components/WaveformSeekBar.tsx +++ b/src/components/WaveformSeekBar.tsx @@ -25,7 +25,7 @@ import { formatDuration } from '@/lib/format'; import { downsampleWaveform, getWaveform, - mergeProgressiveWaveform, + normalizeProgressiveWaveform, subscribeWaveformProgress, } from '@/scope/waveform'; import { useAnimatedPlaybackProgress } from '@/audio/useAnimatedPlaybackProgress'; @@ -42,8 +42,8 @@ const BAR_WIDTH = 3; const BAR_GAP = 2; const MIN_BAR = 0.05; // floor so silent/idle sections still show a sliver const PLAYHEAD_WIDTH = 2; -/** Ascending confidence — a lower quality never overwrites a higher one for the same track. */ -type WaveformQuality = 'preview' | 'partial' | 'accurate'; +/** Ascending confidence — a partial result never overwrites an accurate cache hit. */ +type WaveformQuality = 'partial' | 'accurate'; interface WaveformSeekBarProps { onSeek: (seconds: number) => void; @@ -115,15 +115,6 @@ export function WaveformSeekBar({ trackKey: trackPath, overrideFraction: scrubFraction ?? heldFraction, }); - // The coarse preview is kept aside as well as rendered: it's the amplitude reference the - // partially-decoded prefix is scaled against, and it supplies the not-yet-decoded tail. - const previewRef = useRef<{ path: string; peaks: Float32Array } | null>(null); - // Whether progressive fill has already begun for the current track. A preview that shows - // up after that point is worse than useless: adopting it mid-fill rescales every bar at - // once (the prefix is scaled against the preview's amplitude), which reads as two - // different waveforms fighting. Once we're filling, the preview is dropped. - const progressStartedRef = useRef(false); - // Load (cache-first) the offline peaks whenever the track changes, and follow the decode // as it runs so the bars resolve left-to-right rather than snapping in at the end. The // progress subscription is independent of who started the decode — usually the queue @@ -131,40 +122,22 @@ export function WaveformSeekBar({ useEffect(() => { if (!trackPath) return; let cancelled = false; - previewRef.current = null; - progressStartedRef.current = false; const unsubscribe = subscribeWaveformProgress(trackPath, ({ peaks, totalBins }) => { if (cancelled) return; - progressStartedRef.current = true; - const preview = previewRef.current?.path === trackPath ? previewRef.current.peaks : null; - const merged = mergeProgressiveWaveform(peaks, totalBins, preview); + const progressive = normalizeProgressiveWaveform(peaks, totalBins); setLoaded((current) => { if (current?.path === trackPath && current.quality === 'accurate' && current.peaks) { return current; } - return { path: trackPath, peaks: merged, quality: 'partial' }; + return { path: trackPath, peaks: progressive, quality: 'partial' }; }); }); - void getWaveform(trackPath, { - onPreview: (peaks) => { - if (cancelled) return; - // Lost the race — the real decode is already painting. Adopting the preview now - // would rescale the whole bar in one frame. - if (progressStartedRef.current) return; - previewRef.current = { path: trackPath, peaks }; - setLoaded((current) => { - if (current?.path === trackPath && current.quality !== 'preview' && current.peaks) { - return current; - } - return { path: trackPath, peaks, quality: 'preview' }; - }); - }, - }).then((peaks) => { + void getWaveform(trackPath).then((peaks) => { if (cancelled) return; setLoaded((current) => { - // A failed decode must not wipe a good preview or partial fill. + // A failed decode must not wipe a good partial fill. if (!peaks && current?.path === trackPath && current.peaks) return current; return { path: trackPath, peaks, quality: 'accurate' }; }); diff --git a/src/scope/waveform.ts b/src/scope/waveform.ts index b9888e8..c8f680e 100644 --- a/src/scope/waveform.ts +++ b/src/scope/waveform.ts @@ -1,58 +1,27 @@ -// Waveform peaks for the seek bar: cache-first, preview-on-miss, then the accurate -// decode. The accurate pass lives in trackAnalysis (it shares one decode with loudness) -// and streams partial results back through onWaveformProgress, so the bar fills in -// left-to-right instead of snapping in when the whole file is done. +// Waveform peaks for the seek bar: cache-first, then the accurate decode. The accurate pass +// lives in trackAnalysis (it shares one decode with loudness) and streams partial results +// back through onWaveformProgress, so the bar fills in left-to-right instead of snapping in +// when the whole file is done. import { AstraLibraryData, AstraLibraryScanner } from '../../modules/astra-library-scanner'; import { WAVEFORM_BINS, clearWaveformCache, ensureTrackAnalysis, - isAnalysisRunning, - recordPreviewTiming, } from '@/audio/trackAnalysis'; export { WAVEFORM_BINS }; -export { downsampleWaveform, mergeProgressiveWaveform } from '@/scope/waveformMath'; -export const WAVEFORM_PREVIEW_BINS = 96; +export { downsampleWaveform, normalizeProgressiveWaveform } from '@/scope/waveformMath'; -export interface WaveformLoadOptions { - onPreview?: (peaks: Float32Array) => void; -} - -// Dedupe concurrent preview requests for the same track (e.g. mini-player + now-playing). -// The accurate decode is deduped inside trackAnalysis. -const previewInflight = new Map>(); - -export function getWaveform( - trackPath: string, - options: WaveformLoadOptions = {} -): Promise { +export function getWaveform(trackPath: string): Promise { if (!isLocalWaveformPath(trackPath)) return Promise.resolve(null); - return loadWaveform(trackPath, options); + return loadWaveform(trackPath); } -async function loadWaveform( - trackPath: string, - options: WaveformLoadOptions -): Promise { +async function loadWaveform(trackPath: string): Promise { const cached = await AstraLibraryData.getWaveform(trackPath).catch(() => null); if (cached && cached.length > 0) return Float32Array.from(cached); - // The preview is a SECOND native decode competing for the same two permits as the real - // pass. It only earns that cost when it can beat the real decode's first progress event - // to the screen. If a decode for this track is already running — the common case, since - // the queue prefetch starts one several tracks ahead — progress events are about to - // arrive immediately, and the preview would land late enough only to cause a visible - // rescale. Skip it entirely there. - if (options.onPreview && !isAnalysisRunning(trackPath)) { - const startedAt = Date.now(); - void getWaveformPreview(trackPath).then((preview) => { - recordPreviewTiming(trackPath, Date.now() - startedAt); - if (preview && preview.length > 0) options.onPreview?.(preview); - }); - } - // Shares one decode pass with loudness, and may already be running from the queue // prefetch — in which case this just joins it. Failures fall back to flat bars. try { @@ -65,7 +34,6 @@ async function loadWaveform( /** Deletes waveform rows and prevents decodes already in flight from writing them back. */ export async function clearAllWaveformCache(): Promise { - previewInflight.clear(); await clearWaveformCache(); } @@ -120,27 +88,6 @@ export function subscribeWaveformProgress( }; } -function getWaveformPreview(trackPath: string): Promise { - const existing = previewInflight.get(trackPath); - if (existing) return existing; - const task = decodePreviewWaveform(trackPath).finally(() => { - if (previewInflight.get(trackPath) === task) previewInflight.delete(trackPath); - }); - previewInflight.set(trackPath, task); - return task; -} - -async function decodePreviewWaveform(trackPath: string): Promise { - let raw: number[]; - try { - raw = await AstraLibraryScanner.extractWaveformPreview(trackPath, WAVEFORM_PREVIEW_BINS); - } catch { - return null; - } - if (!raw || raw.length === 0) return null; - return Float32Array.from(raw); -} - export function isLocalWaveformPath(trackPath: string): boolean { return trackPath.startsWith('content://') || trackPath.startsWith('file://'); } diff --git a/src/scope/waveformMath.test.mts b/src/scope/waveformMath.test.mts index 6a4f49c..726093a 100644 --- a/src/scope/waveformMath.test.mts +++ b/src/scope/waveformMath.test.mts @@ -1,81 +1,35 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { downsampleWaveform, mergeProgressiveWaveform } from './waveformMath.ts'; +import { downsampleWaveform, normalizeProgressiveWaveform } from './waveformMath.ts'; const TOTAL = 512; -/** Preview normalized to [0,1] across the whole track, as the native preview returns it. */ -function makePreview(values: number[]): Float32Array { - return Float32Array.from(values); -} - -test('merge with no prefix is just the stretched preview', () => { - const preview = makePreview([0.2, 0.8, 0.4, 1]); - const merged = mergeProgressiveWaveform(new Float32Array(0), TOTAL, preview); - assert.equal(merged.length, TOTAL); - assert.ok(Math.abs(merged[0] - 0.2) < 1e-6); - assert.ok(Math.abs(merged[TOTAL - 1] - 1) < 1e-6); - // Each preview value should occupy an equal quarter of the width. - assert.ok(Math.abs(merged[Math.floor(TOTAL * 0.3)] - 0.8) < 1e-6); +test('an empty progressive prefix leaves the waveform empty', () => { + const progressive = normalizeProgressiveWaveform(new Float32Array(0), TOTAL); + assert.equal(progressive.length, TOTAL); + assert.ok(progressive.every((value) => value === 0)); }); -test('merge with no preview normalizes the prefix against its own max', () => { +test('progressive prefix normalizes against its current maximum', () => { const prefix = Float32Array.from([0.01, 0.02, 0.04]); // raw RMS, tiny absolute values - const merged = mergeProgressiveWaveform(prefix, TOTAL, null); - assert.ok(Math.abs(merged[2] - 1) < 1e-6, 'loudest decoded bin should reach full scale'); - assert.ok(Math.abs(merged[0] - 0.25) < 1e-6); - assert.equal(merged[3], 0, 'undecoded tail stays empty without a preview'); + const progressive = normalizeProgressiveWaveform(prefix, TOTAL); + assert.ok(Math.abs(progressive[2] - 1) < 1e-6, 'loudest decoded bin should reach full scale'); + assert.ok(Math.abs(progressive[0] - 0.25) < 1e-6); + assert.equal(progressive[3], 0, 'undecoded tail stays empty'); }); -test('prefix is rescaled to the preview, not to its own max', () => { - // Preview says the first half is quiet (0.25) and the second half is loud (1.0). - const preview = makePreview([0.25, 0.25, 1, 1]); - // We have decoded the quiet first half only. Raw RMS values are arbitrary in scale. - const prefix = new Float32Array(TOTAL / 2).fill(0.003); - const merged = mergeProgressiveWaveform(prefix, TOTAL, preview); - - // Naive self-normalization would put the decoded half at 1.0 — far louder than the - // preview says it is, and louder than the not-yet-decoded loud half. It must stay at - // the preview's amplitude for that region instead. - assert.ok(Math.abs(merged[0] - 0.25) < 1e-6, `decoded region should match preview scale, got ${merged[0]}`); - assert.ok(merged[0] < merged[TOTAL - 1], 'quiet decoded half must stay below the loud undecoded half'); - assert.ok(Math.abs(merged[TOTAL - 1] - 1) < 1e-6, 'undecoded tail keeps the preview value'); -}); - -test('merge never exceeds full scale', () => { - const preview = makePreview([1, 1, 1, 1]); +test('progressive waveform never exceeds full scale', () => { const prefix = Float32Array.from([5, 10, 2]); - const merged = mergeProgressiveWaveform(prefix, TOTAL, preview); - for (let i = 0; i < merged.length; i++) { - assert.ok(merged[i] <= 1, `bin ${i} exceeded 1: ${merged[i]}`); - assert.ok(merged[i] >= 0, `bin ${i} went negative: ${merged[i]}`); + const progressive = normalizeProgressiveWaveform(prefix, TOTAL); + for (let i = 0; i < progressive.length; i++) { + assert.ok(progressive[i] <= 1, `bin ${i} exceeded 1: ${progressive[i]}`); + assert.ok(progressive[i] >= 0, `bin ${i} went negative: ${progressive[i]}`); } }); -test('an all-silent prefix falls back to the preview rather than blanking the bar', () => { - const preview = makePreview([0.5, 0.6, 0.7, 0.8]); - const merged = mergeProgressiveWaveform(new Float32Array(64), TOTAL, preview); - assert.ok(Math.abs(merged[0] - 0.5) < 1e-6); -}); - -test('downsampling a partially-filled merge keeps the decoded region proportionate', () => { - // Decoded half is quiet, undecoded half is loud — after downsampling the relationship - // must survive, i.e. the global normalize must not lift the quiet decoded half. - const preview = makePreview([0.25, 0.25, 1, 1]); - const prefix = new Float32Array(TOTAL / 2).fill(0.003); - const merged = mergeProgressiveWaveform(prefix, TOTAL, preview); - - const bars = downsampleWaveform(merged, 64); - assert.equal(bars.length, 64); - const firstQuarter = bars[8]; - const lastQuarter = bars[56]; - assert.ok( - lastQuarter > firstQuarter * 4, - `loud half (${lastQuarter}) should dominate the quiet decoded half (${firstQuarter})` - ); - for (let i = 0; i < bars.length; i++) { - assert.ok(bars[i] >= 0 && bars[i] <= 1, `bar ${i} out of range: ${bars[i]}`); - } +test('an all-silent progressive prefix remains empty', () => { + const progressive = normalizeProgressiveWaveform(new Float32Array(64), TOTAL); + assert.ok(progressive.every((value) => value === 0)); }); test('downsample is unchanged for a fully accurate waveform', () => { diff --git a/src/scope/waveformMath.ts b/src/scope/waveformMath.ts index 45aa543..8918988 100644 --- a/src/scope/waveformMath.ts +++ b/src/scope/waveformMath.ts @@ -1,33 +1,14 @@ // Pure waveform shaping — no native imports, so it stays unit-testable under `node --test` // (see waveformMath.test.mts). waveform.ts re-exports these. -/** - * Splice a partially-decoded raw RMS prefix over the coarse preview, so the bar fills - * left-to-right with no visible seam. - * - * The prefix is raw — mid-decode the native side can't know the track's global max — while - * the preview is already normalized against the whole track. So the prefix is rescaled to - * the preview's amplitude over the region it covers rather than to its own max; normalizing - * it independently would make the decoded part read far louder than the rest until a loud - * section happened to arrive. - */ -export function mergeProgressiveWaveform( +/** Normalize the raw RMS prefix emitted mid-decode and leave the unfinished tail empty. */ +export function normalizeProgressiveWaveform( prefix: Float32Array, - totalBins: number, - preview: Float32Array | null + totalBins: number ): Float32Array { const out = new Float32Array(Math.max(0, totalBins)); if (totalBins <= 0) return out; - // Stretch the (much coarser) preview across the full width first. - const hasPreview = !!preview && preview.length > 0; - if (preview && hasPreview) { - for (let i = 0; i < totalBins; i++) { - const p = Math.min(preview.length - 1, Math.floor((i / totalBins) * preview.length)); - out[i] = preview[p]; - } - } - const filled = Math.min(prefix.length, totalBins); if (filled === 0) return out; @@ -35,12 +16,7 @@ export function mergeProgressiveWaveform( for (let i = 0; i < filled; i++) if (prefix[i] > prefixMax) prefixMax = prefix[i]; if (prefixMax <= 0) return out; - // Match the preview's scale over the decoded region so the seam is continuous. - let reference = 0; - if (hasPreview) { - for (let i = 0; i < filled; i++) if (out[i] > reference) reference = out[i]; - } - const scale = (reference > 0 ? reference : 1) / prefixMax; + const scale = 1 / prefixMax; for (let i = 0; i < filled; i++) out[i] = Math.min(1, prefix[i] * scale); return out; }