astra/astra-mobile
1
0
Fork 0
mirror of https://github.com/Boof2015/astra-mobile.git synced 2026-08-28 17:11:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

faster waveform generation

Browse Source
This commit is contained in:
Boof2015
2026-08-28 02:53:08 -04:00
parent 5ad80a36dc
commit 0763ba0294
10 changed files with 252 additions and 428 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/astra-library-scanner/android/src/main/java/expo/modules/astralibraryscanner/AstraLibraryScannerModule.kt
+82 -186
View File
@@ -64,6 +64,8 @@ import kotlin.math.roundToInt
private const val LIBRARY_SCAN_LOG_TAG = "AstraLibraryScan"
private fun nanosToDoubleMs(nanos: Long): Double = nanos / 1_000_000.0
private data class MetadataStageTiming(
val androidMetadataNanos: Long,
val multiArtistNanos: Long,
@@ -170,6 +172,24 @@ class AudioAnalysis : Record {
@Field var mime: String? = null
/** Whether the loudness meter rode along on this pass. */
@Field var withLoudness: Boolean = false
/** Time spent waiting for one of the two shared native analysis lanes. */
@Field var queueWaitMs: Double = 0.0
/** Extractor discovery + codec creation/configuration/start. */
@Field var setupMs: Double? = null
/** Active-pass time until the first non-empty PCM output buffer. */
@Field var firstPcmMs: Double? = null
/** Active-pass time when the first progressive waveform could be emitted. */
@Field var firstProgressMs: Double? = null
/** Codec-start through end-of-stream, including the selected PCM workload. */
@Field var decodeToEosMs: Double? = null
/** Final RMS normalization and/or LUFS gate resolution. */
@Field var finalizeMs: Double? = null
/** Handler/codec/extractor teardown after the result was finalized. */
@Field var cleanupMs: Double? = null
/** Whole native request including semaphore wait and teardown. */
@Field var endToEndMs: Double? = null
@Field var sampleRate: Int? = null
@Field var channelCount: Int? = null
}
/** ReplayGain tags read from the container (no audio decode). Null = tag absent. */
@@ -293,12 +313,26 @@ class AstraLibraryScannerModule : Module() {
AsyncFunction("analyzeTrack") Coroutine { uri: String, bins: Int, withLoudness: Boolean ->
val flag = AtomicBoolean(false)
activeAnalyses[uri] = flag
val requestedNanos = System.nanoTime()
try {
waveformSemaphore.withPermit {
val queueWaitMs = nanosToDoubleMs(System.nanoTime() - requestedNanos)
// Cancelled while queued behind another decode — don't start at all.
if (flag.get()) AudioAnalysis().apply { cancelled = true }
if (flag.get()) AudioAnalysis().apply {
cancelled = true
this.queueWaitMs = queueWaitMs
endToEndMs = nanosToDoubleMs(System.nanoTime() - requestedNanos)
}
else withContext(Dispatchers.IO) {
runAnalysis(uri, if (bins > 0) bins else 512, withLoudness, flag)
runAnalysis(
uriStr = uri,
bins = if (bins > 0) bins else 512,
withLoudness = withLoudness,
cancelFlag = flag,
).also { analysis ->
analysis.queueWaitMs = queueWaitMs
analysis.endToEndMs = nanosToDoubleMs(System.nanoTime() - requestedNanos)
}
}
}
} finally {
@@ -312,15 +346,6 @@ class AstraLibraryScannerModule : Module() {
activeAnalyses[uri]?.set(true)
}
// Fast waveform preview for first paint: sparse short-window decode across
// the file. The JS side shows this immediately as the coarse full-width shape
// that the progressive accurate pass then fills over; only analyzeTrack persists.
AsyncFunction("extractWaveformPreview") Coroutine { uri: String, bins: Int ->
waveformSemaphore.withPermit {
withContext(Dispatchers.IO) { decodeWaveformPreview(uri, if (bins > 0) bins else 96) }
}
}
// ReplayGain tags (M4): reads container metadata only (no PCM decode), so it is
// cheap and lets us normalize a tagged library without the slow loudness decode.
AsyncFunction("readReplayGain") Coroutine { uri: String ->
@@ -914,6 +939,8 @@ class AstraLibraryScannerModule : Module() {
var codec: MediaCodec? = null
var handlerThread: HandlerThread? = null
val startNanos = System.nanoTime()
var codecStartNanos = 0L
var finalizedNanos = 0L
try {
extractor.setDataSource(context, uri, null)
@@ -934,15 +961,19 @@ class AstraLibraryScannerModule : Module() {
val sampleRate =
if (format.containsKey(MediaFormat.KEY_SAMPLE_RATE)) format.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100
result.sampleRate = sampleRate
val durationUs =
if (format.containsKey(MediaFormat.KEY_DURATION)) format.getLong(MediaFormat.KEY_DURATION) else 0L
result.durationMs = durationUs / 1000.0
val totalFrames = max(1L, (durationUs / 1_000_000.0 * sampleRate).toLong())
val acc = AnalyzeAccumulator(bins, totalFrames, withLoudness)
acc.sampleRate = sampleRate
acc.channelCount =
val initialChannels =
if (format.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) else 2
result.channelCount = initialChannels
val acc = AnalyzeAccumulator(bins, totalFrames, withLoudness).also {
it.sampleRate = sampleRate
it.channelCount = initialChannels
}
val decoder = createAnalysisDecoder(mime)
codec = decoder
@@ -992,6 +1023,9 @@ class AstraLibraryScannerModule : Module() {
return
}
if (info.size > 0) {
if (result.firstPcmMs == null) {
result.firstPcmMs = nanosToDoubleMs(System.nanoTime() - startNanos)
}
val out = mc.getOutputBuffer(index)
if (out != null) {
out.position(info.offset)
@@ -1007,12 +1041,22 @@ class AstraLibraryScannerModule : Module() {
// in at the end. Skipped on the EOS buffer — the promise carries the final,
// globally-normalized result a moment later.
val now = System.nanoTime()
if (!eos && acc.filledBins > lastEmitBin && now - lastEmitNanos >= progressEmitNanos) {
if (
!eos &&
acc.filledBins > lastEmitBin &&
now - lastEmitNanos >= progressEmitNanos
) {
lastEmitNanos = now
lastEmitBin = acc.filledBins
if (result.firstProgressMs == null) {
result.firstProgressMs = nanosToDoubleMs(now - startNanos)
}
emitWaveformProgress(uriStr, acc, lastEmitBin)
}
if (eos) done.complete(Unit)
if (eos) {
result.decodeToEosMs = nanosToDoubleMs(now - codecStartNanos)
done.complete(Unit)
}
} catch (_: Throwable) {
done.complete(Unit)
}
@@ -1020,10 +1064,14 @@ class AstraLibraryScannerModule : Module() {
override fun onOutputFormatChanged(mc: MediaCodec, fmt: MediaFormat) {
if (fmt.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) {
acc.channelCount = fmt.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
val channels = fmt.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
result.channelCount = channels
acc.channelCount = channels
}
if (fmt.containsKey(MediaFormat.KEY_SAMPLE_RATE)) {
acc.sampleRate = fmt.getInteger(MediaFormat.KEY_SAMPLE_RATE)
val rate = fmt.getInteger(MediaFormat.KEY_SAMPLE_RATE)
result.sampleRate = rate
acc.sampleRate = rate
}
if (fmt.containsKey(MediaFormat.KEY_PCM_ENCODING)) {
acc.pcmFloat =
@@ -1039,7 +1087,9 @@ class AstraLibraryScannerModule : Module() {
)
codec.configure(format, null, null, 0)
codecStartNanos = System.nanoTime()
codec.start()
result.setupMs = nanosToDoubleMs(System.nanoTime() - startNanos)
if (withTimeoutOrNull(analysisTimeoutMs) { done.await() } == null) {
// Timed out: peaks are partial, so treat it as a cancellation rather than caching
@@ -1048,12 +1098,15 @@ class AstraLibraryScannerModule : Module() {
}
if (result.cancelled) return result
val finalizeStartNanos = System.nanoTime()
result.peaks = acc.finalPeaks()
if (withLoudness) {
result.lufs = acc.loudness
result.peak = acc.samplePeak
}
val decodeMs = (System.nanoTime() - startNanos) / 1_000_000.0
finalizedNanos = System.nanoTime()
result.finalizeMs = nanosToDoubleMs(finalizedNanos - finalizeStartNanos)
val decodeMs = nanosToDoubleMs(finalizedNanos - startNanos)
result.decodeMs = decodeMs
val dur = result.durationMs
if (dur != null && dur > 0 && decodeMs > 0) result.realtimeFactor = dur / decodeMs
@@ -1061,6 +1114,7 @@ class AstraLibraryScannerModule : Module() {
} catch (_: Throwable) {
return result
} finally {
val cleanupStartNanos = System.nanoTime()
// Order matters: stopping the codec while a callback is mid-flight on the handler
// thread can crash. Quit the looper and wait for the in-flight callback to drain
// first, THEN tear the codec down.
@@ -1071,6 +1125,12 @@ class AstraLibraryScannerModule : Module() {
try { codec?.stop() } catch (_: Throwable) {}
try { codec?.release() } catch (_: Throwable) {}
try { extractor.release() } catch (_: Throwable) {}
val finishedNanos = System.nanoTime()
result.cleanupMs = nanosToDoubleMs(finishedNanos - cleanupStartNanos)
if (result.decodeMs == null && finalizedNanos == 0L) {
result.decodeMs = nanosToDoubleMs(cleanupStartNanos - startNanos)
}
result.endToEndMs = result.queueWaitMs + nanosToDoubleMs(finishedNanos - startNanos)
}
}
@@ -1097,7 +1157,9 @@ class AstraLibraryScannerModule : Module() {
// realtime playback, not bulk decode, and the instance is a scarce global resource shared
// with the track that is actually playing — so prefer a software decoder and fall back to
// the platform's default pick.
private fun createAnalysisDecoder(mime: String): MediaCodec {
private fun createAnalysisDecoder(
mime: String,
): MediaCodec {
try {
val info = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.firstOrNull { c ->
!c.isEncoder &&
@@ -1244,172 +1306,6 @@ class AstraLibraryScannerModule : Module() {
}
}
private data class PcmEnergy(
val sumSquares: Double,
val sampleCount: Long,
val frameCount: Long
)
// Sparse preview waveform: seek to a bounded number of points, decode a very
// short audio window at each point, and normalize those RMS samples. This is
// intentionally approximate; runAnalysis remains the accurate cache fill.
private fun decodeWaveformPreview(uriStr: String, bins: Int): FloatArray {
val context = requireContext()
val previewBins = bins.coerceIn(16, 128)
val uri = Uri.parse(uriStr)
val extractor = MediaExtractor()
var codec: MediaCodec? = null
try {
extractor.setDataSource(context, uri, null)
var trackFormat: MediaFormat? = null
var trackIndex = -1
for (i in 0 until extractor.trackCount) {
val f = extractor.getTrackFormat(i)
if (f.getString(MediaFormat.KEY_MIME)?.startsWith("audio/") == true) {
trackFormat = f; trackIndex = i; break
}
}
val format = trackFormat ?: return FloatArray(0)
extractor.selectTrack(trackIndex)
val mime = format.getString(MediaFormat.KEY_MIME) ?: return FloatArray(0)
val durationUs =
if (format.containsKey(MediaFormat.KEY_DURATION)) format.getLong(MediaFormat.KEY_DURATION) else 0L
if (durationUs <= 0L) return FloatArray(0)
val sampleRate =
if (format.containsKey(MediaFormat.KEY_SAMPLE_RATE)) format.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100
var channelCount =
if (format.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) else 2
var pcmFloat = false
val windowFrames = max(512L, (sampleRate * 0.018).roundToInt().toLong())
val peaks = FloatArray(previewBins)
codec = MediaCodec.createDecoderByType(mime)
codec.configure(format, null, null, 0)
codec.start()
val info = MediaCodec.BufferInfo()
for (bin in 0 until previewBins) {
val targetUs = ((durationUs.toDouble() * bin) / previewBins).toLong()
.coerceIn(0L, max(0L, durationUs - 1))
try {
extractor.seekTo(targetUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC)
codec.flush()
} catch (_: Throwable) {
continue
}
var sawInputEOS = false
var frames = 0L
var sumSquares = 0.0
var sampleCount = 0L
var safety = 0
while (frames < windowFrames && safety++ < 180) {
if (!sawInputEOS) {
val inIndex = codec.dequeueInputBuffer(2_000)
if (inIndex >= 0) {
val inBuf = codec.getInputBuffer(inIndex)!!
val size = extractor.readSampleData(inBuf, 0)
if (size < 0) {
codec.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
sawInputEOS = true
} else {
codec.queueInputBuffer(inIndex, 0, size, extractor.sampleTime, 0)
extractor.advance()
}
}
}
when (val outIndex = codec.dequeueOutputBuffer(info, 2_000)) {
MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
val nf = codec.outputFormat
if (nf.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) channelCount = nf.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
if (nf.containsKey(MediaFormat.KEY_PCM_ENCODING)) {
pcmFloat = nf.getInteger(MediaFormat.KEY_PCM_ENCODING) == AudioFormat.ENCODING_PCM_FLOAT
}
}
MediaCodec.INFO_TRY_AGAIN_LATER -> {
if (sawInputEOS) break
}
else -> if (outIndex >= 0) {
if (info.size > 0) {
val out = codec.getOutputBuffer(outIndex)!!
out.position(info.offset)
out.limit(info.offset + info.size)
out.order(ByteOrder.nativeOrder())
val energy = collectEnergy(out, pcmFloat, channelCount, windowFrames - frames)
frames += energy.frameCount
sumSquares += energy.sumSquares
sampleCount += energy.sampleCount
}
val ended = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
codec.releaseOutputBuffer(outIndex, false)
if (ended) break
}
}
}
if (sampleCount > 0) {
peaks[bin] = sqrt(sumSquares / sampleCount).toFloat()
}
}
var globalMax = 0f
for (value in peaks) if (value > globalMax) globalMax = value
if (globalMax > 0f) {
for (i in peaks.indices) peaks[i] /= globalMax
}
return peaks
} catch (_: Throwable) {
return FloatArray(0)
} finally {
try { codec?.stop() } catch (_: Throwable) {}
try { codec?.release() } catch (_: Throwable) {}
try { extractor.release() } catch (_: Throwable) {}
}
}
private fun collectEnergy(
out: java.nio.ByteBuffer,
pcmFloat: Boolean,
channelCount: Int,
maxFrames: Long
): PcmEnergy {
if (maxFrames <= 0) return PcmEnergy(0.0, 0L, 0L)
var sumSquares = 0.0
var sampleCount = 0L
var frameCount = 0L
if (pcmFloat) {
val fb = out.asFloatBuffer()
while (fb.hasRemaining() && frameCount < maxFrames) {
var c = 0
while (c < channelCount && fb.hasRemaining()) {
val s = fb.get().toDouble()
sumSquares += s * s
sampleCount++
c++
}
frameCount++
}
} else {
val sb = out.asShortBuffer()
while (sb.hasRemaining() && frameCount < maxFrames) {
var c = 0
while (c < channelCount && sb.hasRemaining()) {
val s = sb.get() / 32768.0
sumSquares += s * s
sampleCount++
c++
}
frameCount++
}
}
return PcmEnergy(sumSquares, sampleCount, frameCount)
}
// Gated integrated K-weighted loudness per ITU-R BS.1770 + absolute sample peak.
// Two cascaded biquads (high-shelf pre-filter + RLB high-pass) per channel with
// pyloudnorm-reference coefficients (so the -0.691 offset holds), accumulated into
modules/astra-library-scanner/index.ts
+10 -5
View File
@@ -130,6 +130,16 @@ export interface TrackAnalysis {
decoderName: string | null;
mime: string | null;
withLoudness: boolean;
queueWaitMs: number;
setupMs: number | null;
firstPcmMs: number | null;
firstProgressMs: number | null;
decodeToEosMs: number | null;
finalizeMs: number | null;
cleanupMs: number | null;
endToEndMs: number | null;
sampleRate: number | null;
channelCount: number | null;
}
declare class AstraLibraryScannerModuleType extends NativeModule<AstraLibraryScannerEvents> {
@@ -155,11 +165,6 @@ declare class AstraLibraryScannerModuleType extends NativeModule<AstraLibrarySca
* track stops burning CPU. Safe to call when nothing is running.
*/
cancelAnalysis(uri: string): Promise<void>;
/**
* Decode short windows across the file and return approximate RMS peaks for
* immediate seek-bar paint. Cheap preview only; callers should not persist it.
*/
extractWaveformPreview(uri: string, bins: number): Promise<number[]>;
/**
* Read ReplayGain track/album gain (dB) + peak (linear) from container tags
* (ID3 TXXX / Vorbis comments / MP4 freeform) without decoding audio. All fields
src/app/settings/troubleshooting.tsx
+8 -9
View File
@@ -220,17 +220,16 @@ function AnalysisTimingPanel() {
{timings.map((timing) => (
<View key={`${timing.path}-${timing.at}`} style={styles.timingRow}>
<Text variant="mono" color={colors.textPrimary}>
{timing.kind === 'preview'
? `preview · ${Math.round(timing.decodeMs)}ms`
: `${(timing.mime ?? 'audio/?').replace('audio/', '')} · ${Math.round(timing.decodeMs)}ms` +
(timing.realtimeFactor ? ` · ${Math.round(timing.realtimeFactor)}× realtime` : '')}
{`${(timing.mime ?? 'audio/?').replace('audio/', '')} · ${Math.round(timing.decodeMs)}ms` +
(timing.realtimeFactor ? ` · ${Math.round(timing.realtimeFactor)}× realtime` : '')}
</Text>
<Text variant="caption" color={colors.textSecondary} numberOfLines={1}>
{timing.kind === 'preview'
? 'sparse first-paint pass'
: `${timing.decoderName ?? 'unknown decoder'}${
timing.withLoudness ? ' · loudness folded in' : ''
}`}
{`${timing.decoderName ?? 'unknown decoder'}${
timing.withLoudness ? ' · loudness folded in' : ''
} · wait ${Math.round(timing.nativeQueueWaitMs ?? 0)}ms`}
</Text>
<Text variant="caption" color={colors.textTertiary} numberOfLines={2}>
prep {Math.round(timing.preparationMs ?? 0)} · setup {Math.round(timing.setupMs ?? 0)} · first PCM {Math.round(timing.firstPcmMs ?? 0)} · first fill {timing.firstProgressEndToEndMs == null ? '—' : Math.round(timing.firstProgressEndToEndMs)} · persist {Math.round(timing.persistenceMs ?? 0)} · total {Math.round(timing.endToEndMs)}ms
</Text>
</View>
))}
src/audio/applyNormalization.ts
+6
View File
@@ -43,6 +43,12 @@ export async function applyNormalizationForActiveTrack(): Promise<void> {
await useAudioSettingsStore.getState().load();
const settings = useAudioSettingsStore.getState().asNormalizationSettings();
if (!settings.enabled) {
setNormalizationGainNative(1);
setTrackGainNative(url, 1);
activateTrackGainNative(url);
return;
}
let facts = EMPTY_FACTS;
try {
src/audio/trackAnalysis.ts
+90 -41
View File
@@ -95,22 +95,30 @@ function start(path: string, wantPeaks: boolean): Promise<TrackAnalysisResult> {
}
async function run(path: string, wantPeaks: boolean): Promise<TrackAnalysisResult> {
const startedAt = Date.now();
const generation = cacheGate.capture();
const lookupStartedAt = Date.now();
const [row, cachedPeaks] = await Promise.all([
AstraLibraryData.getTrackLoudness([path])
.then((rows) => rows[0] ?? null)
.catch(() => null),
AstraLibraryData.getWaveform(path).catch(() => null),
]);
const cacheLookupMs = Date.now() - lookupStartedAt;
let facts = factsFromRow(row);
let peaks = cachedPeaks && cachedPeaks.length > 0 ? Float32Array.from(cachedPeaks) : null;
const waveformCacheHit = peaks !== null;
const loudnessCacheHit = facts.loudnessLufs !== null;
let replayGainMs = 0;
const settings = useAudioSettingsStore.getState().asNormalizationSettings();
// ReplayGain tags: container-only, no decode. Decoupled from loudness so a track measured
// before ReplayGain was enabled still picks up its tags; rg_scanned stays unset on failure
// so it retries next touch.
if (!row || row.rg_scanned !== 1) {
if (settings.enabled && (!row || row.rg_scanned !== 1)) {
const replayGainStartedAt = Date.now();
try {
const rg = await AstraLibraryScanner.readReplayGain(path);
await AstraLibraryData.setTrackReplayGain(
@@ -129,27 +137,41 @@ async function run(path: string, wantPeaks: boolean): Promise<TrackAnalysisResul
};
} catch {
/* tag read failed — fall through to a loudness measure */
} finally {
replayGainMs = Date.now() - replayGainStartedAt;
}
}
// Loudness only needs measuring when it's unknown AND ReplayGain can't cover the track.
const settings = useAudioSettingsStore.getState().asNormalizationSettings();
const needLoudness = facts.loudnessLufs == null && !hasUsableReplayGain(facts, settings);
const needLoudness =
settings.enabled && facts.loudnessLufs == null && !hasUsableReplayGain(facts, settings);
const needPeaks = wantPeaks && !peaks;
if (!needLoudness && !needPeaks) return { peaks, facts };
// Skipped past while we were reading the DB — don't start the decode at all.
if (cancelledPaths.has(path)) return { peaks, facts };
const preparationMs = Date.now() - startedAt;
let analysis: TrackAnalysis;
try {
analysis = await AstraLibraryScanner.analyzeTrack(path, WAVEFORM_BINS, needLoudness);
} catch {
return { peaks, facts };
}
recordTiming(path, analysis);
// Skipped past / timed out: peaks are truncated and loudness is partial. Cache neither.
if (analysis.cancelled) return { peaks, facts };
if (analysis.cancelled) {
recordTiming(path, analysis, {
cacheLookupMs,
replayGainMs,
preparationMs,
persistenceMs: 0,
endToEndMs: Date.now() - startedAt,
waveformCacheHit,
loudnessCacheHit,
});
return { peaks, facts };
}
const persistenceStartedAt = Date.now();
if (analysis.peaks && analysis.peaks.length > 0) {
peaks = Float32Array.from(analysis.peaks);
await persistPeaks(path, peaks, generation);
@@ -158,6 +180,15 @@ async function run(path: string, wantPeaks: boolean): Promise<TrackAnalysisResul
await AstraLibraryData.setTrackLoudness(path, analysis.lufs, analysis.peak).catch(() => {});
facts = { ...facts, loudnessLufs: analysis.lufs, samplePeak: analysis.peak };
}
recordTiming(path, analysis, {
cacheLookupMs,
replayGainMs,
preparationMs,
persistenceMs: Date.now() - persistenceStartedAt,
endToEndMs: Date.now() - startedAt,
waveformCacheHit,
loudnessCacheHit,
});
return { peaks, facts };
}
@@ -200,15 +231,6 @@ export function activeAnalysisPaths(): string[] {
return Array.from(inflight.keys());
}
/**
* Whether a decode for this path is already under way — i.e. progress events are about to
* start arriving, so a second "fast preview" decode would only land late and cause a
* visible rescale rather than buying a faster first paint.
*/
export function isAnalysisRunning(path: string): boolean {
return inflight.has(path);
}
/** Drops cached waveform rows and stops in-flight decodes from writing them back. */
export async function clearWaveformCache(): Promise<void> {
inflight.clear();
@@ -224,8 +246,7 @@ export async function clearWaveformCache(): Promise<void> {
export interface AnalysisTiming {
path: string;
/** 'preview' entries are the sparse first-paint decode, 'analysis' the real pass. */
kind: 'analysis' | 'preview';
kind: 'analysis';
decodeMs: number;
durationMs: number | null;
/** durationMs / decodeMs — how many times faster than realtime the decode ran. */
@@ -233,9 +254,35 @@ export interface AnalysisTiming {
decoderName: string | null;
mime: string | null;
withLoudness: boolean;
cancelled: boolean;
cacheLookupMs: number | null;
replayGainMs: number | null;
preparationMs: number | null;
nativeQueueWaitMs: number | null;
setupMs: number | null;
firstPcmMs: number | null;
firstProgressMs: number | null;
firstProgressEndToEndMs: number | null;
decodeToEosMs: number | null;
finalizeMs: number | null;
persistenceMs: number | null;
cleanupMs: number | null;
endToEndMs: number;
waveformCacheHit: boolean | null;
loudnessCacheHit: boolean | null;
at: number;
}
interface AnalysisJsTiming {
cacheLookupMs: number;
replayGainMs: number;
preparationMs: number;
persistenceMs: number;
endToEndMs: number;
waveformCacheHit: boolean;
loudnessCacheHit: boolean;
}
const MAX_TIMINGS = 20;
const recentTimings: AnalysisTiming[] = [];
@@ -244,43 +291,45 @@ function push(timing: AnalysisTiming): void {
if (recentTimings.length > MAX_TIMINGS) recentTimings.length = MAX_TIMINGS;
}
/**
* Record how long the sparse preview decode took, measured end to end from JS (so it
* includes the wait for a native permit — which is the number that decides whether the
* preview can still beat the real decode's first progress event to the screen).
*/
export function recordPreviewTiming(path: string, elapsedMs: number): void {
push({
path,
kind: 'preview',
decodeMs: elapsedMs,
durationMs: null,
realtimeFactor: null,
decoderName: null,
mime: null,
withLoudness: false,
at: Date.now(),
});
if (__DEV__) console.log(`[analysis] preview ${elapsedMs.toFixed(0)}ms`);
}
function recordTiming(path: string, analysis: TrackAnalysis): void {
if (analysis.cancelled || analysis.decodeMs == null) return;
function recordTiming(path: string, analysis: TrackAnalysis, js: AnalysisJsTiming): void {
if (analysis.decodeMs == null && analysis.endToEndMs == null) return;
const nativeQueueWaitMs = analysis.queueWaitMs ?? null;
const firstProgressMs = analysis.firstProgressMs ?? null;
push({
path,
kind: 'analysis',
decodeMs: analysis.decodeMs,
decodeMs: analysis.decodeMs ?? analysis.endToEndMs ?? 0,
durationMs: analysis.durationMs,
realtimeFactor: analysis.realtimeFactor,
decoderName: analysis.decoderName,
mime: analysis.mime,
withLoudness: analysis.withLoudness,
cancelled: analysis.cancelled,
cacheLookupMs: js.cacheLookupMs,
replayGainMs: js.replayGainMs,
preparationMs: js.preparationMs,
nativeQueueWaitMs,
setupMs: analysis.setupMs,
firstPcmMs: analysis.firstPcmMs,
firstProgressMs,
firstProgressEndToEndMs:
firstProgressMs == null
? null
: js.preparationMs + (nativeQueueWaitMs ?? 0) + firstProgressMs,
decodeToEosMs: analysis.decodeToEosMs,
finalizeMs: analysis.finalizeMs,
persistenceMs: js.persistenceMs,
cleanupMs: analysis.cleanupMs,
endToEndMs: js.endToEndMs,
waveformCacheHit: js.waveformCacheHit,
loudnessCacheHit: js.loudnessCacheHit,
at: Date.now(),
});
if (__DEV__) {
if (__DEV__ && !analysis.cancelled) {
const rt = analysis.realtimeFactor;
const decodeMs = analysis.decodeMs ?? analysis.endToEndMs ?? 0;
console.log(
`[analysis] ${analysis.mime ?? '?'} ${analysis.decodeMs.toFixed(0)}ms` +
`[analysis] ${analysis.mime ?? '?'} ${decodeMs.toFixed(0)}ms` +
`${rt ? ` (${rt.toFixed(0)}x realtime)` : ''}` +
` via ${analysis.decoderName ?? '?'}${analysis.withLoudness ? ' +loudness' : ''}`
);
src/audio/useNormalizationSync.ts
+19
View File
@@ -57,6 +57,16 @@ export function useNormalizationSync(): void {
return;
}
// Disabled means unity immediately and no metadata/decode work. The seek bar owns
// waveform-on-demand independently, so normalization must not precompute it here.
if (!settings.enabled) {
setNormalizationGainNative(1);
setTrackGainNative(path, 1);
activateTrackGainNative(path);
useScopeStore.getState().setOscGain(DEFAULT_OSC_GAIN);
return;
}
// Remote tracks have no local file to decode and no synced loudness/RG facts,
// so normalization is unity. Skip analysis (it would try to download the stream).
if (current?.sourceType && current.sourceType !== 'local') {
@@ -111,6 +121,15 @@ export function useNormalizationSync(): void {
const { tracks, activeIndex } = useQueueStore.getState();
if (activeIndex < 0) return;
const settings = useAudioSettingsStore.getState().asNormalizationSettings();
if (!settings.enabled) {
// A settings change may land while decode-ahead is already running. Preserve the
// current track (the visible seek bar may be awaiting it), but release queue work.
const currentPath = usePlayerStore.getState().currentTrack?.path;
for (const path of activeAnalysisPaths()) {
if (path !== currentPath) cancelTrackAnalysis(path);
}
return;
}
// The set of tracks worth spending a decode on right now. Everything else that is
// still decoding has been skipped past.
src/components/WaveformSeekBar.tsx
+7 -34
View File
@@ -25,7 +25,7 @@ import { formatDuration } from '@/lib/format';
import {
downsampleWaveform,
getWaveform,
mergeProgressiveWaveform,
normalizeProgressiveWaveform,
subscribeWaveformProgress,
} from '@/scope/waveform';
import { useAnimatedPlaybackProgress } from '@/audio/useAnimatedPlaybackProgress';
@@ -42,8 +42,8 @@ const BAR_WIDTH = 3;
const BAR_GAP = 2;
const MIN_BAR = 0.05; // floor so silent/idle sections still show a sliver
const PLAYHEAD_WIDTH = 2;
/** Ascending confidence — a lower quality never overwrites a higher one for the same track. */
type WaveformQuality = 'preview' | 'partial' | 'accurate';
/** Ascending confidence — a partial result never overwrites an accurate cache hit. */
type WaveformQuality = 'partial' | 'accurate';
interface WaveformSeekBarProps {
onSeek: (seconds: number) => void;
@@ -115,15 +115,6 @@ export function WaveformSeekBar({
trackKey: trackPath,
overrideFraction: scrubFraction ?? heldFraction,
});
// The coarse preview is kept aside as well as rendered: it's the amplitude reference the
// partially-decoded prefix is scaled against, and it supplies the not-yet-decoded tail.
const previewRef = useRef<{ path: string; peaks: Float32Array } | null>(null);
// Whether progressive fill has already begun for the current track. A preview that shows
// up after that point is worse than useless: adopting it mid-fill rescales every bar at
// once (the prefix is scaled against the preview's amplitude), which reads as two
// different waveforms fighting. Once we're filling, the preview is dropped.
const progressStartedRef = useRef(false);
// Load (cache-first) the offline peaks whenever the track changes, and follow the decode
// as it runs so the bars resolve left-to-right rather than snapping in at the end. The
// progress subscription is independent of who started the decode — usually the queue
@@ -131,40 +122,22 @@ export function WaveformSeekBar({
useEffect(() => {
if (!trackPath) return;
let cancelled = false;
previewRef.current = null;
progressStartedRef.current = false;
const unsubscribe = subscribeWaveformProgress(trackPath, ({ peaks, totalBins }) => {
if (cancelled) return;
progressStartedRef.current = true;
const preview = previewRef.current?.path === trackPath ? previewRef.current.peaks : null;
const merged = mergeProgressiveWaveform(peaks, totalBins, preview);
const progressive = normalizeProgressiveWaveform(peaks, totalBins);
setLoaded((current) => {
if (current?.path === trackPath && current.quality === 'accurate' && current.peaks) {
return current;
}
return { path: trackPath, peaks: merged, quality: 'partial' };
return { path: trackPath, peaks: progressive, quality: 'partial' };
});
});
void getWaveform(trackPath, {
onPreview: (peaks) => {
if (cancelled) return;
// Lost the race — the real decode is already painting. Adopting the preview now
// would rescale the whole bar in one frame.
if (progressStartedRef.current) return;
previewRef.current = { path: trackPath, peaks };
setLoaded((current) => {
if (current?.path === trackPath && current.quality !== 'preview' && current.peaks) {
return current;
}
return { path: trackPath, peaks, quality: 'preview' };
});
},
}).then((peaks) => {
void getWaveform(trackPath).then((peaks) => {
if (cancelled) return;
setLoaded((current) => {
// A failed decode must not wipe a good preview or partial fill.
// A failed decode must not wipe a good partial fill.
if (!peaks && current?.path === trackPath && current.peaks) return current;
return { path: trackPath, peaks, quality: 'accurate' };
});
src/scope/waveform.ts
+8 -61
View File
@@ -1,58 +1,27 @@
// Waveform peaks for the seek bar: cache-first, preview-on-miss, then the accurate
// decode. The accurate pass lives in trackAnalysis (it shares one decode with loudness)
// and streams partial results back through onWaveformProgress, so the bar fills in
// left-to-right instead of snapping in when the whole file is done.
// Waveform peaks for the seek bar: cache-first, then the accurate decode. The accurate pass
// lives in trackAnalysis (it shares one decode with loudness) and streams partial results
// back through onWaveformProgress, so the bar fills in left-to-right instead of snapping in
// when the whole file is done.
import { AstraLibraryData, AstraLibraryScanner } from '../../modules/astra-library-scanner';
import {
WAVEFORM_BINS,
clearWaveformCache,
ensureTrackAnalysis,
isAnalysisRunning,
recordPreviewTiming,
} from '@/audio/trackAnalysis';
export { WAVEFORM_BINS };
export { downsampleWaveform, mergeProgressiveWaveform } from '@/scope/waveformMath';
export const WAVEFORM_PREVIEW_BINS = 96;
export { downsampleWaveform, normalizeProgressiveWaveform } from '@/scope/waveformMath';
export interface WaveformLoadOptions {
onPreview?: (peaks: Float32Array) => void;
}
// Dedupe concurrent preview requests for the same track (e.g. mini-player + now-playing).
// The accurate decode is deduped inside trackAnalysis.
const previewInflight = new Map<string, Promise<Float32Array | null>>();
export function getWaveform(
trackPath: string,
options: WaveformLoadOptions = {}
): Promise<Float32Array | null> {
export function getWaveform(trackPath: string): Promise<Float32Array | null> {
if (!isLocalWaveformPath(trackPath)) return Promise.resolve(null);
return loadWaveform(trackPath, options);
return loadWaveform(trackPath);
}
async function loadWaveform(
trackPath: string,
options: WaveformLoadOptions
): Promise<Float32Array | null> {
async function loadWaveform(trackPath: string): Promise<Float32Array | null> {
const cached = await AstraLibraryData.getWaveform(trackPath).catch(() => null);
if (cached && cached.length > 0) return Float32Array.from(cached);
// The preview is a SECOND native decode competing for the same two permits as the real
// pass. It only earns that cost when it can beat the real decode's first progress event
// to the screen. If a decode for this track is already running — the common case, since
// the queue prefetch starts one several tracks ahead — progress events are about to
// arrive immediately, and the preview would land late enough only to cause a visible
// rescale. Skip it entirely there.
if (options.onPreview && !isAnalysisRunning(trackPath)) {
const startedAt = Date.now();
void getWaveformPreview(trackPath).then((preview) => {
recordPreviewTiming(trackPath, Date.now() - startedAt);
if (preview && preview.length > 0) options.onPreview?.(preview);
});
}
// Shares one decode pass with loudness, and may already be running from the queue
// prefetch — in which case this just joins it. Failures fall back to flat bars.
try {
@@ -65,7 +34,6 @@ async function loadWaveform(
/** Deletes waveform rows and prevents decodes already in flight from writing them back. */
export async function clearAllWaveformCache(): Promise<void> {
previewInflight.clear();
await clearWaveformCache();
}
@@ -120,27 +88,6 @@ export function subscribeWaveformProgress(
};
}
function getWaveformPreview(trackPath: string): Promise<Float32Array | null> {
const existing = previewInflight.get(trackPath);
if (existing) return existing;
const task = decodePreviewWaveform(trackPath).finally(() => {
if (previewInflight.get(trackPath) === task) previewInflight.delete(trackPath);
});
previewInflight.set(trackPath, task);
return task;
}
async function decodePreviewWaveform(trackPath: string): Promise<Float32Array | null> {
let raw: number[];
try {
raw = await AstraLibraryScanner.extractWaveformPreview(trackPath, WAVEFORM_PREVIEW_BINS);
} catch {
return null;
}
if (!raw || raw.length === 0) return null;
return Float32Array.from(raw);
}
export function isLocalWaveformPath(trackPath: string): boolean {
return trackPath.startsWith('content://') || trackPath.startsWith('file://');
}
src/scope/waveformMath.test.mts
+18 -64
View File
@@ -1,81 +1,35 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { downsampleWaveform, mergeProgressiveWaveform } from './waveformMath.ts';
import { downsampleWaveform, normalizeProgressiveWaveform } from './waveformMath.ts';
const TOTAL = 512;
/** Preview normalized to [0,1] across the whole track, as the native preview returns it. */
function makePreview(values: number[]): Float32Array {
return Float32Array.from(values);
}
test('merge with no prefix is just the stretched preview', () => {
const preview = makePreview([0.2, 0.8, 0.4, 1]);
const merged = mergeProgressiveWaveform(new Float32Array(0), TOTAL, preview);
assert.equal(merged.length, TOTAL);
assert.ok(Math.abs(merged[0] - 0.2) < 1e-6);
assert.ok(Math.abs(merged[TOTAL - 1] - 1) < 1e-6);
// Each preview value should occupy an equal quarter of the width.
assert.ok(Math.abs(merged[Math.floor(TOTAL * 0.3)] - 0.8) < 1e-6);
test('an empty progressive prefix leaves the waveform empty', () => {
const progressive = normalizeProgressiveWaveform(new Float32Array(0), TOTAL);
assert.equal(progressive.length, TOTAL);
assert.ok(progressive.every((value) => value === 0));
});
test('merge with no preview normalizes the prefix against its own max', () => {
test('progressive prefix normalizes against its current maximum', () => {
const prefix = Float32Array.from([0.01, 0.02, 0.04]); // raw RMS, tiny absolute values
const merged = mergeProgressiveWaveform(prefix, TOTAL, null);
assert.ok(Math.abs(merged[2] - 1) < 1e-6, 'loudest decoded bin should reach full scale');
assert.ok(Math.abs(merged[0] - 0.25) < 1e-6);
assert.equal(merged[3], 0, 'undecoded tail stays empty without a preview');
const progressive = normalizeProgressiveWaveform(prefix, TOTAL);
assert.ok(Math.abs(progressive[2] - 1) < 1e-6, 'loudest decoded bin should reach full scale');
assert.ok(Math.abs(progressive[0] - 0.25) < 1e-6);
assert.equal(progressive[3], 0, 'undecoded tail stays empty');
});
test('prefix is rescaled to the preview, not to its own max', () => {
// Preview says the first half is quiet (0.25) and the second half is loud (1.0).
const preview = makePreview([0.25, 0.25, 1, 1]);
// We have decoded the quiet first half only. Raw RMS values are arbitrary in scale.
const prefix = new Float32Array(TOTAL / 2).fill(0.003);
const merged = mergeProgressiveWaveform(prefix, TOTAL, preview);
// Naive self-normalization would put the decoded half at 1.0 — far louder than the
// preview says it is, and louder than the not-yet-decoded loud half. It must stay at
// the preview's amplitude for that region instead.
assert.ok(Math.abs(merged[0] - 0.25) < 1e-6, `decoded region should match preview scale, got ${merged[0]}`);
assert.ok(merged[0] < merged[TOTAL - 1], 'quiet decoded half must stay below the loud undecoded half');
assert.ok(Math.abs(merged[TOTAL - 1] - 1) < 1e-6, 'undecoded tail keeps the preview value');
});
test('merge never exceeds full scale', () => {
const preview = makePreview([1, 1, 1, 1]);
test('progressive waveform never exceeds full scale', () => {
const prefix = Float32Array.from([5, 10, 2]);
const merged = mergeProgressiveWaveform(prefix, TOTAL, preview);
for (let i = 0; i < merged.length; i++) {
assert.ok(merged[i] <= 1, `bin ${i} exceeded 1: ${merged[i]}`);
assert.ok(merged[i] >= 0, `bin ${i} went negative: ${merged[i]}`);
const progressive = normalizeProgressiveWaveform(prefix, TOTAL);
for (let i = 0; i < progressive.length; i++) {
assert.ok(progressive[i] <= 1, `bin ${i} exceeded 1: ${progressive[i]}`);
assert.ok(progressive[i] >= 0, `bin ${i} went negative: ${progressive[i]}`);
}
});
test('an all-silent prefix falls back to the preview rather than blanking the bar', () => {
const preview = makePreview([0.5, 0.6, 0.7, 0.8]);
const merged = mergeProgressiveWaveform(new Float32Array(64), TOTAL, preview);
assert.ok(Math.abs(merged[0] - 0.5) < 1e-6);
});
test('downsampling a partially-filled merge keeps the decoded region proportionate', () => {
// Decoded half is quiet, undecoded half is loud — after downsampling the relationship
// must survive, i.e. the global normalize must not lift the quiet decoded half.
const preview = makePreview([0.25, 0.25, 1, 1]);
const prefix = new Float32Array(TOTAL / 2).fill(0.003);
const merged = mergeProgressiveWaveform(prefix, TOTAL, preview);
const bars = downsampleWaveform(merged, 64);
assert.equal(bars.length, 64);
const firstQuarter = bars[8];
const lastQuarter = bars[56];
assert.ok(
lastQuarter > firstQuarter * 4,
`loud half (${lastQuarter}) should dominate the quiet decoded half (${firstQuarter})`
);
for (let i = 0; i < bars.length; i++) {
assert.ok(bars[i] >= 0 && bars[i] <= 1, `bar ${i} out of range: ${bars[i]}`);
}
test('an all-silent progressive prefix remains empty', () => {
const progressive = normalizeProgressiveWaveform(new Float32Array(64), TOTAL);
assert.ok(progressive.every((value) => value === 0));
});
test('downsample is unchanged for a fully accurate waveform', () => {
src/scope/waveformMath.ts
+4 -28
View File
@@ -1,33 +1,14 @@
// Pure waveform shaping — no native imports, so it stays unit-testable under `node --test`
// (see waveformMath.test.mts). waveform.ts re-exports these.
/**
* Splice a partially-decoded raw RMS prefix over the coarse preview, so the bar fills
* left-to-right with no visible seam.
*
* The prefix is raw — mid-decode the native side can't know the track's global max — while
* the preview is already normalized against the whole track. So the prefix is rescaled to
* the preview's amplitude over the region it covers rather than to its own max; normalizing
* it independently would make the decoded part read far louder than the rest until a loud
* section happened to arrive.
*/
export function mergeProgressiveWaveform(
/** Normalize the raw RMS prefix emitted mid-decode and leave the unfinished tail empty. */
export function normalizeProgressiveWaveform(
prefix: Float32Array,
totalBins: number,
preview: Float32Array | null
totalBins: number
): Float32Array {
const out = new Float32Array(Math.max(0, totalBins));
if (totalBins <= 0) return out;
// Stretch the (much coarser) preview across the full width first.
const hasPreview = !!preview && preview.length > 0;
if (preview && hasPreview) {
for (let i = 0; i < totalBins; i++) {
const p = Math.min(preview.length - 1, Math.floor((i / totalBins) * preview.length));
out[i] = preview[p];
}
}
const filled = Math.min(prefix.length, totalBins);
if (filled === 0) return out;
@@ -35,12 +16,7 @@ export function mergeProgressiveWaveform(
for (let i = 0; i < filled; i++) if (prefix[i] > prefixMax) prefixMax = prefix[i];
if (prefixMax <= 0) return out;
// Match the preview's scale over the decoded region so the seam is continuous.
let reference = 0;
if (hasPreview) {
for (let i = 0; i < filled; i++) if (out[i] > reference) reference = out[i];
}
const scale = (reference > 0 ? reference : 1) / prefixMax;
const scale = 1 / prefixMax;
for (let i = 0; i < filled; i++) out[i] = Math.min(1, prefix[i] * scale);
return out;
}