redo waveform and nomalization analysis

This commit is contained in:
Boof2015
2026-07-25 16:41:56 -04:00
parent a33a3b7137
commit 8b07b9e0e9
11 changed files with 1059 additions and 396 deletions
@@ -6,11 +6,15 @@ import android.graphics.Bitmap
import android.graphics.BitmapFactory import android.graphics.BitmapFactory
import android.media.AudioFormat import android.media.AudioFormat
import android.media.MediaCodec import android.media.MediaCodec
import android.media.MediaCodecInfo
import android.media.MediaCodecList
import android.media.MediaExtractor import android.media.MediaExtractor
import android.media.MediaFormat import android.media.MediaFormat
import android.media.MediaMetadataRetriever import android.media.MediaMetadataRetriever
import android.net.Uri import android.net.Uri
import android.os.Build import android.os.Build
import android.os.Handler
import android.os.HandlerThread
import android.provider.DocumentsContract import android.provider.DocumentsContract
import com.google.android.exoplayer2.MediaItem import com.google.android.exoplayer2.MediaItem
import com.google.android.exoplayer2.MetadataRetriever import com.google.android.exoplayer2.MetadataRetriever
@@ -18,8 +22,10 @@ import com.google.android.exoplayer2.metadata.id3.BinaryFrame
import com.google.android.exoplayer2.metadata.id3.InternalFrame import com.google.android.exoplayer2.metadata.id3.InternalFrame
import com.google.android.exoplayer2.metadata.id3.TextInformationFrame import com.google.android.exoplayer2.metadata.id3.TextInformationFrame
import com.google.android.exoplayer2.metadata.flac.VorbisComment import com.google.android.exoplayer2.metadata.flac.VorbisComment
import java.nio.ByteBuffer
import java.nio.ByteOrder import java.nio.ByteOrder
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import kotlin.math.PI import kotlin.math.PI
import kotlin.math.abs import kotlin.math.abs
import kotlin.math.log10 import kotlin.math.log10
@@ -35,6 +41,7 @@ import expo.modules.kotlin.records.Record
import expo.modules.astralibraryscanner.data.AstraLibraryRepository import expo.modules.astralibraryscanner.data.AstraLibraryRepository
import expo.modules.astralibraryscanner.data.LocalAudioFile import expo.modules.astralibraryscanner.data.LocalAudioFile
import expo.modules.astralibraryscanner.data.LocalAudioMetadata import expo.modules.astralibraryscanner.data.LocalAudioMetadata
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll import kotlinx.coroutines.awaitAll
@@ -42,6 +49,7 @@ import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.sync.withPermit
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeoutOrNull
import java.io.File import java.io.File
import java.security.MessageDigest import java.security.MessageDigest
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
@@ -52,11 +60,29 @@ class FileRequest : Record {
@Field val coverUri: String? = null @Field val coverUri: String? = null
} }
/** Result of one scan-time decode: waveform peaks + integrated loudness + sample peak. */ /**
* Result of ONE decode pass over a track: waveform peaks + integrated loudness + sample
* peak, plus timing so the JS side can report how fast the decode actually ran. Peaks and
* loudness share a pass because both need every sample; decoding twice was pure waste.
*/
class AudioAnalysis : Record { class AudioAnalysis : Record {
@Field var peaks: FloatArray = FloatArray(0) @Field var peaks: FloatArray = FloatArray(0)
@Field var lufs: Double? = null // integrated LUFS (negative dB); null if unmeasured @Field var lufs: Double? = null // integrated LUFS (negative dB); null if unmeasured
@Field var peak: Double? = null // absolute sample peak, linear [0,1]; null if unmeasured @Field var peak: Double? = null // absolute sample peak, linear [0,1]; null if unmeasured
/** True when the decode was cancelled mid-flight; peaks/lufs are then meaningless. */
@Field var cancelled: Boolean = false
/** Wall-clock decode time in ms — the number that decides whether we need a native decoder. */
@Field var decodeMs: Double? = null
/** Track duration in ms, from the container. */
@Field var durationMs: Double? = null
/** durationMs / decodeMs — "how many times faster than realtime". Higher is better. */
@Field var realtimeFactor: Double? = null
/** Which MediaCodec actually ran (e.g. "c2.android.flac.decoder"). */
@Field var decoderName: String? = null
/** Audio track mime (e.g. "audio/flac"). */
@Field var mime: String? = null
/** Whether the loudness meter rode along on this pass. */
@Field var withLoudness: Boolean = false
} }
/** ReplayGain tags read from the container (no audio decode). Null = tag absent. */ /** ReplayGain tags read from the container (no audio decode). Null = tag absent. */
@@ -75,13 +101,20 @@ class AstraLibraryScannerModule : Module() {
// read and hashed once, not once per track. // read and hashed once, not once per track.
private val coverHashMemo = ConcurrentHashMap<String, String>() private val coverHashMemo = ConcurrentHashMap<String, String>()
// Waveform decode is whole-file and CPU-heavy; throttle concurrent decodes. // Analysis decode is whole-file and CPU-heavy; throttle concurrent decodes. Also keeps
// us from monopolising decoder instances while a track is actually playing.
private val waveformSemaphore = Semaphore(2) private val waveformSemaphore = Semaphore(2)
// Cancellation flags for in-flight analyses, keyed by track URI. Set by cancelAnalysis
// so a skipped-past track stops burning CPU instead of running to completion holding a
// semaphore permit. Registered before the permit is acquired, so a queued-but-unstarted
// decode can be cancelled too.
private val activeAnalyses = ConcurrentHashMap<String, AtomicBoolean>()
override fun definition() = ModuleDefinition { override fun definition() = ModuleDefinition {
Name("AstraLibraryScanner") Name("AstraLibraryScanner")
Events("onScanProgress") Events("onScanProgress", "onWaveformProgress")
AsyncFunction("listAudioFiles") Coroutine { treeUri: String, extensions: List<String> -> AsyncFunction("listAudioFiles") Coroutine { treeUri: String, extensions: List<String> ->
withContext(Dispatchers.IO) { listAudioFiles(treeUri, extensions) } withContext(Dispatchers.IO) { listAudioFiles(treeUri, extensions) }
@@ -144,33 +177,42 @@ class AstraLibraryScannerModule : Module() {
} }
} }
// Offline waveform peaks for the seek bar: full PCM decode -> RMS per bin, // ONE whole-file PCM decode producing waveform peaks and (when withLoudness) gated
// normalized to [0,1]. Heavy (whole-file decode), so cap concurrency and // integrated loudness + sample peak. Both need every sample, so they ride the same
// run lazily per track on the JS side; results are cached in SQLite there. // pass — running them separately meant decoding each track twice. Heavy, so cap
AsyncFunction("extractWaveform") Coroutine { uri: String, bins: Int -> // concurrency; the JS side caches results in SQLite and prefetches the queue ahead.
waveformSemaphore.withPermit { // Emits onWaveformProgress as bins finalize so the seek bar can fill in left-to-right.
withContext(Dispatchers.IO) { decodeAndAnalyze(uri, if (bins > 0) bins else 512).peaks } AsyncFunction("analyzeTrack") Coroutine { uri: String, bins: Int, withLoudness: Boolean ->
val flag = AtomicBoolean(false)
activeAnalyses[uri] = flag
try {
waveformSemaphore.withPermit {
// Cancelled while queued behind another decode — don't start at all.
if (flag.get()) AudioAnalysis().apply { cancelled = true }
else withContext(Dispatchers.IO) {
runAnalysis(uri, if (bins > 0) bins else 512, withLoudness, flag)
}
}
} finally {
activeAnalyses.remove(uri, flag)
} }
} }
// Stop an in-flight (or still-queued) analysis for this URI. Safe to call for a URI
// with no analysis running. The decode bails at the next buffer boundary.
AsyncFunction("cancelAnalysis") Coroutine { uri: String ->
activeAnalyses[uri]?.set(true)
}
// Fast waveform preview for first paint: sparse short-window decode across // Fast waveform preview for first paint: sparse short-window decode across
// the file. The JS side shows this immediately but only persists the full // the file. The JS side shows this immediately as the coarse full-width shape
// extractWaveform result. // that the progressive accurate pass then fills over; only analyzeTrack persists.
AsyncFunction("extractWaveformPreview") Coroutine { uri: String, bins: Int -> AsyncFunction("extractWaveformPreview") Coroutine { uri: String, bins: Int ->
waveformSemaphore.withPermit { waveformSemaphore.withPermit {
withContext(Dispatchers.IO) { decodeWaveformPreview(uri, if (bins > 0) bins else 96) } withContext(Dispatchers.IO) { decodeWaveformPreview(uri, if (bins > 0) bins else 96) }
} }
} }
// Fast loudness (M4): decodes only a few short windows spread across the track
// (not the whole file) + gated K-weighting -> integrated LUFS + sample peak.
// Waveform peaks stay lazy/full-decode (extractWaveform), decoupled from this.
AsyncFunction("measureLoudness") Coroutine { uri: String ->
waveformSemaphore.withPermit {
withContext(Dispatchers.IO) { measureLoudness(uri) }
}
}
// ReplayGain tags (M4): reads container metadata only (no PCM decode), so it is // ReplayGain tags (M4): reads container metadata only (no PCM decode), so it is
// cheap and lets us normalize a tagged library without the slow loudness decode. // cheap and lets us normalize a tagged library without the slow loudness decode.
AsyncFunction("readReplayGain") Coroutine { uri: String -> AsyncFunction("readReplayGain") Coroutine { uri: String ->
@@ -644,18 +686,43 @@ class AstraLibraryScannerModule : Module() {
) )
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Waveform peaks (offline RMS bins) // Track analysis: waveform peaks + loudness in ONE decode pass
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// One whole-file PCM decode -> per-bin RMS waveform peaks (normalized [0,1]) for the /** Throttle for onWaveformProgress — ~12 emits/sec is plenty for a fill animation. */
// seek bar. Returns empty peaks on any failure (caller falls back to a flat seek private val progressEmitNanos = 80L * 1_000_000L
// bar). Loudness is measured separately by measureLoudness.
private fun decodeAndAnalyze(uriStr: String, bins: Int): AudioAnalysis { /** Hard ceiling so a corrupt file can't hang a decode forever holding a semaphore permit. */
private val analysisTimeoutMs = 180_000L
/**
* One whole-file PCM decode producing per-bin RMS waveform peaks (normalized to [0,1])
* and, when `withLoudness`, gated integrated LUFS + absolute sample peak. Both analyses
* need every sample, so they share a pass.
*
* Uses MediaCodec in async (callback) mode: the old synchronous dequeue loop burned a
* 10ms timeout every time a buffer wasn't ready, thousands of times per track. All four
* callbacks land on one handler thread, so the extractor and accumulator are touched from
* exactly one thread and need no locking.
*
* Returns empty peaks on any failure and sets `cancelled` if it bailed early — callers
* must not persist a cancelled result.
*/
private suspend fun runAnalysis(
uriStr: String,
bins: Int,
withLoudness: Boolean,
cancelFlag: AtomicBoolean,
): AudioAnalysis {
val context = requireContext() val context = requireContext()
val result = AudioAnalysis() val result = AudioAnalysis()
result.withLoudness = withLoudness
val uri = Uri.parse(uriStr) val uri = Uri.parse(uriStr)
val extractor = MediaExtractor() val extractor = MediaExtractor()
var codec: MediaCodec? = null var codec: MediaCodec? = null
var handlerThread: HandlerThread? = null
val startNanos = System.nanoTime()
try { try {
extractor.setDataSource(context, uri, null) extractor.setDataSource(context, uri, null)
@@ -670,63 +737,305 @@ class AstraLibraryScannerModule : Module() {
val format = trackFormat ?: return result val format = trackFormat ?: return result
extractor.selectTrack(trackIndex) extractor.selectTrack(trackIndex)
val mime = format.getString(MediaFormat.KEY_MIME) ?: return result
result.mime = mime
val sampleRate = val sampleRate =
if (format.containsKey(MediaFormat.KEY_SAMPLE_RATE)) format.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100 if (format.containsKey(MediaFormat.KEY_SAMPLE_RATE)) format.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100
val durationUs = val durationUs =
if (format.containsKey(MediaFormat.KEY_DURATION)) format.getLong(MediaFormat.KEY_DURATION) else 0L if (format.containsKey(MediaFormat.KEY_DURATION)) format.getLong(MediaFormat.KEY_DURATION) else 0L
result.durationMs = durationUs / 1000.0
val totalFrames = max(1L, (durationUs / 1_000_000.0 * sampleRate).toLong()) val totalFrames = max(1L, (durationUs / 1_000_000.0 * sampleRate).toLong())
var channelCount =
val acc = AnalyzeAccumulator(bins, totalFrames, withLoudness)
acc.sampleRate = sampleRate
acc.channelCount =
if (format.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) else 2 if (format.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) else 2
var pcmFloat = false
val sumSquares = DoubleArray(bins) val decoder = createAnalysisDecoder(mime)
val counts = LongArray(bins) codec = decoder
result.decoderName = decoder.name
handlerThread = HandlerThread("astra-analyze").also { it.start() }
val done = CompletableDeferred<Unit>()
var sawInputEOS = false
var lastEmitNanos = 0L
var lastEmitBin = 0
decoder.setCallback(
object : MediaCodec.Callback() {
override fun onInputBufferAvailable(mc: MediaCodec, index: Int) {
if (done.isCompleted) return
try {
if (cancelFlag.get()) {
result.cancelled = true
done.complete(Unit)
return
}
if (sawInputEOS) return
val buf = mc.getInputBuffer(index) ?: return
val size = extractor.readSampleData(buf, 0)
if (size < 0) {
mc.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
sawInputEOS = true
} else {
mc.queueInputBuffer(index, 0, size, extractor.sampleTime, 0)
extractor.advance()
}
} catch (_: Throwable) {
done.complete(Unit)
}
}
override fun onOutputBufferAvailable(
mc: MediaCodec,
index: Int,
info: MediaCodec.BufferInfo,
) {
if (done.isCompleted) return
try {
if (cancelFlag.get()) {
result.cancelled = true
done.complete(Unit)
return
}
if (info.size > 0) {
val out = mc.getOutputBuffer(index)
if (out != null) {
out.position(info.offset)
out.limit(info.offset + info.size)
out.order(ByteOrder.nativeOrder())
acc.accumulate(out)
}
}
val eos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
mc.releaseOutputBuffer(index, false)
// Progressive emit so the seek bar fills left-to-right instead of snapping
// in at the end. Skipped on the EOS buffer — the promise carries the final,
// globally-normalized result a moment later.
val now = System.nanoTime()
if (!eos && acc.filledBins > lastEmitBin && now - lastEmitNanos >= progressEmitNanos) {
lastEmitNanos = now
lastEmitBin = acc.filledBins
emitWaveformProgress(uriStr, acc, lastEmitBin)
}
if (eos) done.complete(Unit)
} catch (_: Throwable) {
done.complete(Unit)
}
}
override fun onOutputFormatChanged(mc: MediaCodec, fmt: MediaFormat) {
if (fmt.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) {
acc.channelCount = fmt.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
}
if (fmt.containsKey(MediaFormat.KEY_SAMPLE_RATE)) {
acc.sampleRate = fmt.getInteger(MediaFormat.KEY_SAMPLE_RATE)
}
if (fmt.containsKey(MediaFormat.KEY_PCM_ENCODING)) {
acc.pcmFloat =
fmt.getInteger(MediaFormat.KEY_PCM_ENCODING) == AudioFormat.ENCODING_PCM_FLOAT
}
}
override fun onError(mc: MediaCodec, e: MediaCodec.CodecException) {
done.complete(Unit)
}
},
Handler(handlerThread.looper),
)
codec = MediaCodec.createDecoderByType(format.getString(MediaFormat.KEY_MIME)!!)
codec.configure(format, null, null, 0) codec.configure(format, null, null, 0)
codec.start() codec.start()
val info = MediaCodec.BufferInfo() if (withTimeoutOrNull(analysisTimeoutMs) { done.await() } == null) {
var sawInputEOS = false // Timed out: peaks are partial, so treat it as a cancellation rather than caching
var sawOutputEOS = false // a truncated waveform.
var frame = 0L result.cancelled = true
}
if (result.cancelled) return result
while (!sawOutputEOS) { result.peaks = acc.finalPeaks()
if (!sawInputEOS) { if (withLoudness) {
val inIndex = codec.dequeueInputBuffer(10_000) result.lufs = acc.loudness
if (inIndex >= 0) { result.peak = acc.samplePeak
val inBuf = codec.getInputBuffer(inIndex)!! }
val size = extractor.readSampleData(inBuf, 0) val decodeMs = (System.nanoTime() - startNanos) / 1_000_000.0
if (size < 0) { result.decodeMs = decodeMs
codec.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM) val dur = result.durationMs
sawInputEOS = true if (dur != null && dur > 0 && decodeMs > 0) result.realtimeFactor = dur / decodeMs
} else { return result
codec.queueInputBuffer(inIndex, 0, size, extractor.sampleTime, 0) } catch (_: Throwable) {
extractor.advance() return result
} finally {
// Order matters: stopping the codec while a callback is mid-flight on the handler
// thread can crash. Quit the looper and wait for the in-flight callback to drain
// first, THEN tear the codec down.
try {
handlerThread?.quitSafely()
handlerThread?.join(1_000)
} catch (_: Throwable) {}
try { codec?.stop() } catch (_: Throwable) {}
try { codec?.release() } catch (_: Throwable) {}
try { extractor.release() } catch (_: Throwable) {}
}
}
// Emits the raw (un-normalized) RMS prefix; JS normalizes against its own max, so the
// bars rescale slightly as louder material arrives rather than needing a global max we
// don't have yet.
private fun emitWaveformProgress(uri: String, acc: AnalyzeAccumulator, filledBins: Int) {
try {
sendEvent(
"onWaveformProgress",
mapOf(
"uri" to uri,
"filledBins" to filledBins,
"totalBins" to acc.bins,
"peaks" to acc.rmsPrefix(filledBins),
),
)
} catch (_: Throwable) {
// Best-effort: the final result still arrives via the promise.
}
}
// Offline analysis wants raw throughput. Hardware audio decoders are tuned for low-power
// realtime playback, not bulk decode, and the instance is a scarce global resource shared
// with the track that is actually playing — so prefer a software decoder and fall back to
// the platform's default pick.
private fun createAnalysisDecoder(mime: String): MediaCodec {
try {
val info = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.firstOrNull { c ->
!c.isEncoder &&
c.supportedTypes.any { it.equals(mime, ignoreCase = true) } &&
isSoftwareDecoder(c)
}
if (info != null) return MediaCodec.createByCodecName(info.name)
} catch (_: Throwable) {
// Fall through to the platform default.
}
return MediaCodec.createDecoderByType(mime)
}
private fun isSoftwareDecoder(info: MediaCodecInfo): Boolean {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) return info.isSoftwareOnly
val name = info.name.lowercase()
return name.startsWith("omx.google.") || name.startsWith("c2.android.")
}
/**
* Per-bin RMS accumulation over the decoded PCM stream, with the K-weighted loudness
* meter folded in when requested.
*
* Two things matter here because this runs ~20M times for a 4-minute stereo track: PCM is
* bulk-copied out of the codec buffer into a reused scratch array rather than read one
* sample at a time, and frames are consumed in runs (every frame landing in the same bin
* is summed in one tight loop) instead of recomputing the bin index per frame with a
* floating-point division.
*/
private class AnalyzeAccumulator(
val bins: Int,
private val totalFrames: Long,
private val withLoudness: Boolean,
) {
private val sumSquares = DoubleArray(bins)
private val counts = LongArray(bins)
var channelCount = 2
var sampleRate = 44100
var pcmFloat = false
/** Index of the highest bin reached — every bin below it is fully accumulated. */
var filledBins = 0
private set
private var frame = 0L
private var meter: LoudnessMeter? = null
private var floatScratch = FloatArray(0)
private var shortScratch = ShortArray(0)
val loudness: Double? get() = meter?.lufs()
val samplePeak: Double? get() = meter?.peak
fun accumulate(out: ByteBuffer) {
val ch = channelCount.coerceAtLeast(1)
// Created lazily: the true output channel count / rate only arrive with the first
// onOutputFormatChanged, which always precedes the first output buffer.
if (withLoudness && meter == null) meter = LoudnessMeter(ch, sampleRate)
if (pcmFloat) {
val fb = out.asFloatBuffer()
val n = fb.remaining()
if (floatScratch.size < n) floatScratch = FloatArray(n)
fb.get(floatScratch, 0, n)
consume(floatScratch, null, n, ch)
} else {
val sb = out.asShortBuffer()
val n = sb.remaining()
if (shortScratch.size < n) shortScratch = ShortArray(n)
sb.get(shortScratch, 0, n)
consume(null, shortScratch, n, ch)
}
}
private fun consume(f: FloatArray?, s: ShortArray?, n: Int, ch: Int) {
val m = meter
var k = 0
while (k < n) {
var bin = ((frame * bins) / totalFrames).toInt()
if (bin < 0) bin = 0 else if (bin >= bins) bin = bins - 1
// First frame belonging to the next bin: ceil((bin + 1) * totalFrames / bins).
val boundary = ((bin + 1).toLong() * totalFrames + bins - 1L) / bins
val framesAvail = (n - k) / ch
if (framesAvail <= 0) break // trailing partial frame; drop it
var run = (boundary - frame).coerceAtLeast(1L)
if (run > framesAvail) run = framesAvail.toLong()
val end = k + run.toInt() * ch
var acc = 0.0
var j = k
if (m == null) {
if (f != null) {
while (j < end) { val v = f[j].toDouble(); acc += v * v; j++ }
} else if (s != null) {
while (j < end) { val v = s[j] / 32768.0; acc += v * v; j++ }
}
} else {
var c = 0
if (f != null) {
while (j < end) {
val v = f[j].toDouble(); acc += v * v; m.process(v, c)
j++; c++; if (c == ch) c = 0
}
} else if (s != null) {
while (j < end) {
val v = s[j] / 32768.0; acc += v * v; m.process(v, c)
j++; c++; if (c == ch) c = 0
} }
} }
} }
val outIndex = codec.dequeueOutputBuffer(info, 10_000) sumSquares[bin] += acc
if (outIndex >= 0) { counts[bin] += run * ch
if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) sawOutputEOS = true frame += run
if (info.size > 0) { k = end
val out = codec.getOutputBuffer(outIndex)!! if (bin > filledBins) filledBins = bin
out.position(info.offset)
out.limit(info.offset + info.size)
out.order(ByteOrder.nativeOrder())
frame = accumulateAnalyze(out, pcmFloat, channelCount, bins, totalFrames, frame, sumSquares, counts)
}
codec.releaseOutputBuffer(outIndex, false)
} else if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
val nf = codec.outputFormat
if (nf.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) channelCount = nf.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
if (nf.containsKey(MediaFormat.KEY_PCM_ENCODING)) {
pcmFloat = nf.getInteger(MediaFormat.KEY_PCM_ENCODING) == AudioFormat.ENCODING_PCM_FLOAT
}
}
} }
}
/** Raw, un-normalized RMS for the first `count` bins (progressive emit). */
fun rmsPrefix(count: Int): FloatArray {
val n = count.coerceIn(0, bins)
val out = FloatArray(n)
for (i in 0 until n) {
if (counts[i] > 0) out[i] = sqrt(sumSquares[i] / counts[i]).toFloat()
}
return out
}
/** Final peaks, normalized against the global max across every bin. */
fun finalPeaks(): FloatArray {
val peaks = FloatArray(bins) val peaks = FloatArray(bins)
var globalMax = 0.0 var globalMax = 0.0
for (i in 0 until bins) { for (i in 0 until bins) {
@@ -739,14 +1048,7 @@ class AstraLibraryScannerModule : Module() {
if (globalMax > 0) { if (globalMax > 0) {
for (i in 0 until bins) peaks[i] = (peaks[i] / globalMax).toFloat() for (i in 0 until bins) peaks[i] = (peaks[i] / globalMax).toFloat()
} }
result.peaks = peaks return peaks
return result
} catch (_: Throwable) {
return result
} finally {
try { codec?.stop() } catch (_: Throwable) {}
try { codec?.release() } catch (_: Throwable) {}
try { extractor.release() } catch (_: Throwable) {}
} }
} }
@@ -758,7 +1060,7 @@ class AstraLibraryScannerModule : Module() {
// Sparse preview waveform: seek to a bounded number of points, decode a very // Sparse preview waveform: seek to a bounded number of points, decode a very
// short audio window at each point, and normalize those RMS samples. This is // short audio window at each point, and normalize those RMS samples. This is
// intentionally approximate; decodeAndAnalyze remains the accurate cache fill. // intentionally approximate; runAnalysis remains the accurate cache fill.
private fun decodeWaveformPreview(uriStr: String, bins: Int): FloatArray { private fun decodeWaveformPreview(uriStr: String, bins: Int): FloatArray {
val context = requireContext() val context = requireContext()
val previewBins = bins.coerceIn(16, 128) val previewBins = bins.coerceIn(16, 128)
@@ -916,172 +1218,6 @@ class AstraLibraryScannerModule : Module() {
return PcmEnergy(sumSquares, sampleCount, frameCount) return PcmEnergy(sumSquares, sampleCount, frameCount)
} }
// Integrated gated loudness over the WHOLE file (accurate — subset sampling caused
// too much loudness inconsistency). Decodes the full track and feeds the gated
// K-weighting meter. Measured on the fly per track (current + queue lookahead) and
// cached, so the cost is paid once per track, never in a bulk background pass.
private fun measureLoudness(uriStr: String): AudioAnalysis {
val context = requireContext()
val result = AudioAnalysis()
val uri = Uri.parse(uriStr)
val extractor = MediaExtractor()
var codec: MediaCodec? = null
try {
extractor.setDataSource(context, uri, null)
var trackFormat: MediaFormat? = null
var trackIndex = -1
for (i in 0 until extractor.trackCount) {
val f = extractor.getTrackFormat(i)
if (f.getString(MediaFormat.KEY_MIME)?.startsWith("audio/") == true) {
trackFormat = f; trackIndex = i; break
}
}
val format = trackFormat ?: return result
extractor.selectTrack(trackIndex)
val sampleRate =
if (format.containsKey(MediaFormat.KEY_SAMPLE_RATE)) format.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100
var channelCount =
if (format.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) else 2
var pcmFloat = false
codec = MediaCodec.createDecoderByType(format.getString(MediaFormat.KEY_MIME)!!)
codec.configure(format, null, null, 0)
codec.start()
val info = MediaCodec.BufferInfo()
var meter: LoudnessMeter? = null
var sawInputEOS = false
var sawOutputEOS = false
while (!sawOutputEOS) {
if (!sawInputEOS) {
val inIndex = codec.dequeueInputBuffer(10_000)
if (inIndex >= 0) {
val inBuf = codec.getInputBuffer(inIndex)!!
val size = extractor.readSampleData(inBuf, 0)
if (size < 0) {
codec.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
sawInputEOS = true
} else {
codec.queueInputBuffer(inIndex, 0, size, extractor.sampleTime, 0)
extractor.advance()
}
}
}
val outIndex = codec.dequeueOutputBuffer(info, 10_000)
if (outIndex >= 0) {
if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) sawOutputEOS = true
if (info.size > 0) {
val out = codec.getOutputBuffer(outIndex)!!
out.position(info.offset)
out.limit(info.offset + info.size)
out.order(ByteOrder.nativeOrder())
val m = meter ?: LoudnessMeter(channelCount, sampleRate).also { meter = it }
feedMeter(out, pcmFloat, channelCount, m)
}
codec.releaseOutputBuffer(outIndex, false)
} else if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
val nf = codec.outputFormat
if (nf.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) channelCount = nf.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
if (nf.containsKey(MediaFormat.KEY_PCM_ENCODING)) {
pcmFloat = nf.getInteger(MediaFormat.KEY_PCM_ENCODING) == AudioFormat.ENCODING_PCM_FLOAT
}
}
}
meter?.let {
result.lufs = it.lufs()
result.peak = it.peak
}
return result
} catch (_: Throwable) {
return result
} finally {
try { codec?.stop() } catch (_: Throwable) {}
try { codec?.release() } catch (_: Throwable) {}
try { extractor.release() } catch (_: Throwable) {}
}
}
// Feeds one decoded PCM buffer (16-bit or float) to the loudness meter.
private fun feedMeter(
out: java.nio.ByteBuffer,
pcmFloat: Boolean,
channelCount: Int,
meter: LoudnessMeter
) {
if (pcmFloat) {
val fb = out.asFloatBuffer()
val n = fb.remaining()
var k = 0
while (k < n) {
var c = 0
while (c < channelCount && k < n) {
meter.process(fb.get(k).toDouble(), c); k++; c++
}
}
} else {
val sb = out.asShortBuffer()
val n = sb.remaining()
var k = 0
while (k < n) {
var c = 0
while (c < channelCount && k < n) {
meter.process(sb.get(k) / 32768.0, c); k++; c++
}
}
}
}
// Folds one decoded PCM buffer into the per-bin RMS accumulators and, when a
// loudness meter is provided, the K-weighted loudness + sample peak. Handles
// 16-bit (default) and float PCM. Returns the updated running frame index.
private fun accumulateAnalyze(
out: java.nio.ByteBuffer,
pcmFloat: Boolean,
channelCount: Int,
bins: Int,
totalFrames: Long,
startFrame: Long,
sumSquares: DoubleArray,
counts: LongArray
): Long {
var frame = startFrame
if (pcmFloat) {
val fb = out.asFloatBuffer()
val n = fb.remaining()
var k = 0
while (k < n) {
val bin = ((frame.toDouble() / totalFrames) * bins).toInt().coerceIn(0, bins - 1)
var c = 0
while (c < channelCount && k < n) {
val s = fb.get(k).toDouble()
sumSquares[bin] += s * s
k++; c++
}
counts[bin] += c.toLong()
frame++
}
} else {
val sb = out.asShortBuffer()
val n = sb.remaining()
var k = 0
while (k < n) {
val bin = ((frame.toDouble() / totalFrames) * bins).toInt().coerceIn(0, bins - 1)
var c = 0
while (c < channelCount && k < n) {
val s = sb.get(k) / 32768.0
sumSquares[bin] += s * s
k++; c++
}
counts[bin] += c.toLong()
frame++
}
}
return frame
}
// Gated integrated K-weighted loudness per ITU-R BS.1770 + absolute sample peak. // Gated integrated K-weighted loudness per ITU-R BS.1770 + absolute sample peak.
// Two cascaded biquads (high-shelf pre-filter + RLB high-pass) per channel with // Two cascaded biquads (high-shelf pre-filter + RLB high-pass) per channel with
// pyloudnorm-reference coefficients (so the -0.691 offset holds), accumulated into // pyloudnorm-reference coefficients (so the -0.691 offset holds), accumulated into
+45 -9
View File
@@ -89,10 +89,44 @@ export interface NativeScanResult {
catalogRevision: string; catalogRevision: string;
} }
/**
* Partial waveform emitted while `analyzeTrack` decodes, so the seek bar can fill in
* left-to-right. `peaks` holds RAW (un-normalized) RMS for bins `[0, filledBins)` — the
* global max isn't known until the decode ends, so callers normalize against the max of
* what they've received so far and accept a slight rescale as louder material arrives.
*/
export interface WaveformProgressEvent {
/** Track URI this partial belongs to — callers must filter, decodes overlap. */
uri: string;
filledBins: number;
totalBins: number;
peaks: number[];
}
type AstraLibraryScannerEvents = { type AstraLibraryScannerEvents = {
onScanProgress: (event: ScanProgressEvent) => void; onScanProgress: (event: ScanProgressEvent) => void;
onWaveformProgress: (event: WaveformProgressEvent) => void;
}; };
/** One decode pass: waveform peaks + (optionally) loudness, plus timing. */
export interface TrackAnalysis {
/** `bins` RMS peaks normalized to [0,1]; empty on failure. */
peaks: number[];
/** Integrated LUFS; null when unmeasured (withLoudness false) or unmeasurable. */
lufs: number | null;
/** Absolute sample peak, linear [0,1]; null when unmeasured. */
peak: number | null;
/** True if the decode bailed early — do NOT persist peaks or loudness. */
cancelled: boolean;
decodeMs: number | null;
durationMs: number | null;
/** durationMs / decodeMs — how many times faster than realtime the decode ran. */
realtimeFactor: number | null;
decoderName: string | null;
mime: string | null;
withLoudness: boolean;
}
declare class AstraLibraryScannerModuleType extends NativeModule<AstraLibraryScannerEvents> { declare class AstraLibraryScannerModuleType extends NativeModule<AstraLibraryScannerEvents> {
listAudioFiles(treeUri: string, extensions: string[]): Promise<ListResult>; listAudioFiles(treeUri: string, extensions: string[]): Promise<ListResult>;
extractMetadata(files: { uri: string; coverUri?: string | null }[]): Promise<ExtractedMetadata[]>; extractMetadata(files: { uri: string; coverUri?: string | null }[]): Promise<ExtractedMetadata[]>;
@@ -102,21 +136,23 @@ declare class AstraLibraryScannerModuleType extends NativeModule<AstraLibrarySca
extensions: string[] extensions: string[]
): Promise<NativeScanResult>; ): Promise<NativeScanResult>;
/** /**
* Decode the file's PCM and return `bins` RMS peaks normalized to [0,1] for * ONE whole-file PCM decode producing `bins` RMS waveform peaks and, when
* the waveform seek bar. Whole-file decode (heavy); returns [] on failure. * `withLoudness`, gated integrated LUFS + sample peak. Both analyses need every
* sample, so they share a pass — ask for loudness here whenever you'd otherwise
* measure it separately. Heavy; concurrency is capped natively at 2 and results
* should be cached. Emits `onWaveformProgress` as bins finalize.
*/ */
extractWaveform(uri: string, bins: number): Promise<number[]>; analyzeTrack(uri: string, bins: number, withLoudness: boolean): Promise<TrackAnalysis>;
/**
* Stop an in-flight (or still-queued) `analyzeTrack` for this URI so a skipped-past
* track stops burning CPU. Safe to call when nothing is running.
*/
cancelAnalysis(uri: string): Promise<void>;
/** /**
* Decode short windows across the file and return approximate RMS peaks for * Decode short windows across the file and return approximate RMS peaks for
* immediate seek-bar paint. Cheap preview only; callers should not persist it. * immediate seek-bar paint. Cheap preview only; callers should not persist it.
*/ */
extractWaveformPreview(uri: string, bins: number): Promise<number[]>; extractWaveformPreview(uri: string, bins: number): Promise<number[]>;
/**
* Fast integrated loudness (M4): decodes only a few short windows across the
* track + gated K-weighting -> integrated LUFS + absolute sample peak. Null on
* failure / unmeasurable audio. Waveform peaks are separate (extractWaveform).
*/
measureLoudness(uri: string): Promise<{ lufs: number | null; peak: number | null }>;
/** /**
* Read ReplayGain track/album gain (dB) + peak (linear) from container tags * Read ReplayGain track/album gain (dB) + peak (linear) from container tags
* (ID3 TXXX / Vorbis comments / MP4 freeform) without decoding audio. All fields * (ID3 TXXX / Vorbis comments / MP4 freeform) without decoding audio. All fields
+1
View File
@@ -69,6 +69,7 @@
"test:dynamic-playlists": "node --experimental-strip-types --experimental-specifier-resolution=node --test src/shared/playlists/dynamicPlaylist.test.mts", "test:dynamic-playlists": "node --experimental-strip-types --experimental-specifier-resolution=node --test src/shared/playlists/dynamicPlaylist.test.mts",
"test:album-grouping": "node --experimental-strip-types --experimental-specifier-resolution=node --test src/shared/library/albumGrouping.test.mts src/shared/library/albumEligibility.test.mts src/library/albumIdentity.test.mts src/library/albumSummary.test.mts", "test:album-grouping": "node --experimental-strip-types --experimental-specifier-resolution=node --test src/shared/library/albumGrouping.test.mts src/shared/library/albumEligibility.test.mts src/library/albumIdentity.test.mts src/library/albumSummary.test.mts",
"test:artist-grouping": "node --experimental-strip-types --test src/library/artistGrouping.test.mts", "test:artist-grouping": "node --experimental-strip-types --test src/library/artistGrouping.test.mts",
"test:waveform-math": "node --experimental-strip-types --test src/scope/waveformMath.test.mts",
"test:desktop-sync": "node --experimental-strip-types --experimental-specifier-resolution=node --test src/library/importMatching.test.mts src/services/desktopSyncPlaylistMerge.test.mts src/services/desktopSyncPolicy.test.mts src/shared/sync/conflictPreview.test.mts", "test:desktop-sync": "node --experimental-strip-types --experimental-specifier-resolution=node --test src/library/importMatching.test.mts src/services/desktopSyncPlaylistMerge.test.mts src/services/desktopSyncPolicy.test.mts src/shared/sync/conflictPreview.test.mts",
"test:eq-share": "node --experimental-strip-types --experimental-specifier-resolution=node --test src/audio/eqShare.test.mts", "test:eq-share": "node --experimental-strip-types --experimental-specifier-resolution=node --test src/audio/eqShare.test.mts",
"test:signal": "node --experimental-strip-types --experimental-specifier-resolution=node --test src/audio/signalShare.test.mts src/audio/signalShareIntent.test.mts src/audio/signalScanGeometry.test.mts src/audio/signalLocalMatch.test.mts", "test:signal": "node --experimental-strip-types --experimental-specifier-resolution=node --test src/audio/signalShare.test.mts src/audio/signalShareIntent.test.mts src/audio/signalScanGeometry.test.mts src/audio/signalLocalMatch.test.mts",
+56
View File
@@ -14,6 +14,7 @@ import { getLyricsCacheCount } from '@/db/lyricsQueries';
import { AstraLibraryData } from '../../../modules/astra-library-scanner'; import { AstraLibraryData } from '../../../modules/astra-library-scanner';
import { clearAllLyricsCache } from '@/lyrics/lyrics'; import { clearAllLyricsCache } from '@/lyrics/lyrics';
import { clearAllWaveformCache } from '@/scope/waveform'; import { clearAllWaveformCache } from '@/scope/waveform';
import { getRecentAnalysisTimings, type AnalysisTiming } from '@/audio/trackAnalysis';
import { useLyricsStore } from '@/stores/lyricsStore'; import { useLyricsStore } from '@/stores/lyricsStore';
import { useLibraryStore } from '@/stores/libraryStore'; import { useLibraryStore } from '@/stores/libraryStore';
import { useOnboardingStore } from '@/stores/onboardingStore'; import { useOnboardingStore } from '@/stores/onboardingStore';
@@ -183,10 +184,59 @@ export default function TroubleshootingSettingsScreen() {
subtitle="Audition semantic feedback, device primitives, and signature candidates." subtitle="Audition semantic feedback, device primitives, and signature candidates."
onPress={() => router.push('/settings/haptics-lab' as never)} onPress={() => router.push('/settings/haptics-lab' as never)}
/> />
<AnalysisTimingPanel />
</SettingsSectionScreen> </SettingsSectionScreen>
); );
} }
/**
* How fast waveform/loudness decodes are actually running, per format and decoder. The
* realtime multiple is the number that decides whether MediaCodec is fast enough or whether
* the analysis path needs its own in-process decoder.
*/
function AnalysisTimingPanel() {
const styles = useStyles();
const colors = useColors();
const [timings, setTimings] = useState<readonly AnalysisTiming[]>([]);
useEffect(() => {
const read = () => setTimings(getRecentAnalysisTimings().slice(0, 6));
read();
const timer = setInterval(read, 2000);
return () => clearInterval(timer);
}, []);
if (timings.length === 0) {
return (
<Text variant="caption" color={colors.textSecondary} style={styles.timingEmpty}>
Decode speed appears here after a track with no cached waveform plays.
</Text>
);
}
return (
<View style={styles.timingPanel}>
{timings.map((timing) => (
<View key={`${timing.path}-${timing.at}`} style={styles.timingRow}>
<Text variant="mono" color={colors.textPrimary}>
{timing.kind === 'preview'
? `preview · ${Math.round(timing.decodeMs)}ms`
: `${(timing.mime ?? 'audio/?').replace('audio/', '')} · ${Math.round(timing.decodeMs)}ms` +
(timing.realtimeFactor ? ` · ${Math.round(timing.realtimeFactor)}× realtime` : '')}
</Text>
<Text variant="caption" color={colors.textSecondary} numberOfLines={1}>
{timing.kind === 'preview'
? 'sparse first-paint pass'
: `${timing.decoderName ?? 'unknown decoder'}${
timing.withLoudness ? ' · loudness folded in' : ''
}`}
</Text>
</View>
))}
</View>
);
}
function MaintenanceRow({ function MaintenanceRow({
icon, icon,
title, title,
@@ -247,4 +297,10 @@ const useStyles = createThemedStyles((colors) => ({
}, },
errorFeedback: { borderColor: colors.warning }, errorFeedback: { borderColor: colors.warning },
feedbackText: { flex: 1 }, feedbackText: { flex: 1 },
timingPanel: {
gap: spacing.sm, padding: spacing.md, borderRadius: radius.sm,
borderWidth: 1, borderColor: colors.glassBorder, backgroundColor: colors.glassBg,
},
timingRow: { gap: 1 },
timingEmpty: { paddingHorizontal: spacing.md, lineHeight: 16 },
})); }));
+237 -45
View File
@@ -1,18 +1,28 @@
// Per-track normalization facts: ReplayGain tags (cheap, container-only) + measured // Per-track analysis facts: waveform peaks + ReplayGain tags + measured integrated LUFS /
// integrated LUFS / sample peak (a decode, only when ReplayGain can't cover the track). // sample peak.
// //
// ensureTrackLoudness is the single deduped entry point used by the normalization sync // Peaks and loudness come from ONE native decode pass (analyzeTrack). Both need every
// (current track + queue prefetch). It reads ReplayGain tags once per track, and only // sample, so running them as separate whole-file decodes meant decoding each track twice —
// falls back to the expensive loudness decode when ReplayGain is off or absent — so a // and the two decodes then competed for the same native concurrency permits, which is why
// fully tagged library normalizes with no decoding at all. // the waveform used to arrive so late. Peaks fall out of the pass regardless, so we persist
// them even when loudness was the only reason we decoded.
//
// ensureTrackAnalysis is the single deduped entry point. It reads what's already cached,
// decodes only what's missing, and stores both halves. ReplayGain tags are read first
// (container-only, no decode), so a fully tagged library still normalizes without decoding.
import { import {
AstraLibraryData, AstraLibraryData,
AstraLibraryScanner, AstraLibraryScanner,
type NativeTrackLoudness, type NativeTrackLoudness,
type TrackAnalysis,
} from '../../modules/astra-library-scanner'; } from '../../modules/astra-library-scanner';
import { hasUsableReplayGain, type LoudnessFacts } from '@/audio/normalization'; import { hasUsableReplayGain, type LoudnessFacts } from '@/audio/normalization';
import { useAudioSettingsStore } from '@/stores/audioSettingsStore'; import { useAudioSettingsStore } from '@/stores/audioSettingsStore';
import { CacheInvalidationGate } from '@/lib/cacheInvalidation';
/** Stored waveform resolution. Downsampled to the bar count at render time. */
export const WAVEFORM_BINS = 512;
/** Map a loudness DB row (or a miss) to the resolver's facts shape. */ /** Map a loudness DB row (or a miss) to the resolver's facts shape. */
export function factsFromRow(row: NativeTrackLoudness | null): LoudnessFacts { export function factsFromRow(row: NativeTrackLoudness | null): LoudnessFacts {
@@ -26,46 +36,80 @@ export function factsFromRow(row: NativeTrackLoudness | null): LoudnessFacts {
}; };
} }
/** export interface TrackAnalysisResult {
* Measure + store integrated loudness + sample peak for one track (always /** Normalized [0,1] peaks, or null when unavailable / not requested and uncached. */
* re-measures). The decode is the expensive part; failures leave loudness NULL. peaks: Float32Array | null;
*/ facts: LoudnessFacts;
export async function measureAndStoreLoudness(
path: string
): Promise<{ lufs: number | null; peak: number | null }> {
try {
const res = await AstraLibraryScanner.measureLoudness(path);
const lufs = res?.lufs ?? null;
const peak = res?.peak ?? null;
await AstraLibraryData.setTrackLoudness(path, lufs, peak).catch(() => {});
return { lufs, peak };
} catch {
return { lufs: null, peak: null };
}
} }
const inflight = new Map<string, Promise<LoudnessFacts>>(); export interface EnsureAnalysisOptions {
/**
* Decode for waveform peaks when they're missing. Pass false from headless paths
* (Android Auto / Bluetooth with no UI) that only need loudness — peaks are still
* persisted if a loudness decode happens to run, since they come out free.
*/
peaks?: boolean;
}
interface InflightRun {
promise: Promise<TrackAnalysisResult>;
wantPeaks: boolean;
}
const inflight = new Map<string, InflightRun>();
// Paths cancelled while their run was still in its DB-read phase. The native cancel flag
// only exists once analyzeTrack has been called, so without this a cancel landing in that
// window would be silently lost and the decode would run to completion anyway.
const cancelledPaths = new Set<string>();
const cacheGate = new CacheInvalidationGate();
/** /**
* Loudness facts for a track, reading ReplayGain tags and decoding only as needed * Analysis facts for a track, decoding at most once and only for what's actually missing
* (deduped by path). Cheap when already analyzed (single DB read). The normalization * (deduped by path). Cheap when already analyzed — two DB reads and no decode.
* sync uses this so tracks from a pre-M4 library still normalize before a full rescan.
*/ */
export function ensureTrackLoudness(path: string): Promise<LoudnessFacts> { export function ensureTrackAnalysis(
path: string,
options: EnsureAnalysisOptions = {}
): Promise<TrackAnalysisResult> {
const wantPeaks = options.peaks !== false;
const existing = inflight.get(path); const existing = inflight.get(path);
if (existing) return existing; // A run that already covers what we need — join it.
const task = run(path).finally(() => inflight.delete(path)); if (existing && (existing.wantPeaks || !wantPeaks)) return existing.promise;
inflight.set(path, task); // A loudness-only run is going and we need peaks: let it finish (so we don't decode the
return task; // same file twice concurrently), then fill in the peaks.
if (existing) return existing.promise.then(() => start(path, wantPeaks));
return start(path, wantPeaks);
} }
async function run(path: string): Promise<LoudnessFacts> { function start(path: string, wantPeaks: boolean): Promise<TrackAnalysisResult> {
const row = (await AstraLibraryData.getTrackLoudness([path]))[0] ?? null; const existing = inflight.get(path);
let facts = factsFromRow(row); if (existing?.wantPeaks) return existing.promise;
const promise = run(path, wantPeaks).finally(() => {
if (inflight.get(path)?.promise === promise) {
inflight.delete(path);
cancelledPaths.delete(path);
}
});
inflight.set(path, { promise, wantPeaks });
return promise;
}
// 1. Read ReplayGain tags once per track (container-only, no decode). Decoupled async function run(path: string, wantPeaks: boolean): Promise<TrackAnalysisResult> {
// from loudness so a track measured before ReplayGain was enabled still picks const generation = cacheGate.capture();
// up its tags; rg_scanned stays unset on failure so it retries next touch.
const [row, cachedPeaks] = await Promise.all([
AstraLibraryData.getTrackLoudness([path])
.then((rows) => rows[0] ?? null)
.catch(() => null),
AstraLibraryData.getWaveform(path).catch(() => null),
]);
let facts = factsFromRow(row);
let peaks = cachedPeaks && cachedPeaks.length > 0 ? Float32Array.from(cachedPeaks) : null;
// ReplayGain tags: container-only, no decode. Decoupled from loudness so a track measured
// before ReplayGain was enabled still picks up its tags; rg_scanned stays unset on failure
// so it retries next touch.
if (!row || row.rg_scanned !== 1) { if (!row || row.rg_scanned !== 1) {
try { try {
const rg = await AstraLibraryScanner.readReplayGain(path); const rg = await AstraLibraryScanner.readReplayGain(path);
@@ -88,14 +132,162 @@ async function run(path: string): Promise<LoudnessFacts> {
} }
} }
// 2. Loudness already measured — nothing more to do. // Loudness only needs measuring when it's unknown AND ReplayGain can't cover the track.
if (facts.loudnessLufs != null) return facts;
// 3. ReplayGain alone can normalize this track — skip the expensive decode.
const settings = useAudioSettingsStore.getState().asNormalizationSettings(); const settings = useAudioSettingsStore.getState().asNormalizationSettings();
if (hasUsableReplayGain(facts, settings)) return facts; const needLoudness = facts.loudnessLufs == null && !hasUsableReplayGain(facts, settings);
const needPeaks = wantPeaks && !peaks;
if (!needLoudness && !needPeaks) return { peaks, facts };
// Skipped past while we were reading the DB — don't start the decode at all.
if (cancelledPaths.has(path)) return { peaks, facts };
// 4. Otherwise measure loudness now (decode) and merge it in. let analysis: TrackAnalysis;
const measured = await measureAndStoreLoudness(path); try {
return { ...facts, loudnessLufs: measured.lufs, samplePeak: measured.peak }; analysis = await AstraLibraryScanner.analyzeTrack(path, WAVEFORM_BINS, needLoudness);
} catch {
return { peaks, facts };
}
recordTiming(path, analysis);
// Skipped past / timed out: peaks are truncated and loudness is partial. Cache neither.
if (analysis.cancelled) return { peaks, facts };
if (analysis.peaks && analysis.peaks.length > 0) {
peaks = Float32Array.from(analysis.peaks);
await persistPeaks(path, peaks, generation);
}
if (needLoudness) {
await AstraLibraryData.setTrackLoudness(path, analysis.lufs, analysis.peak).catch(() => {});
facts = { ...facts, loudnessLufs: analysis.lufs, samplePeak: analysis.peak };
}
return { peaks, facts };
}
async function persistPeaks(
path: string,
peaks: Float32Array,
generation: number
): Promise<void> {
await cacheGate
.enqueue(async () => {
if (!cacheGate.isCurrent(generation)) return;
await AstraLibraryData.putWaveform(path, Array.from(peaks));
})
.catch(() => {
/* cache write failure is non-fatal */
});
}
/**
* Loudness facts only — the normalization path's entry point. Does not decode purely to
* fill in a missing waveform, but keeps the peaks if a loudness decode produces them.
*/
export async function ensureTrackLoudness(path: string): Promise<LoudnessFacts> {
const { facts } = await ensureTrackAnalysis(path, { peaks: false });
return facts;
}
/**
* Stop an in-flight analysis for a track we've skipped past, so it stops burning CPU and
* frees a native decode permit for the track the user is actually on.
*/
export function cancelTrackAnalysis(path: string): void {
if (!inflight.has(path)) return;
cancelledPaths.add(path);
void AstraLibraryScanner.cancelAnalysis(path).catch(() => {});
}
/** Paths with an analysis currently running or queued. */
export function activeAnalysisPaths(): string[] {
return Array.from(inflight.keys());
}
/**
* Whether a decode for this path is already under way — i.e. progress events are about to
* start arriving, so a second "fast preview" decode would only land late and cause a
* visible rescale rather than buying a faster first paint.
*/
export function isAnalysisRunning(path: string): boolean {
return inflight.has(path);
}
/** Drops cached waveform rows and stops in-flight decodes from writing them back. */
export async function clearWaveformCache(): Promise<void> {
inflight.clear();
cancelledPaths.clear();
await cacheGate.invalidate(async () => {
await AstraLibraryData.clearWaveforms();
});
}
// ---------------------------------------------------------------------------
// Timing instrumentation
// ---------------------------------------------------------------------------
export interface AnalysisTiming {
path: string;
/** 'preview' entries are the sparse first-paint decode, 'analysis' the real pass. */
kind: 'analysis' | 'preview';
decodeMs: number;
durationMs: number | null;
/** durationMs / decodeMs — how many times faster than realtime the decode ran. */
realtimeFactor: number | null;
decoderName: string | null;
mime: string | null;
withLoudness: boolean;
at: number;
}
const MAX_TIMINGS = 20;
const recentTimings: AnalysisTiming[] = [];
function push(timing: AnalysisTiming): void {
recentTimings.unshift(timing);
if (recentTimings.length > MAX_TIMINGS) recentTimings.length = MAX_TIMINGS;
}
/**
* Record how long the sparse preview decode took, measured end to end from JS (so it
* includes the wait for a native permit — which is the number that decides whether the
* preview can still beat the real decode's first progress event to the screen).
*/
export function recordPreviewTiming(path: string, elapsedMs: number): void {
push({
path,
kind: 'preview',
decodeMs: elapsedMs,
durationMs: null,
realtimeFactor: null,
decoderName: null,
mime: null,
withLoudness: false,
at: Date.now(),
});
if (__DEV__) console.log(`[analysis] preview ${elapsedMs.toFixed(0)}ms`);
}
function recordTiming(path: string, analysis: TrackAnalysis): void {
if (analysis.cancelled || analysis.decodeMs == null) return;
push({
path,
kind: 'analysis',
decodeMs: analysis.decodeMs,
durationMs: analysis.durationMs,
realtimeFactor: analysis.realtimeFactor,
decoderName: analysis.decoderName,
mime: analysis.mime,
withLoudness: analysis.withLoudness,
at: Date.now(),
});
if (__DEV__) {
const rt = analysis.realtimeFactor;
console.log(
`[analysis] ${analysis.mime ?? '?'} ${analysis.decodeMs.toFixed(0)}ms` +
`${rt ? ` (${rt.toFixed(0)}x realtime)` : ''}` +
` via ${analysis.decoderName ?? '?'}${analysis.withLoudness ? ' +loudness' : ''}`
);
}
}
/** Most recent decodes, newest first — surfaced in Settings → Troubleshooting. */
export function getRecentAnalysisTimings(): readonly AnalysisTiming[] {
return recentTimings;
} }
+36 -15
View File
@@ -13,7 +13,11 @@ import { usePlayerStore } from '@/stores/playerStore';
import { useQueueStore } from '@/stores/queueStore'; import { useQueueStore } from '@/stores/queueStore';
import { useAudioSettingsStore } from '@/stores/audioSettingsStore'; import { useAudioSettingsStore } from '@/stores/audioSettingsStore';
import { resolveNormalizationGain, type LoudnessFacts } from '@/audio/normalization'; import { resolveNormalizationGain, type LoudnessFacts } from '@/audio/normalization';
import { ensureTrackLoudness } from '@/audio/trackAnalysis'; import {
activeAnalysisPaths,
cancelTrackAnalysis,
ensureTrackAnalysis,
} from '@/audio/trackAnalysis';
import { import {
setNormalizationGainNative, setNormalizationGainNative,
setTrackGainNative, setTrackGainNative,
@@ -61,13 +65,13 @@ export function useNormalizationSync(): void {
return; return;
} }
// ensureTrackLoudness is cheap when already analyzed (single DB read) and // ensureTrackAnalysis is cheap when already analyzed (two DB reads) and decodes on a
// decodes+stores on a miss (lazy backfill for pre-scan tracks). During the // miss — one pass covering both loudness and the seek bar's waveform. During the
// await the track is already playing at the conservative fallback gain // await the track is already playing at the conservative fallback gain
// (gainRegistry) — never at unity/full volume. // (gainRegistry) — never at unity/full volume.
let facts = EMPTY_FACTS; let facts = EMPTY_FACTS;
try { try {
facts = await ensureTrackLoudness(path); ({ facts } = await ensureTrackAnalysis(path));
if (cancelled) return; if (cancelled) return;
// Track changed during the await — let the newer recompute win. // Track changed during the await — let the newer recompute win.
if (usePlayerStore.getState().currentTrack?.path !== path) return; if (usePlayerStore.getState().currentTrack?.path !== path) return;
@@ -91,27 +95,38 @@ export function useNormalizationSync(): void {
useScopeStore.getState().setOscGain(computeOscilloscopeGain(basePeak, resolved.linearGain)); useScopeStore.getState().setOscGain(computeOscilloscopeGain(basePeak, resolved.linearGain));
} }
// MEASURE the next several upcoming tracks' loudness while the current one plays // ANALYZE the next several upcoming tracks while the current one plays (decode-ahead
// (decode-ahead for tracks with no facts yet), and register each late-arriving // for tracks with no facts yet), and register each late-arriving gain natively by URL —
// result natively by URL — so when the player advances, the gain is in the map // so when the player advances, the gain is in the map and applies at the transition with
// and applies at the transition with no JS in the loop. Already-analyzed tracks // no JS in the loop. Already-analyzed tracks are bulk-registered by gainRegistry;
// are bulk-registered by gainRegistry; re-registering them here is a harmless // re-registering them here is a harmless cheap DB hit with the same value. Looking a few
// cheap DB hit with the same value. Looking a few ahead (not just the immediate // ahead (not just the immediate next) means a song added several positions back is still
// next) means a song added several positions back is still measured with plenty // analyzed with plenty of lead time. Derived from the queue mirror, so it re-runs on
// of lead time. Derived from the queue mirror, so it re-runs on reorder / // reorder / add-next / advance. Deduped + DB-cached + native-semaphore-capped.
// add-next / advance. Deduped + DB-cached + native-semaphore-capped. //
// The same pass fills the seek bar's waveform, which is why the waveform is usually
// already cached by the time you open now-playing: it used to only start decoding when
// WaveformSeekBar mounted, from cold, queued behind these very decodes.
function prefetchUpcoming(): void { function prefetchUpcoming(): void {
const { tracks, activeIndex } = useQueueStore.getState(); const { tracks, activeIndex } = useQueueStore.getState();
if (activeIndex < 0) return; if (activeIndex < 0) return;
const settings = useAudioSettingsStore.getState().asNormalizationSettings(); const settings = useAudioSettingsStore.getState().asNormalizationSettings();
// The set of tracks worth spending a decode on right now. Everything else that is
// still decoding has been skipped past.
const wanted = new Set<string>();
const currentPath = usePlayerStore.getState().currentTrack?.path;
if (currentPath) wanted.add(currentPath);
for (let i = 1; i <= PREFETCH_AHEAD; i++) { for (let i = 1; i <= PREFETCH_AHEAD; i++) {
const queued = tracks[activeIndex + i]; const queued = tracks[activeIndex + i];
const url = queued?.url; const url = queued?.url;
if (typeof url !== 'string' || url.length === 0) continue; if (typeof url !== 'string' || url.length === 0) continue;
// Remote tracks: unity gain, and decoding the stream URL would download it. // Remote tracks: unity gain, and decoding the stream URL would download it.
if (queued?.sourceType && queued.sourceType !== 'local') continue; if (queued?.sourceType && queued.sourceType !== 'local') continue;
void ensureTrackLoudness(url) wanted.add(url);
.then((facts) => { void ensureTrackAnalysis(url)
.then(({ facts }) => {
if (cancelled) return; if (cancelled) return;
const resolved = resolveNormalizationGain(facts, settings); const resolved = resolveNormalizationGain(facts, settings);
setTrackGainNative(url, resolved.linearGain); setTrackGainNative(url, resolved.linearGain);
@@ -120,6 +135,12 @@ export function useNormalizationSync(): void {
/* leave unregistered — defaults to unity at the transition */ /* leave unregistered — defaults to unity at the transition */
}); });
} }
// Free the native decode permits: a decode for a track the user has skipped past is
// pure waste, and it would otherwise block the track they're actually on.
for (const path of activeAnalysisPaths()) {
if (!wanted.has(path)) cancelTrackAnalysis(path);
}
} }
// The queue can change rapidly (drag-reorder); coalesce re-warms. // The queue can change rapidly (drag-reorder); coalesce re-warms.
+45 -5
View File
@@ -21,7 +21,12 @@ import { Text } from './Text';
import { spacing } from '@/theme'; import { spacing } from '@/theme';
import { createThemedStyles, useColors } from '@/theme/themed'; import { createThemedStyles, useColors } from '@/theme/themed';
import { formatDuration } from '@/lib/format'; import { formatDuration } from '@/lib/format';
import { downsampleWaveform, getWaveform } from '@/scope/waveform'; import {
downsampleWaveform,
getWaveform,
mergeProgressiveWaveform,
subscribeWaveformProgress,
} from '@/scope/waveform';
import { useSmoothPlaybackTime } from '@/audio/useSmoothPlaybackTime'; import { useSmoothPlaybackTime } from '@/audio/useSmoothPlaybackTime';
import { usePlayerStore } from '@/stores/playerStore'; import { usePlayerStore } from '@/stores/playerStore';
import { playHaptic } from '@/lib/haptics'; import { playHaptic } from '@/lib/haptics';
@@ -36,7 +41,8 @@ const BAR_WIDTH = 3;
const BAR_GAP = 2; const BAR_GAP = 2;
const MIN_BAR = 0.05; // floor so silent/idle sections still show a sliver const MIN_BAR = 0.05; // floor so silent/idle sections still show a sliver
const PLAYHEAD_WIDTH = 2; const PLAYHEAD_WIDTH = 2;
type WaveformQuality = 'preview' | 'accurate'; /** Ascending confidence — a lower quality never overwrites a higher one for the same track. */
type WaveformQuality = 'preview' | 'partial' | 'accurate';
interface WaveformSeekBarProps { interface WaveformSeekBarProps {
onSeek: (seconds: number) => void; onSeek: (seconds: number) => void;
@@ -90,16 +96,47 @@ export function WaveformSeekBar({
const grantRef = useRef({ fraction: 0, pageX: 0 }); const grantRef = useRef({ fraction: 0, pageX: 0 });
const detentRef = useRef<ScrubDetentState | null>(null); const detentRef = useRef<ScrubDetentState | null>(null);
const smoothTime = useSmoothPlaybackTime(currentTime, duration, isPlaying); const smoothTime = useSmoothPlaybackTime(currentTime, duration, isPlaying);
// The coarse preview is kept aside as well as rendered: it's the amplitude reference the
// partially-decoded prefix is scaled against, and it supplies the not-yet-decoded tail.
const previewRef = useRef<{ path: string; peaks: Float32Array } | null>(null);
// Whether progressive fill has already begun for the current track. A preview that shows
// up after that point is worse than useless: adopting it mid-fill rescales every bar at
// once (the prefix is scaled against the preview's amplitude), which reads as two
// different waveforms fighting. Once we're filling, the preview is dropped.
const progressStartedRef = useRef(false);
// Load (cache-first) the offline peaks whenever the track changes. // Load (cache-first) the offline peaks whenever the track changes, and follow the decode
// as it runs so the bars resolve left-to-right rather than snapping in at the end. The
// progress subscription is independent of who started the decode — usually the queue
// prefetch got there first, in which case this only ever sees the cache hit.
useEffect(() => { useEffect(() => {
if (!trackPath) return; if (!trackPath) return;
let cancelled = false; let cancelled = false;
previewRef.current = null;
progressStartedRef.current = false;
const unsubscribe = subscribeWaveformProgress(trackPath, ({ peaks, totalBins }) => {
if (cancelled) return;
progressStartedRef.current = true;
const preview = previewRef.current?.path === trackPath ? previewRef.current.peaks : null;
const merged = mergeProgressiveWaveform(peaks, totalBins, preview);
setLoaded((current) => {
if (current?.path === trackPath && current.quality === 'accurate' && current.peaks) {
return current;
}
return { path: trackPath, peaks: merged, quality: 'partial' };
});
});
void getWaveform(trackPath, { void getWaveform(trackPath, {
onPreview: (peaks) => { onPreview: (peaks) => {
if (cancelled) return; if (cancelled) return;
// Lost the race — the real decode is already painting. Adopting the preview now
// would rescale the whole bar in one frame.
if (progressStartedRef.current) return;
previewRef.current = { path: trackPath, peaks };
setLoaded((current) => { setLoaded((current) => {
if (current?.path === trackPath && current.quality === 'accurate' && current.peaks) { if (current?.path === trackPath && current.quality !== 'preview' && current.peaks) {
return current; return current;
} }
return { path: trackPath, peaks, quality: 'preview' }; return { path: trackPath, peaks, quality: 'preview' };
@@ -108,12 +145,15 @@ export function WaveformSeekBar({
}).then((peaks) => { }).then((peaks) => {
if (cancelled) return; if (cancelled) return;
setLoaded((current) => { setLoaded((current) => {
if (!peaks && current?.path === trackPath && current.quality === 'preview') return current; // A failed decode must not wipe a good preview or partial fill.
if (!peaks && current?.path === trackPath && current.peaks) return current;
return { path: trackPath, peaks, quality: 'accurate' }; return { path: trackPath, peaks, quality: 'accurate' };
}); });
}); });
return () => { return () => {
cancelled = true; cancelled = true;
unsubscribe();
}; };
}, [trackPath]); }, [trackPath]);
+82 -79
View File
@@ -1,22 +1,28 @@
// Waveform peaks for the seek bar: cache-first, preview-on-miss, accurate // Waveform peaks for the seek bar: cache-first, preview-on-miss, then the accurate
// decode-on-miss, store. The heavy native decode (extractWaveform) still runs // decode. The accurate pass lives in trackAnalysis (it shares one decode with loudness)
// once per track and persists; extractWaveformPreview gives uncached local // and streams partial results back through onWaveformProgress, so the bar fills in
// tracks a fast first paint. // left-to-right instead of snapping in when the whole file is done.
import { AstraLibraryData, AstraLibraryScanner } from '../../modules/astra-library-scanner'; import { AstraLibraryData, AstraLibraryScanner } from '../../modules/astra-library-scanner';
import { CacheInvalidationGate } from '@/lib/cacheInvalidation'; import {
WAVEFORM_BINS,
clearWaveformCache,
ensureTrackAnalysis,
isAnalysisRunning,
recordPreviewTiming,
} from '@/audio/trackAnalysis';
export const WAVEFORM_BINS = 512; export { WAVEFORM_BINS };
export { downsampleWaveform, mergeProgressiveWaveform } from '@/scope/waveformMath';
export const WAVEFORM_PREVIEW_BINS = 96; export const WAVEFORM_PREVIEW_BINS = 96;
export interface WaveformLoadOptions { export interface WaveformLoadOptions {
onPreview?: (peaks: Float32Array) => void; onPreview?: (peaks: Float32Array) => void;
} }
// Dedupe concurrent requests for the same track (e.g. mini-player + now-playing). // Dedupe concurrent preview requests for the same track (e.g. mini-player + now-playing).
const inflight = new Map<string, Promise<Float32Array | null>>(); // The accurate decode is deduped inside trackAnalysis.
const previewInflight = new Map<string, Promise<Float32Array | null>>(); const previewInflight = new Map<string, Promise<Float32Array | null>>();
const cacheGate = new CacheInvalidationGate();
export function getWaveform( export function getWaveform(
trackPath: string, trackPath: string,
@@ -30,52 +36,88 @@ async function loadWaveform(
trackPath: string, trackPath: string,
options: WaveformLoadOptions options: WaveformLoadOptions
): Promise<Float32Array | null> { ): Promise<Float32Array | null> {
const cached = await AstraLibraryData.getWaveform(trackPath); const cached = await AstraLibraryData.getWaveform(trackPath).catch(() => null);
if (cached && cached.length > 0) return Float32Array.from(cached); if (cached && cached.length > 0) return Float32Array.from(cached);
if (options.onPreview) { // The preview is a SECOND native decode competing for the same two permits as the real
// pass. It only earns that cost when it can beat the real decode's first progress event
// to the screen. If a decode for this track is already running — the common case, since
// the queue prefetch starts one several tracks ahead — progress events are about to
// arrive immediately, and the preview would land late enough only to cause a visible
// rescale. Skip it entirely there.
if (options.onPreview && !isAnalysisRunning(trackPath)) {
const startedAt = Date.now();
void getWaveformPreview(trackPath).then((preview) => { void getWaveformPreview(trackPath).then((preview) => {
recordPreviewTiming(trackPath, Date.now() - startedAt);
if (preview && preview.length > 0) options.onPreview?.(preview); if (preview && preview.length > 0) options.onPreview?.(preview);
}); });
} }
const existing = inflight.get(trackPath); // Shares one decode pass with loudness, and may already be running from the queue
if (existing) return existing; // prefetch — in which case this just joins it. Failures fall back to flat bars.
const generation = cacheGate.capture();
const task = decodeAccurateWaveform(trackPath, generation).finally(() => {
if (inflight.get(trackPath) === task) inflight.delete(trackPath);
});
inflight.set(trackPath, task);
return task;
}
async function decodeAccurateWaveform(trackPath: string, generation: number): Promise<Float32Array | null> {
let raw: number[];
try { try {
raw = await AstraLibraryScanner.extractWaveform(trackPath, WAVEFORM_BINS); const { peaks } = await ensureTrackAnalysis(trackPath);
return peaks;
} catch { } catch {
return null; return null;
} }
if (!raw || raw.length === 0) return null;
const peaks = Float32Array.from(raw);
await cacheGate.enqueue(async () => {
if (!cacheGate.isCurrent(generation)) return;
if (!cacheGate.isCurrent(generation)) return;
await AstraLibraryData.putWaveform(trackPath, Array.from(peaks));
}).catch(() => {
/* cache write failure is non-fatal */
});
return peaks;
} }
/** Deletes waveform rows and prevents decodes already in flight from writing them back. */ /** Deletes waveform rows and prevents decodes already in flight from writing them back. */
export async function clearAllWaveformCache(): Promise<void> { export async function clearAllWaveformCache(): Promise<void> {
inflight.clear();
previewInflight.clear(); previewInflight.clear();
await cacheGate.invalidate(async () => { await clearWaveformCache();
await AstraLibraryData.clearWaveforms(); }
});
// ---------------------------------------------------------------------------
// Progressive decode updates
// ---------------------------------------------------------------------------
export type WaveformProgressListener = (partial: {
/** Raw (un-normalized) RMS for the bins decoded so far. */
peaks: Float32Array;
filledBins: number;
totalBins: number;
}) => void;
const progressListeners = new Map<string, Set<WaveformProgressListener>>();
let nativeProgressSub: { remove(): void } | null = null;
/**
* Listen for partial waveforms while a track decodes. Independent of who started the
* decode, so the seek bar still fills progressively when the queue prefetch kicked it off.
* Returns an unsubscribe function.
*/
export function subscribeWaveformProgress(
trackPath: string,
listener: WaveformProgressListener
): () => void {
if (!nativeProgressSub) {
nativeProgressSub = AstraLibraryScanner.addListener('onWaveformProgress', (event) => {
const listeners = progressListeners.get(event.uri);
if (!listeners || listeners.size === 0) return;
const partial = {
peaks: Float32Array.from(event.peaks),
filledBins: event.filledBins,
totalBins: event.totalBins,
};
for (const cb of listeners) cb(partial);
});
}
let listeners = progressListeners.get(trackPath);
if (!listeners) {
listeners = new Set();
progressListeners.set(trackPath, listeners);
}
listeners.add(listener);
return () => {
const current = progressListeners.get(trackPath);
if (!current) return;
current.delete(listener);
if (current.size === 0) progressListeners.delete(trackPath);
};
} }
function getWaveformPreview(trackPath: string): Promise<Float32Array | null> { function getWaveformPreview(trackPath: string): Promise<Float32Array | null> {
@@ -99,45 +141,6 @@ async function decodePreviewWaveform(trackPath: string): Promise<Float32Array |
return Float32Array.from(raw); return Float32Array.from(raw);
} }
function isLocalWaveformPath(trackPath: string): boolean { export function isLocalWaveformPath(trackPath: string): boolean {
return trackPath.startsWith('content://') || trackPath.startsWith('file://'); return trackPath.startsWith('content://') || trackPath.startsWith('file://');
} }
/**
* Downsample high-res peaks to `barCount` bars with a power curve and two
* smoothing passes. Ported verbatim from desktop waveformExtractor.ts so the
* mobile seek bar matches the desktop look.
*/
export function downsampleWaveform(source: Float32Array, barCount: number): Float32Array {
if (source.length === 0 || barCount <= 0) return new Float32Array(0);
const binsPerBar = source.length / barCount;
const peaks = new Float32Array(barCount);
for (let i = 0; i < barCount; i++) {
const start = Math.floor(i * binsPerBar);
const end = Math.max(start + 1, Math.floor((i + 1) * binsPerBar));
let sum = 0;
for (let j = start; j < end; j++) sum += source[j];
peaks[i] = sum / (end - start);
}
let max = 0;
for (let i = 0; i < barCount; i++) if (peaks[i] > max) max = peaks[i];
if (max > 0) for (let i = 0; i < barCount; i++) peaks[i] /= max;
// Power curve — exaggerate dynamic range.
for (let i = 0; i < barCount; i++) peaks[i] = peaks[i] ** 2;
// Two smoothing passes.
let current = peaks;
for (let p = 0; p < 2; p++) {
const smoothed = new Float32Array(current.length);
smoothed[0] = current[0];
smoothed[current.length - 1] = current[current.length - 1];
for (let i = 1; i < current.length - 1; i++) {
smoothed[i] = current[i - 1] * 0.25 + current[i] * 0.5 + current[i + 1] * 0.25;
}
current = smoothed;
}
return current;
}
+87
View File
@@ -0,0 +1,87 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { downsampleWaveform, mergeProgressiveWaveform } from './waveformMath.ts';
const TOTAL = 512;
/** Preview normalized to [0,1] across the whole track, as the native preview returns it. */
function makePreview(values: number[]): Float32Array {
return Float32Array.from(values);
}
test('merge with no prefix is just the stretched preview', () => {
const preview = makePreview([0.2, 0.8, 0.4, 1]);
const merged = mergeProgressiveWaveform(new Float32Array(0), TOTAL, preview);
assert.equal(merged.length, TOTAL);
assert.ok(Math.abs(merged[0] - 0.2) < 1e-6);
assert.ok(Math.abs(merged[TOTAL - 1] - 1) < 1e-6);
// Each preview value should occupy an equal quarter of the width.
assert.ok(Math.abs(merged[Math.floor(TOTAL * 0.3)] - 0.8) < 1e-6);
});
test('merge with no preview normalizes the prefix against its own max', () => {
const prefix = Float32Array.from([0.01, 0.02, 0.04]); // raw RMS, tiny absolute values
const merged = mergeProgressiveWaveform(prefix, TOTAL, null);
assert.ok(Math.abs(merged[2] - 1) < 1e-6, 'loudest decoded bin should reach full scale');
assert.ok(Math.abs(merged[0] - 0.25) < 1e-6);
assert.equal(merged[3], 0, 'undecoded tail stays empty without a preview');
});
test('prefix is rescaled to the preview, not to its own max', () => {
// Preview says the first half is quiet (0.25) and the second half is loud (1.0).
const preview = makePreview([0.25, 0.25, 1, 1]);
// We have decoded the quiet first half only. Raw RMS values are arbitrary in scale.
const prefix = new Float32Array(TOTAL / 2).fill(0.003);
const merged = mergeProgressiveWaveform(prefix, TOTAL, preview);
// Naive self-normalization would put the decoded half at 1.0 — far louder than the
// preview says it is, and louder than the not-yet-decoded loud half. It must stay at
// the preview's amplitude for that region instead.
assert.ok(Math.abs(merged[0] - 0.25) < 1e-6, `decoded region should match preview scale, got ${merged[0]}`);
assert.ok(merged[0] < merged[TOTAL - 1], 'quiet decoded half must stay below the loud undecoded half');
assert.ok(Math.abs(merged[TOTAL - 1] - 1) < 1e-6, 'undecoded tail keeps the preview value');
});
test('merge never exceeds full scale', () => {
const preview = makePreview([1, 1, 1, 1]);
const prefix = Float32Array.from([5, 10, 2]);
const merged = mergeProgressiveWaveform(prefix, TOTAL, preview);
for (let i = 0; i < merged.length; i++) {
assert.ok(merged[i] <= 1, `bin ${i} exceeded 1: ${merged[i]}`);
assert.ok(merged[i] >= 0, `bin ${i} went negative: ${merged[i]}`);
}
});
test('an all-silent prefix falls back to the preview rather than blanking the bar', () => {
const preview = makePreview([0.5, 0.6, 0.7, 0.8]);
const merged = mergeProgressiveWaveform(new Float32Array(64), TOTAL, preview);
assert.ok(Math.abs(merged[0] - 0.5) < 1e-6);
});
test('downsampling a partially-filled merge keeps the decoded region proportionate', () => {
// Decoded half is quiet, undecoded half is loud — after downsampling the relationship
// must survive, i.e. the global normalize must not lift the quiet decoded half.
const preview = makePreview([0.25, 0.25, 1, 1]);
const prefix = new Float32Array(TOTAL / 2).fill(0.003);
const merged = mergeProgressiveWaveform(prefix, TOTAL, preview);
const bars = downsampleWaveform(merged, 64);
assert.equal(bars.length, 64);
const firstQuarter = bars[8];
const lastQuarter = bars[56];
assert.ok(
lastQuarter > firstQuarter * 4,
`loud half (${lastQuarter}) should dominate the quiet decoded half (${firstQuarter})`
);
for (let i = 0; i < bars.length; i++) {
assert.ok(bars[i] >= 0 && bars[i] <= 1, `bar ${i} out of range: ${bars[i]}`);
}
});
test('downsample is unchanged for a fully accurate waveform', () => {
const source = Float32Array.from({ length: TOTAL }, (_, i) => (i < TOTAL / 2 ? 0.2 : 1));
const bars = downsampleWaveform(source, 32);
assert.equal(bars.length, 32);
assert.ok(Math.abs(bars[31] - 1) < 1e-6, 'loudest bar normalizes to full scale');
assert.ok(bars[0] < 0.1, 'x^2 power curve should push the quiet region well down');
});
+85
View File
@@ -0,0 +1,85 @@
// Pure waveform shaping — no native imports, so it stays unit-testable under `node --test`
// (see waveformMath.test.mts). waveform.ts re-exports these.
/**
* Splice a partially-decoded raw RMS prefix over the coarse preview, so the bar fills
* left-to-right with no visible seam.
*
* The prefix is raw — mid-decode the native side can't know the track's global max — while
* the preview is already normalized against the whole track. So the prefix is rescaled to
* the preview's amplitude over the region it covers rather than to its own max; normalizing
* it independently would make the decoded part read far louder than the rest until a loud
* section happened to arrive.
*/
export function mergeProgressiveWaveform(
prefix: Float32Array,
totalBins: number,
preview: Float32Array | null
): Float32Array {
const out = new Float32Array(Math.max(0, totalBins));
if (totalBins <= 0) return out;
// Stretch the (much coarser) preview across the full width first.
const hasPreview = !!preview && preview.length > 0;
if (preview && hasPreview) {
for (let i = 0; i < totalBins; i++) {
const p = Math.min(preview.length - 1, Math.floor((i / totalBins) * preview.length));
out[i] = preview[p];
}
}
const filled = Math.min(prefix.length, totalBins);
if (filled === 0) return out;
let prefixMax = 0;
for (let i = 0; i < filled; i++) if (prefix[i] > prefixMax) prefixMax = prefix[i];
if (prefixMax <= 0) return out;
// Match the preview's scale over the decoded region so the seam is continuous.
let reference = 0;
if (hasPreview) {
for (let i = 0; i < filled; i++) if (out[i] > reference) reference = out[i];
}
const scale = (reference > 0 ? reference : 1) / prefixMax;
for (let i = 0; i < filled; i++) out[i] = Math.min(1, prefix[i] * scale);
return out;
}
/**
* Downsample high-res peaks to `barCount` bars with a power curve and two
* smoothing passes. Ported verbatim from desktop waveformExtractor.ts so the
* mobile seek bar matches the desktop look.
*/
export function downsampleWaveform(source: Float32Array, barCount: number): Float32Array {
if (source.length === 0 || barCount <= 0) return new Float32Array(0);
const binsPerBar = source.length / barCount;
const peaks = new Float32Array(barCount);
for (let i = 0; i < barCount; i++) {
const start = Math.floor(i * binsPerBar);
const end = Math.max(start + 1, Math.floor((i + 1) * binsPerBar));
let sum = 0;
for (let j = start; j < end; j++) sum += source[j];
peaks[i] = sum / (end - start);
}
let max = 0;
for (let i = 0; i < barCount; i++) if (peaks[i] > max) max = peaks[i];
if (max > 0) for (let i = 0; i < barCount; i++) peaks[i] /= max;
// Power curve — exaggerate dynamic range.
for (let i = 0; i < barCount; i++) peaks[i] = peaks[i] ** 2;
// Two smoothing passes.
let current = peaks;
for (let p = 0; p < 2; p++) {
const smoothed = new Float32Array(current.length);
smoothed[0] = current[0];
smoothed[current.length - 1] = current[current.length - 1];
for (let i = 1; i < current.length - 1; i++) {
smoothed[i] = current[i - 1] * 0.25 + current[i] * 0.5 + current[i + 1] * 0.25;
}
current = smoothed;
}
return current;
}
+10 -4
View File
@@ -1,9 +1,15 @@
// Vendored fork of com.github.doublesymmetry:kotlinaudio v2.1.0 (Apache-2.0). // Vendored fork of com.github.doublesymmetry:kotlinaudio v2.1.0 (Apache-2.0).
// Substituted in for the Jitpack binary so we can inject a PCM-tap AudioProcessor // Substituted in for the Jitpack binary so we can inject a PCM-tap AudioProcessor
// into the ExoPlayer it builds (see players/BaseAudioPlayer.kt + scope/). The // into the ExoPlayer it builds (see players/BaseAudioPlayer.kt + scope/).
// ONLY source change vs upstream v2.1.0 is the single .setRenderersFactory(...) //
// line in BaseAudioPlayer's init and the new scope/ package. Keep that diff // Source changes vs upstream v2.1.0 — keep this list accurate, it is what makes
// minimal so re-vendoring on a kotlin-audio bump stays mechanical. // re-vendoring on a kotlin-audio bump mechanical:
// 1. .setRenderersFactory(buildScopeRenderersFactory(context)) in BaseAudioPlayer's init
// 2. the new scope/ package (taps, EQ, normalization gain)
// 3. GainBridge.activateFor(url) in onMediaItemTransition
// 4. shared per-player media-source factories + the fast path in
// getMediaSourceFromAudioItem (perf: Util.getUserAgent was a PackageManager
// lookup per queue item, which stalled the main thread on large queues)
plugins { plugins {
id 'com.android.library' id 'com.android.library'