mirror of
https://github.com/Boof2015/astra-mobile.git
synced 2026-08-17 19:24:22 +02:00
redo waveform and nomalization analysis
This commit is contained in:
+375
-239
@@ -6,11 +6,15 @@ import android.graphics.Bitmap
|
||||
import android.graphics.BitmapFactory
|
||||
import android.media.AudioFormat
|
||||
import android.media.MediaCodec
|
||||
import android.media.MediaCodecInfo
|
||||
import android.media.MediaCodecList
|
||||
import android.media.MediaExtractor
|
||||
import android.media.MediaFormat
|
||||
import android.media.MediaMetadataRetriever
|
||||
import android.net.Uri
|
||||
import android.os.Build
|
||||
import android.os.Handler
|
||||
import android.os.HandlerThread
|
||||
import android.provider.DocumentsContract
|
||||
import com.google.android.exoplayer2.MediaItem
|
||||
import com.google.android.exoplayer2.MetadataRetriever
|
||||
@@ -18,8 +22,10 @@ import com.google.android.exoplayer2.metadata.id3.BinaryFrame
|
||||
import com.google.android.exoplayer2.metadata.id3.InternalFrame
|
||||
import com.google.android.exoplayer2.metadata.id3.TextInformationFrame
|
||||
import com.google.android.exoplayer2.metadata.flac.VorbisComment
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import kotlin.math.PI
|
||||
import kotlin.math.abs
|
||||
import kotlin.math.log10
|
||||
@@ -35,6 +41,7 @@ import expo.modules.kotlin.records.Record
|
||||
import expo.modules.astralibraryscanner.data.AstraLibraryRepository
|
||||
import expo.modules.astralibraryscanner.data.LocalAudioFile
|
||||
import expo.modules.astralibraryscanner.data.LocalAudioMetadata
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.awaitAll
|
||||
@@ -42,6 +49,7 @@ import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.sync.Semaphore
|
||||
import kotlinx.coroutines.sync.withPermit
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.withTimeoutOrNull
|
||||
import java.io.File
|
||||
import java.security.MessageDigest
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
@@ -52,11 +60,29 @@ class FileRequest : Record {
|
||||
@Field val coverUri: String? = null
|
||||
}
|
||||
|
||||
/** Result of one scan-time decode: waveform peaks + integrated loudness + sample peak. */
|
||||
/**
|
||||
* Result of ONE decode pass over a track: waveform peaks + integrated loudness + sample
|
||||
* peak, plus timing so the JS side can report how fast the decode actually ran. Peaks and
|
||||
* loudness share a pass because both need every sample; decoding twice was pure waste.
|
||||
*/
|
||||
class AudioAnalysis : Record {
|
||||
@Field var peaks: FloatArray = FloatArray(0)
|
||||
@Field var lufs: Double? = null // integrated LUFS (negative dB); null if unmeasured
|
||||
@Field var peak: Double? = null // absolute sample peak, linear [0,1]; null if unmeasured
|
||||
/** True when the decode was cancelled mid-flight; peaks/lufs are then meaningless. */
|
||||
@Field var cancelled: Boolean = false
|
||||
/** Wall-clock decode time in ms — the number that decides whether we need a native decoder. */
|
||||
@Field var decodeMs: Double? = null
|
||||
/** Track duration in ms, from the container. */
|
||||
@Field var durationMs: Double? = null
|
||||
/** durationMs / decodeMs — "how many times faster than realtime". Higher is better. */
|
||||
@Field var realtimeFactor: Double? = null
|
||||
/** Which MediaCodec actually ran (e.g. "c2.android.flac.decoder"). */
|
||||
@Field var decoderName: String? = null
|
||||
/** Audio track mime (e.g. "audio/flac"). */
|
||||
@Field var mime: String? = null
|
||||
/** Whether the loudness meter rode along on this pass. */
|
||||
@Field var withLoudness: Boolean = false
|
||||
}
|
||||
|
||||
/** ReplayGain tags read from the container (no audio decode). Null = tag absent. */
|
||||
@@ -75,13 +101,20 @@ class AstraLibraryScannerModule : Module() {
|
||||
// read and hashed once, not once per track.
|
||||
private val coverHashMemo = ConcurrentHashMap<String, String>()
|
||||
|
||||
// Waveform decode is whole-file and CPU-heavy; throttle concurrent decodes.
|
||||
// Analysis decode is whole-file and CPU-heavy; throttle concurrent decodes. Also keeps
|
||||
// us from monopolising decoder instances while a track is actually playing.
|
||||
private val waveformSemaphore = Semaphore(2)
|
||||
|
||||
// Cancellation flags for in-flight analyses, keyed by track URI. Set by cancelAnalysis
|
||||
// so a skipped-past track stops burning CPU instead of running to completion holding a
|
||||
// semaphore permit. Registered before the permit is acquired, so a queued-but-unstarted
|
||||
// decode can be cancelled too.
|
||||
private val activeAnalyses = ConcurrentHashMap<String, AtomicBoolean>()
|
||||
|
||||
override fun definition() = ModuleDefinition {
|
||||
Name("AstraLibraryScanner")
|
||||
|
||||
Events("onScanProgress")
|
||||
Events("onScanProgress", "onWaveformProgress")
|
||||
|
||||
AsyncFunction("listAudioFiles") Coroutine { treeUri: String, extensions: List<String> ->
|
||||
withContext(Dispatchers.IO) { listAudioFiles(treeUri, extensions) }
|
||||
@@ -144,33 +177,42 @@ class AstraLibraryScannerModule : Module() {
|
||||
}
|
||||
}
|
||||
|
||||
// Offline waveform peaks for the seek bar: full PCM decode -> RMS per bin,
|
||||
// normalized to [0,1]. Heavy (whole-file decode), so cap concurrency and
|
||||
// run lazily per track on the JS side; results are cached in SQLite there.
|
||||
AsyncFunction("extractWaveform") Coroutine { uri: String, bins: Int ->
|
||||
waveformSemaphore.withPermit {
|
||||
withContext(Dispatchers.IO) { decodeAndAnalyze(uri, if (bins > 0) bins else 512).peaks }
|
||||
// ONE whole-file PCM decode producing waveform peaks and (when withLoudness) gated
|
||||
// integrated loudness + sample peak. Both need every sample, so they ride the same
|
||||
// pass — running them separately meant decoding each track twice. Heavy, so cap
|
||||
// concurrency; the JS side caches results in SQLite and prefetches the queue ahead.
|
||||
// Emits onWaveformProgress as bins finalize so the seek bar can fill in left-to-right.
|
||||
AsyncFunction("analyzeTrack") Coroutine { uri: String, bins: Int, withLoudness: Boolean ->
|
||||
val flag = AtomicBoolean(false)
|
||||
activeAnalyses[uri] = flag
|
||||
try {
|
||||
waveformSemaphore.withPermit {
|
||||
// Cancelled while queued behind another decode — don't start at all.
|
||||
if (flag.get()) AudioAnalysis().apply { cancelled = true }
|
||||
else withContext(Dispatchers.IO) {
|
||||
runAnalysis(uri, if (bins > 0) bins else 512, withLoudness, flag)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
activeAnalyses.remove(uri, flag)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop an in-flight (or still-queued) analysis for this URI. Safe to call for a URI
|
||||
// with no analysis running. The decode bails at the next buffer boundary.
|
||||
AsyncFunction("cancelAnalysis") Coroutine { uri: String ->
|
||||
activeAnalyses[uri]?.set(true)
|
||||
}
|
||||
|
||||
// Fast waveform preview for first paint: sparse short-window decode across
|
||||
// the file. The JS side shows this immediately but only persists the full
|
||||
// extractWaveform result.
|
||||
// the file. The JS side shows this immediately as the coarse full-width shape
|
||||
// that the progressive accurate pass then fills over; only analyzeTrack persists.
|
||||
AsyncFunction("extractWaveformPreview") Coroutine { uri: String, bins: Int ->
|
||||
waveformSemaphore.withPermit {
|
||||
withContext(Dispatchers.IO) { decodeWaveformPreview(uri, if (bins > 0) bins else 96) }
|
||||
}
|
||||
}
|
||||
|
||||
// Fast loudness (M4): decodes only a few short windows spread across the track
|
||||
// (not the whole file) + gated K-weighting -> integrated LUFS + sample peak.
|
||||
// Waveform peaks stay lazy/full-decode (extractWaveform), decoupled from this.
|
||||
AsyncFunction("measureLoudness") Coroutine { uri: String ->
|
||||
waveformSemaphore.withPermit {
|
||||
withContext(Dispatchers.IO) { measureLoudness(uri) }
|
||||
}
|
||||
}
|
||||
|
||||
// ReplayGain tags (M4): reads container metadata only (no PCM decode), so it is
|
||||
// cheap and lets us normalize a tagged library without the slow loudness decode.
|
||||
AsyncFunction("readReplayGain") Coroutine { uri: String ->
|
||||
@@ -644,18 +686,43 @@ class AstraLibraryScannerModule : Module() {
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Waveform peaks (offline RMS bins)
|
||||
// Track analysis: waveform peaks + loudness in ONE decode pass
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// One whole-file PCM decode -> per-bin RMS waveform peaks (normalized [0,1]) for the
|
||||
// seek bar. Returns empty peaks on any failure (caller falls back to a flat seek
|
||||
// bar). Loudness is measured separately by measureLoudness.
|
||||
private fun decodeAndAnalyze(uriStr: String, bins: Int): AudioAnalysis {
|
||||
/** Throttle for onWaveformProgress — ~12 emits/sec is plenty for a fill animation. */
|
||||
private val progressEmitNanos = 80L * 1_000_000L
|
||||
|
||||
/** Hard ceiling so a corrupt file can't hang a decode forever holding a semaphore permit. */
|
||||
private val analysisTimeoutMs = 180_000L
|
||||
|
||||
/**
|
||||
* One whole-file PCM decode producing per-bin RMS waveform peaks (normalized to [0,1])
|
||||
* and, when `withLoudness`, gated integrated LUFS + absolute sample peak. Both analyses
|
||||
* need every sample, so they share a pass.
|
||||
*
|
||||
* Uses MediaCodec in async (callback) mode: the old synchronous dequeue loop burned a
|
||||
* 10ms timeout every time a buffer wasn't ready, thousands of times per track. All four
|
||||
* callbacks land on one handler thread, so the extractor and accumulator are touched from
|
||||
* exactly one thread and need no locking.
|
||||
*
|
||||
* Returns empty peaks on any failure and sets `cancelled` if it bailed early — callers
|
||||
* must not persist a cancelled result.
|
||||
*/
|
||||
private suspend fun runAnalysis(
|
||||
uriStr: String,
|
||||
bins: Int,
|
||||
withLoudness: Boolean,
|
||||
cancelFlag: AtomicBoolean,
|
||||
): AudioAnalysis {
|
||||
val context = requireContext()
|
||||
val result = AudioAnalysis()
|
||||
result.withLoudness = withLoudness
|
||||
val uri = Uri.parse(uriStr)
|
||||
val extractor = MediaExtractor()
|
||||
var codec: MediaCodec? = null
|
||||
var handlerThread: HandlerThread? = null
|
||||
val startNanos = System.nanoTime()
|
||||
|
||||
try {
|
||||
extractor.setDataSource(context, uri, null)
|
||||
|
||||
@@ -670,63 +737,305 @@ class AstraLibraryScannerModule : Module() {
|
||||
val format = trackFormat ?: return result
|
||||
extractor.selectTrack(trackIndex)
|
||||
|
||||
val mime = format.getString(MediaFormat.KEY_MIME) ?: return result
|
||||
result.mime = mime
|
||||
|
||||
val sampleRate =
|
||||
if (format.containsKey(MediaFormat.KEY_SAMPLE_RATE)) format.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100
|
||||
val durationUs =
|
||||
if (format.containsKey(MediaFormat.KEY_DURATION)) format.getLong(MediaFormat.KEY_DURATION) else 0L
|
||||
result.durationMs = durationUs / 1000.0
|
||||
val totalFrames = max(1L, (durationUs / 1_000_000.0 * sampleRate).toLong())
|
||||
var channelCount =
|
||||
|
||||
val acc = AnalyzeAccumulator(bins, totalFrames, withLoudness)
|
||||
acc.sampleRate = sampleRate
|
||||
acc.channelCount =
|
||||
if (format.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) else 2
|
||||
var pcmFloat = false
|
||||
|
||||
val sumSquares = DoubleArray(bins)
|
||||
val counts = LongArray(bins)
|
||||
val decoder = createAnalysisDecoder(mime)
|
||||
codec = decoder
|
||||
result.decoderName = decoder.name
|
||||
|
||||
handlerThread = HandlerThread("astra-analyze").also { it.start() }
|
||||
val done = CompletableDeferred<Unit>()
|
||||
var sawInputEOS = false
|
||||
var lastEmitNanos = 0L
|
||||
var lastEmitBin = 0
|
||||
|
||||
decoder.setCallback(
|
||||
object : MediaCodec.Callback() {
|
||||
override fun onInputBufferAvailable(mc: MediaCodec, index: Int) {
|
||||
if (done.isCompleted) return
|
||||
try {
|
||||
if (cancelFlag.get()) {
|
||||
result.cancelled = true
|
||||
done.complete(Unit)
|
||||
return
|
||||
}
|
||||
if (sawInputEOS) return
|
||||
val buf = mc.getInputBuffer(index) ?: return
|
||||
val size = extractor.readSampleData(buf, 0)
|
||||
if (size < 0) {
|
||||
mc.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
|
||||
sawInputEOS = true
|
||||
} else {
|
||||
mc.queueInputBuffer(index, 0, size, extractor.sampleTime, 0)
|
||||
extractor.advance()
|
||||
}
|
||||
} catch (_: Throwable) {
|
||||
done.complete(Unit)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onOutputBufferAvailable(
|
||||
mc: MediaCodec,
|
||||
index: Int,
|
||||
info: MediaCodec.BufferInfo,
|
||||
) {
|
||||
if (done.isCompleted) return
|
||||
try {
|
||||
if (cancelFlag.get()) {
|
||||
result.cancelled = true
|
||||
done.complete(Unit)
|
||||
return
|
||||
}
|
||||
if (info.size > 0) {
|
||||
val out = mc.getOutputBuffer(index)
|
||||
if (out != null) {
|
||||
out.position(info.offset)
|
||||
out.limit(info.offset + info.size)
|
||||
out.order(ByteOrder.nativeOrder())
|
||||
acc.accumulate(out)
|
||||
}
|
||||
}
|
||||
val eos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
|
||||
mc.releaseOutputBuffer(index, false)
|
||||
|
||||
// Progressive emit so the seek bar fills left-to-right instead of snapping
|
||||
// in at the end. Skipped on the EOS buffer — the promise carries the final,
|
||||
// globally-normalized result a moment later.
|
||||
val now = System.nanoTime()
|
||||
if (!eos && acc.filledBins > lastEmitBin && now - lastEmitNanos >= progressEmitNanos) {
|
||||
lastEmitNanos = now
|
||||
lastEmitBin = acc.filledBins
|
||||
emitWaveformProgress(uriStr, acc, lastEmitBin)
|
||||
}
|
||||
if (eos) done.complete(Unit)
|
||||
} catch (_: Throwable) {
|
||||
done.complete(Unit)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onOutputFormatChanged(mc: MediaCodec, fmt: MediaFormat) {
|
||||
if (fmt.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) {
|
||||
acc.channelCount = fmt.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
|
||||
}
|
||||
if (fmt.containsKey(MediaFormat.KEY_SAMPLE_RATE)) {
|
||||
acc.sampleRate = fmt.getInteger(MediaFormat.KEY_SAMPLE_RATE)
|
||||
}
|
||||
if (fmt.containsKey(MediaFormat.KEY_PCM_ENCODING)) {
|
||||
acc.pcmFloat =
|
||||
fmt.getInteger(MediaFormat.KEY_PCM_ENCODING) == AudioFormat.ENCODING_PCM_FLOAT
|
||||
}
|
||||
}
|
||||
|
||||
override fun onError(mc: MediaCodec, e: MediaCodec.CodecException) {
|
||||
done.complete(Unit)
|
||||
}
|
||||
},
|
||||
Handler(handlerThread.looper),
|
||||
)
|
||||
|
||||
codec = MediaCodec.createDecoderByType(format.getString(MediaFormat.KEY_MIME)!!)
|
||||
codec.configure(format, null, null, 0)
|
||||
codec.start()
|
||||
|
||||
val info = MediaCodec.BufferInfo()
|
||||
var sawInputEOS = false
|
||||
var sawOutputEOS = false
|
||||
var frame = 0L
|
||||
if (withTimeoutOrNull(analysisTimeoutMs) { done.await() } == null) {
|
||||
// Timed out: peaks are partial, so treat it as a cancellation rather than caching
|
||||
// a truncated waveform.
|
||||
result.cancelled = true
|
||||
}
|
||||
if (result.cancelled) return result
|
||||
|
||||
while (!sawOutputEOS) {
|
||||
if (!sawInputEOS) {
|
||||
val inIndex = codec.dequeueInputBuffer(10_000)
|
||||
if (inIndex >= 0) {
|
||||
val inBuf = codec.getInputBuffer(inIndex)!!
|
||||
val size = extractor.readSampleData(inBuf, 0)
|
||||
if (size < 0) {
|
||||
codec.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
|
||||
sawInputEOS = true
|
||||
} else {
|
||||
codec.queueInputBuffer(inIndex, 0, size, extractor.sampleTime, 0)
|
||||
extractor.advance()
|
||||
result.peaks = acc.finalPeaks()
|
||||
if (withLoudness) {
|
||||
result.lufs = acc.loudness
|
||||
result.peak = acc.samplePeak
|
||||
}
|
||||
val decodeMs = (System.nanoTime() - startNanos) / 1_000_000.0
|
||||
result.decodeMs = decodeMs
|
||||
val dur = result.durationMs
|
||||
if (dur != null && dur > 0 && decodeMs > 0) result.realtimeFactor = dur / decodeMs
|
||||
return result
|
||||
} catch (_: Throwable) {
|
||||
return result
|
||||
} finally {
|
||||
// Order matters: stopping the codec while a callback is mid-flight on the handler
|
||||
// thread can crash. Quit the looper and wait for the in-flight callback to drain
|
||||
// first, THEN tear the codec down.
|
||||
try {
|
||||
handlerThread?.quitSafely()
|
||||
handlerThread?.join(1_000)
|
||||
} catch (_: Throwable) {}
|
||||
try { codec?.stop() } catch (_: Throwable) {}
|
||||
try { codec?.release() } catch (_: Throwable) {}
|
||||
try { extractor.release() } catch (_: Throwable) {}
|
||||
}
|
||||
}
|
||||
|
||||
// Emits the raw (un-normalized) RMS prefix; JS normalizes against its own max, so the
|
||||
// bars rescale slightly as louder material arrives rather than needing a global max we
|
||||
// don't have yet.
|
||||
private fun emitWaveformProgress(uri: String, acc: AnalyzeAccumulator, filledBins: Int) {
|
||||
try {
|
||||
sendEvent(
|
||||
"onWaveformProgress",
|
||||
mapOf(
|
||||
"uri" to uri,
|
||||
"filledBins" to filledBins,
|
||||
"totalBins" to acc.bins,
|
||||
"peaks" to acc.rmsPrefix(filledBins),
|
||||
),
|
||||
)
|
||||
} catch (_: Throwable) {
|
||||
// Best-effort: the final result still arrives via the promise.
|
||||
}
|
||||
}
|
||||
|
||||
// Offline analysis wants raw throughput. Hardware audio decoders are tuned for low-power
|
||||
// realtime playback, not bulk decode, and the instance is a scarce global resource shared
|
||||
// with the track that is actually playing — so prefer a software decoder and fall back to
|
||||
// the platform's default pick.
|
||||
private fun createAnalysisDecoder(mime: String): MediaCodec {
|
||||
try {
|
||||
val info = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.firstOrNull { c ->
|
||||
!c.isEncoder &&
|
||||
c.supportedTypes.any { it.equals(mime, ignoreCase = true) } &&
|
||||
isSoftwareDecoder(c)
|
||||
}
|
||||
if (info != null) return MediaCodec.createByCodecName(info.name)
|
||||
} catch (_: Throwable) {
|
||||
// Fall through to the platform default.
|
||||
}
|
||||
return MediaCodec.createDecoderByType(mime)
|
||||
}
|
||||
|
||||
private fun isSoftwareDecoder(info: MediaCodecInfo): Boolean {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) return info.isSoftwareOnly
|
||||
val name = info.name.lowercase()
|
||||
return name.startsWith("omx.google.") || name.startsWith("c2.android.")
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-bin RMS accumulation over the decoded PCM stream, with the K-weighted loudness
|
||||
* meter folded in when requested.
|
||||
*
|
||||
* Two things matter here because this runs ~20M times for a 4-minute stereo track: PCM is
|
||||
* bulk-copied out of the codec buffer into a reused scratch array rather than read one
|
||||
* sample at a time, and frames are consumed in runs (every frame landing in the same bin
|
||||
* is summed in one tight loop) instead of recomputing the bin index per frame with a
|
||||
* floating-point division.
|
||||
*/
|
||||
private class AnalyzeAccumulator(
|
||||
val bins: Int,
|
||||
private val totalFrames: Long,
|
||||
private val withLoudness: Boolean,
|
||||
) {
|
||||
private val sumSquares = DoubleArray(bins)
|
||||
private val counts = LongArray(bins)
|
||||
|
||||
var channelCount = 2
|
||||
var sampleRate = 44100
|
||||
var pcmFloat = false
|
||||
|
||||
/** Index of the highest bin reached — every bin below it is fully accumulated. */
|
||||
var filledBins = 0
|
||||
private set
|
||||
|
||||
private var frame = 0L
|
||||
private var meter: LoudnessMeter? = null
|
||||
private var floatScratch = FloatArray(0)
|
||||
private var shortScratch = ShortArray(0)
|
||||
|
||||
val loudness: Double? get() = meter?.lufs()
|
||||
val samplePeak: Double? get() = meter?.peak
|
||||
|
||||
fun accumulate(out: ByteBuffer) {
|
||||
val ch = channelCount.coerceAtLeast(1)
|
||||
// Created lazily: the true output channel count / rate only arrive with the first
|
||||
// onOutputFormatChanged, which always precedes the first output buffer.
|
||||
if (withLoudness && meter == null) meter = LoudnessMeter(ch, sampleRate)
|
||||
if (pcmFloat) {
|
||||
val fb = out.asFloatBuffer()
|
||||
val n = fb.remaining()
|
||||
if (floatScratch.size < n) floatScratch = FloatArray(n)
|
||||
fb.get(floatScratch, 0, n)
|
||||
consume(floatScratch, null, n, ch)
|
||||
} else {
|
||||
val sb = out.asShortBuffer()
|
||||
val n = sb.remaining()
|
||||
if (shortScratch.size < n) shortScratch = ShortArray(n)
|
||||
sb.get(shortScratch, 0, n)
|
||||
consume(null, shortScratch, n, ch)
|
||||
}
|
||||
}
|
||||
|
||||
private fun consume(f: FloatArray?, s: ShortArray?, n: Int, ch: Int) {
|
||||
val m = meter
|
||||
var k = 0
|
||||
while (k < n) {
|
||||
var bin = ((frame * bins) / totalFrames).toInt()
|
||||
if (bin < 0) bin = 0 else if (bin >= bins) bin = bins - 1
|
||||
// First frame belonging to the next bin: ceil((bin + 1) * totalFrames / bins).
|
||||
val boundary = ((bin + 1).toLong() * totalFrames + bins - 1L) / bins
|
||||
val framesAvail = (n - k) / ch
|
||||
if (framesAvail <= 0) break // trailing partial frame; drop it
|
||||
var run = (boundary - frame).coerceAtLeast(1L)
|
||||
if (run > framesAvail) run = framesAvail.toLong()
|
||||
val end = k + run.toInt() * ch
|
||||
|
||||
var acc = 0.0
|
||||
var j = k
|
||||
if (m == null) {
|
||||
if (f != null) {
|
||||
while (j < end) { val v = f[j].toDouble(); acc += v * v; j++ }
|
||||
} else if (s != null) {
|
||||
while (j < end) { val v = s[j] / 32768.0; acc += v * v; j++ }
|
||||
}
|
||||
} else {
|
||||
var c = 0
|
||||
if (f != null) {
|
||||
while (j < end) {
|
||||
val v = f[j].toDouble(); acc += v * v; m.process(v, c)
|
||||
j++; c++; if (c == ch) c = 0
|
||||
}
|
||||
} else if (s != null) {
|
||||
while (j < end) {
|
||||
val v = s[j] / 32768.0; acc += v * v; m.process(v, c)
|
||||
j++; c++; if (c == ch) c = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val outIndex = codec.dequeueOutputBuffer(info, 10_000)
|
||||
if (outIndex >= 0) {
|
||||
if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) sawOutputEOS = true
|
||||
if (info.size > 0) {
|
||||
val out = codec.getOutputBuffer(outIndex)!!
|
||||
out.position(info.offset)
|
||||
out.limit(info.offset + info.size)
|
||||
out.order(ByteOrder.nativeOrder())
|
||||
frame = accumulateAnalyze(out, pcmFloat, channelCount, bins, totalFrames, frame, sumSquares, counts)
|
||||
}
|
||||
codec.releaseOutputBuffer(outIndex, false)
|
||||
} else if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
|
||||
val nf = codec.outputFormat
|
||||
if (nf.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) channelCount = nf.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
|
||||
if (nf.containsKey(MediaFormat.KEY_PCM_ENCODING)) {
|
||||
pcmFloat = nf.getInteger(MediaFormat.KEY_PCM_ENCODING) == AudioFormat.ENCODING_PCM_FLOAT
|
||||
}
|
||||
}
|
||||
sumSquares[bin] += acc
|
||||
counts[bin] += run * ch
|
||||
frame += run
|
||||
k = end
|
||||
if (bin > filledBins) filledBins = bin
|
||||
}
|
||||
}
|
||||
|
||||
/** Raw, un-normalized RMS for the first `count` bins (progressive emit). */
|
||||
fun rmsPrefix(count: Int): FloatArray {
|
||||
val n = count.coerceIn(0, bins)
|
||||
val out = FloatArray(n)
|
||||
for (i in 0 until n) {
|
||||
if (counts[i] > 0) out[i] = sqrt(sumSquares[i] / counts[i]).toFloat()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Final peaks, normalized against the global max across every bin. */
|
||||
fun finalPeaks(): FloatArray {
|
||||
val peaks = FloatArray(bins)
|
||||
var globalMax = 0.0
|
||||
for (i in 0 until bins) {
|
||||
@@ -739,14 +1048,7 @@ class AstraLibraryScannerModule : Module() {
|
||||
if (globalMax > 0) {
|
||||
for (i in 0 until bins) peaks[i] = (peaks[i] / globalMax).toFloat()
|
||||
}
|
||||
result.peaks = peaks
|
||||
return result
|
||||
} catch (_: Throwable) {
|
||||
return result
|
||||
} finally {
|
||||
try { codec?.stop() } catch (_: Throwable) {}
|
||||
try { codec?.release() } catch (_: Throwable) {}
|
||||
try { extractor.release() } catch (_: Throwable) {}
|
||||
return peaks
|
||||
}
|
||||
}
|
||||
|
||||
@@ -758,7 +1060,7 @@ class AstraLibraryScannerModule : Module() {
|
||||
|
||||
// Sparse preview waveform: seek to a bounded number of points, decode a very
|
||||
// short audio window at each point, and normalize those RMS samples. This is
|
||||
// intentionally approximate; decodeAndAnalyze remains the accurate cache fill.
|
||||
// intentionally approximate; runAnalysis remains the accurate cache fill.
|
||||
private fun decodeWaveformPreview(uriStr: String, bins: Int): FloatArray {
|
||||
val context = requireContext()
|
||||
val previewBins = bins.coerceIn(16, 128)
|
||||
@@ -916,172 +1218,6 @@ class AstraLibraryScannerModule : Module() {
|
||||
return PcmEnergy(sumSquares, sampleCount, frameCount)
|
||||
}
|
||||
|
||||
// Integrated gated loudness over the WHOLE file (accurate — subset sampling caused
|
||||
// too much loudness inconsistency). Decodes the full track and feeds the gated
|
||||
// K-weighting meter. Measured on the fly per track (current + queue lookahead) and
|
||||
// cached, so the cost is paid once per track, never in a bulk background pass.
|
||||
private fun measureLoudness(uriStr: String): AudioAnalysis {
|
||||
val context = requireContext()
|
||||
val result = AudioAnalysis()
|
||||
val uri = Uri.parse(uriStr)
|
||||
val extractor = MediaExtractor()
|
||||
var codec: MediaCodec? = null
|
||||
try {
|
||||
extractor.setDataSource(context, uri, null)
|
||||
|
||||
var trackFormat: MediaFormat? = null
|
||||
var trackIndex = -1
|
||||
for (i in 0 until extractor.trackCount) {
|
||||
val f = extractor.getTrackFormat(i)
|
||||
if (f.getString(MediaFormat.KEY_MIME)?.startsWith("audio/") == true) {
|
||||
trackFormat = f; trackIndex = i; break
|
||||
}
|
||||
}
|
||||
val format = trackFormat ?: return result
|
||||
extractor.selectTrack(trackIndex)
|
||||
|
||||
val sampleRate =
|
||||
if (format.containsKey(MediaFormat.KEY_SAMPLE_RATE)) format.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100
|
||||
var channelCount =
|
||||
if (format.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) else 2
|
||||
var pcmFloat = false
|
||||
|
||||
codec = MediaCodec.createDecoderByType(format.getString(MediaFormat.KEY_MIME)!!)
|
||||
codec.configure(format, null, null, 0)
|
||||
codec.start()
|
||||
val info = MediaCodec.BufferInfo()
|
||||
|
||||
var meter: LoudnessMeter? = null
|
||||
var sawInputEOS = false
|
||||
var sawOutputEOS = false
|
||||
while (!sawOutputEOS) {
|
||||
if (!sawInputEOS) {
|
||||
val inIndex = codec.dequeueInputBuffer(10_000)
|
||||
if (inIndex >= 0) {
|
||||
val inBuf = codec.getInputBuffer(inIndex)!!
|
||||
val size = extractor.readSampleData(inBuf, 0)
|
||||
if (size < 0) {
|
||||
codec.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
|
||||
sawInputEOS = true
|
||||
} else {
|
||||
codec.queueInputBuffer(inIndex, 0, size, extractor.sampleTime, 0)
|
||||
extractor.advance()
|
||||
}
|
||||
}
|
||||
}
|
||||
val outIndex = codec.dequeueOutputBuffer(info, 10_000)
|
||||
if (outIndex >= 0) {
|
||||
if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) sawOutputEOS = true
|
||||
if (info.size > 0) {
|
||||
val out = codec.getOutputBuffer(outIndex)!!
|
||||
out.position(info.offset)
|
||||
out.limit(info.offset + info.size)
|
||||
out.order(ByteOrder.nativeOrder())
|
||||
val m = meter ?: LoudnessMeter(channelCount, sampleRate).also { meter = it }
|
||||
feedMeter(out, pcmFloat, channelCount, m)
|
||||
}
|
||||
codec.releaseOutputBuffer(outIndex, false)
|
||||
} else if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
|
||||
val nf = codec.outputFormat
|
||||
if (nf.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) channelCount = nf.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
|
||||
if (nf.containsKey(MediaFormat.KEY_PCM_ENCODING)) {
|
||||
pcmFloat = nf.getInteger(MediaFormat.KEY_PCM_ENCODING) == AudioFormat.ENCODING_PCM_FLOAT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
meter?.let {
|
||||
result.lufs = it.lufs()
|
||||
result.peak = it.peak
|
||||
}
|
||||
return result
|
||||
} catch (_: Throwable) {
|
||||
return result
|
||||
} finally {
|
||||
try { codec?.stop() } catch (_: Throwable) {}
|
||||
try { codec?.release() } catch (_: Throwable) {}
|
||||
try { extractor.release() } catch (_: Throwable) {}
|
||||
}
|
||||
}
|
||||
|
||||
// Feeds one decoded PCM buffer (16-bit or float) to the loudness meter.
|
||||
private fun feedMeter(
|
||||
out: java.nio.ByteBuffer,
|
||||
pcmFloat: Boolean,
|
||||
channelCount: Int,
|
||||
meter: LoudnessMeter
|
||||
) {
|
||||
if (pcmFloat) {
|
||||
val fb = out.asFloatBuffer()
|
||||
val n = fb.remaining()
|
||||
var k = 0
|
||||
while (k < n) {
|
||||
var c = 0
|
||||
while (c < channelCount && k < n) {
|
||||
meter.process(fb.get(k).toDouble(), c); k++; c++
|
||||
}
|
||||
}
|
||||
} else {
|
||||
val sb = out.asShortBuffer()
|
||||
val n = sb.remaining()
|
||||
var k = 0
|
||||
while (k < n) {
|
||||
var c = 0
|
||||
while (c < channelCount && k < n) {
|
||||
meter.process(sb.get(k) / 32768.0, c); k++; c++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Folds one decoded PCM buffer into the per-bin RMS accumulators and, when a
|
||||
// loudness meter is provided, the K-weighted loudness + sample peak. Handles
|
||||
// 16-bit (default) and float PCM. Returns the updated running frame index.
|
||||
private fun accumulateAnalyze(
|
||||
out: java.nio.ByteBuffer,
|
||||
pcmFloat: Boolean,
|
||||
channelCount: Int,
|
||||
bins: Int,
|
||||
totalFrames: Long,
|
||||
startFrame: Long,
|
||||
sumSquares: DoubleArray,
|
||||
counts: LongArray
|
||||
): Long {
|
||||
var frame = startFrame
|
||||
if (pcmFloat) {
|
||||
val fb = out.asFloatBuffer()
|
||||
val n = fb.remaining()
|
||||
var k = 0
|
||||
while (k < n) {
|
||||
val bin = ((frame.toDouble() / totalFrames) * bins).toInt().coerceIn(0, bins - 1)
|
||||
var c = 0
|
||||
while (c < channelCount && k < n) {
|
||||
val s = fb.get(k).toDouble()
|
||||
sumSquares[bin] += s * s
|
||||
k++; c++
|
||||
}
|
||||
counts[bin] += c.toLong()
|
||||
frame++
|
||||
}
|
||||
} else {
|
||||
val sb = out.asShortBuffer()
|
||||
val n = sb.remaining()
|
||||
var k = 0
|
||||
while (k < n) {
|
||||
val bin = ((frame.toDouble() / totalFrames) * bins).toInt().coerceIn(0, bins - 1)
|
||||
var c = 0
|
||||
while (c < channelCount && k < n) {
|
||||
val s = sb.get(k) / 32768.0
|
||||
sumSquares[bin] += s * s
|
||||
k++; c++
|
||||
}
|
||||
counts[bin] += c.toLong()
|
||||
frame++
|
||||
}
|
||||
}
|
||||
return frame
|
||||
}
|
||||
|
||||
// Gated integrated K-weighted loudness per ITU-R BS.1770 + absolute sample peak.
|
||||
// Two cascaded biquads (high-shelf pre-filter + RLB high-pass) per channel with
|
||||
// pyloudnorm-reference coefficients (so the -0.691 offset holds), accumulated into
|
||||
|
||||
@@ -89,10 +89,44 @@ export interface NativeScanResult {
|
||||
catalogRevision: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Partial waveform emitted while `analyzeTrack` decodes, so the seek bar can fill in
|
||||
* left-to-right. `peaks` holds RAW (un-normalized) RMS for bins `[0, filledBins)` — the
|
||||
* global max isn't known until the decode ends, so callers normalize against the max of
|
||||
* what they've received so far and accept a slight rescale as louder material arrives.
|
||||
*/
|
||||
export interface WaveformProgressEvent {
|
||||
/** Track URI this partial belongs to — callers must filter, decodes overlap. */
|
||||
uri: string;
|
||||
filledBins: number;
|
||||
totalBins: number;
|
||||
peaks: number[];
|
||||
}
|
||||
|
||||
type AstraLibraryScannerEvents = {
|
||||
onScanProgress: (event: ScanProgressEvent) => void;
|
||||
onWaveformProgress: (event: WaveformProgressEvent) => void;
|
||||
};
|
||||
|
||||
/** One decode pass: waveform peaks + (optionally) loudness, plus timing. */
|
||||
export interface TrackAnalysis {
|
||||
/** `bins` RMS peaks normalized to [0,1]; empty on failure. */
|
||||
peaks: number[];
|
||||
/** Integrated LUFS; null when unmeasured (withLoudness false) or unmeasurable. */
|
||||
lufs: number | null;
|
||||
/** Absolute sample peak, linear [0,1]; null when unmeasured. */
|
||||
peak: number | null;
|
||||
/** True if the decode bailed early — do NOT persist peaks or loudness. */
|
||||
cancelled: boolean;
|
||||
decodeMs: number | null;
|
||||
durationMs: number | null;
|
||||
/** durationMs / decodeMs — how many times faster than realtime the decode ran. */
|
||||
realtimeFactor: number | null;
|
||||
decoderName: string | null;
|
||||
mime: string | null;
|
||||
withLoudness: boolean;
|
||||
}
|
||||
|
||||
declare class AstraLibraryScannerModuleType extends NativeModule<AstraLibraryScannerEvents> {
|
||||
listAudioFiles(treeUri: string, extensions: string[]): Promise<ListResult>;
|
||||
extractMetadata(files: { uri: string; coverUri?: string | null }[]): Promise<ExtractedMetadata[]>;
|
||||
@@ -102,21 +136,23 @@ declare class AstraLibraryScannerModuleType extends NativeModule<AstraLibrarySca
|
||||
extensions: string[]
|
||||
): Promise<NativeScanResult>;
|
||||
/**
|
||||
* Decode the file's PCM and return `bins` RMS peaks normalized to [0,1] for
|
||||
* the waveform seek bar. Whole-file decode (heavy); returns [] on failure.
|
||||
* ONE whole-file PCM decode producing `bins` RMS waveform peaks and, when
|
||||
* `withLoudness`, gated integrated LUFS + sample peak. Both analyses need every
|
||||
* sample, so they share a pass — ask for loudness here whenever you'd otherwise
|
||||
* measure it separately. Heavy; concurrency is capped natively at 2 and results
|
||||
* should be cached. Emits `onWaveformProgress` as bins finalize.
|
||||
*/
|
||||
extractWaveform(uri: string, bins: number): Promise<number[]>;
|
||||
analyzeTrack(uri: string, bins: number, withLoudness: boolean): Promise<TrackAnalysis>;
|
||||
/**
|
||||
* Stop an in-flight (or still-queued) `analyzeTrack` for this URI so a skipped-past
|
||||
* track stops burning CPU. Safe to call when nothing is running.
|
||||
*/
|
||||
cancelAnalysis(uri: string): Promise<void>;
|
||||
/**
|
||||
* Decode short windows across the file and return approximate RMS peaks for
|
||||
* immediate seek-bar paint. Cheap preview only; callers should not persist it.
|
||||
*/
|
||||
extractWaveformPreview(uri: string, bins: number): Promise<number[]>;
|
||||
/**
|
||||
* Fast integrated loudness (M4): decodes only a few short windows across the
|
||||
* track + gated K-weighting -> integrated LUFS + absolute sample peak. Null on
|
||||
* failure / unmeasurable audio. Waveform peaks are separate (extractWaveform).
|
||||
*/
|
||||
measureLoudness(uri: string): Promise<{ lufs: number | null; peak: number | null }>;
|
||||
/**
|
||||
* Read ReplayGain track/album gain (dB) + peak (linear) from container tags
|
||||
* (ID3 TXXX / Vorbis comments / MP4 freeform) without decoding audio. All fields
|
||||
|
||||
Reference in New Issue
Block a user