diff --git a/modules/astra-library-scanner/android/src/androidTest/java/expo/modules/astralibraryscanner/data/LocalScanFastPathTest.kt b/modules/astra-library-scanner/android/src/androidTest/java/expo/modules/astralibraryscanner/data/LocalScanFastPathTest.kt new file mode 100644 index 0000000..cab6168 --- /dev/null +++ b/modules/astra-library-scanner/android/src/androidTest/java/expo/modules/astralibraryscanner/data/LocalScanFastPathTest.kt @@ -0,0 +1,148 @@ +package expo.modules.astralibraryscanner.data + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.filters.LargeTest +import java.util.concurrent.atomic.AtomicInteger +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotEquals +import org.junit.Assert.assertNotNull +import org.junit.Test +import org.junit.runner.RunWith + +@LargeTest +@RunWith(AndroidJUnit4::class) +class LocalScanFastPathTest { + @Test + fun unchangedIncrementalScanKeepsRevisionWhileChangesAndFullScansPublish() = runBlocking { + val context = ApplicationProvider.getApplicationContext() + val repository = AstraLibraryRepository.get(context) + val suffix = System.nanoTime().toString() + val treeUri = + "content://com.android.externalstorage.documents/tree/primary%3AMusic%2Fscan-$suffix" + val parentUri = + "$treeUri/document/primary%3AMusic%2Fscan-$suffix" + val trackUri = "$parentUri%2Fsong.flac" + val otherTrackUri = "$parentUri%2Fother.flac" + val folder = repository.registerFolder(treeUri, "Fast scan test") + val folderId = (folder.getValue("id") as Number).toLong() + val extractionCount = AtomicInteger() + val catalogEventCount = AtomicInteger() + val catalogListener: (Long) -> Unit = { catalogEventCount.incrementAndGet() } + var files = listOf( + file(trackUri, parentUri, size = 1_024, mtime = 10), + file(otherTrackUri, parentUri, size = 2_048, mtime = 20), + ) + + repository.addCatalogListener(catalogListener) + try { + val initial = repository.scanLocalFolder( + folderId = folderId, + full = false, + discover = { files }, + extract = { + extractionCount.incrementAndGet() + metadata("Initial title") + }, + onProgress = { _, _, _, _ -> }, + ) + assertEquals(2, initial.added) + assertEquals(2, extractionCount.get()) + assertEquals(1, catalogEventCount.get()) + + catalogEventCount.set(0) + val unchanged = repository.scanLocalFolder( + folderId = folderId, + full = false, + discover = { files.reversed() }, + extract = { + extractionCount.incrementAndGet() + metadata("Should not be extracted") + }, + onProgress = { _, _, _, _ -> }, + ) + assertEquals(initial.revision, unchanged.revision) + assertEquals(0, unchanged.added) + assertEquals(0, unchanged.updated) + assertEquals(0, unchanged.removed) + assertEquals(2, unchanged.total) + assertEquals(2, extractionCount.get()) + assertEquals(0, catalogEventCount.get()) + assertNotNull(repository.getTrack(trackUri)) + val readyFolder = repository.listFolders() + .single { (it.getValue("id") as Number).toLong() == folderId } + assertEquals("ready", readyFolder["scan_status"]) + assertNotNull(readyFolder["last_scanned_at"]) + + files = files.map { discovered -> + if (discovered.uri == trackUri) discovered.copy(lastModified = 11) else discovered + } + catalogEventCount.set(0) + val changed = repository.scanLocalFolder( + folderId = folderId, + full = false, + discover = { files }, + extract = { + extractionCount.incrementAndGet() + metadata("Changed title") + }, + onProgress = { _, _, _, _ -> }, + ) + assertNotEquals(unchanged.revision, changed.revision) + assertEquals(0, changed.added) + assertEquals(1, changed.updated) + assertEquals(3, extractionCount.get()) + assertEquals(1, catalogEventCount.get()) + + catalogEventCount.set(0) + val rebuilt = repository.scanLocalFolder( + folderId = folderId, + full = true, + discover = { files }, + extract = { + extractionCount.incrementAndGet() + metadata("Rebuilt title") + }, + onProgress = { _, _, _, _ -> }, + ) + assertNotEquals(changed.revision, rebuilt.revision) + assertEquals(2, rebuilt.updated) + assertEquals(5, extractionCount.get()) + assertEquals(1, catalogEventCount.get()) + assertEquals("Rebuilt title", repository.getTrack(trackUri)?.get("title")) + } finally { + repository.removeCatalogListener(catalogListener) + repository.removeFolder(folderId) + } + } + + private fun file( + uri: String, + parentUri: String, + size: Long?, + mtime: Long, + ): LocalAudioFile = LocalAudioFile( + uri = uri, + name = "song.flac", + size = size, + lastModified = mtime, + mimeType = "audio/flac", + parentUri = parentUri, + coverUri = null, + ) + + private fun metadata(title: String): LocalAudioMetadata = LocalAudioMetadata( + ok = true, + title = title, + artist = "Artist", + album = "Album", + mimeType = "audio/flac", + durationMs = 180_000, + sampleRate = 44_100, + channels = 2, + bitsPerSample = 16, + codecMime = "audio/flac", + ) +} diff --git a/modules/astra-library-scanner/android/src/main/java/expo/modules/astralibraryscanner/data/AstraLibraryRepository.kt b/modules/astra-library-scanner/android/src/main/java/expo/modules/astralibraryscanner/data/AstraLibraryRepository.kt index e33da65..ee8b44a 100644 --- a/modules/astra-library-scanner/android/src/main/java/expo/modules/astralibraryscanner/data/AstraLibraryRepository.kt +++ b/modules/astra-library-scanner/android/src/main/java/expo/modules/astralibraryscanner/data/AstraLibraryRepository.kt @@ -1,10 +1,13 @@ package expo.modules.astralibraryscanner.data import android.content.Context +import android.content.pm.ApplicationInfo import android.database.sqlite.SQLiteDatabaseCorruptException import android.database.sqlite.SQLiteException import android.os.Build +import android.os.SystemClock import android.os.Trace +import android.util.Log import androidx.room.Room import androidx.room.RoomDatabase import expo.modules.astralibraryscanner.queue.QueueReorder @@ -44,6 +47,51 @@ internal const val PLAYBACK_WINDOW_SIZE = PLAYBACK_HISTORY_WINDOW + 1 + PLAYBACK_UPCOMING_WINDOW private val traceCookie = AtomicInteger() +private const val SCAN_LOG_TAG = "AstraLibraryScan" + +private class LocalScanTiming( + private val folderId: Long, +) { + private val totalStartedNanos = SystemClock.elapsedRealtimeNanos() + private var lockStartedNanos = totalStartedNanos + + var lockWaitMs: Long = 0 + var discoveryMs: Long = 0 + var comparisonMs: Long = 0 + var extractionAndStagingMs: Long = 0 + var readModelMs: Long = 0 + var publishMs: Long = 0 + var files: Int = 0 + var outcome: String = "failed" + + fun beginLockWait() { + lockStartedNanos = SystemClock.elapsedRealtimeNanos() + } + + fun acquiredLock() { + lockWaitMs = elapsedMs(lockStartedNanos) + } + + fun now(): Long = SystemClock.elapsedRealtimeNanos() + + fun elapsedMs(startedNanos: Long): Long = + (SystemClock.elapsedRealtimeNanos() - startedNanos) / 1_000_000L + + fun logIfDebuggable(context: Context) { + val debuggable = + context.applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE != 0 + if (!debuggable) return + Log.d( + SCAN_LOG_TAG, + "folderId=$folderId outcome=$outcome files=$files" + + " lockWaitMs=$lockWaitMs discoveryMs=$discoveryMs" + + " comparisonMs=$comparisonMs extractionAndStagingMs=$extractionAndStagingMs" + + " readModelMs=$readModelMs publishMs=$publishMs" + + " totalMs=${elapsedMs(totalStartedNanos)}", + ) + } +} + private suspend fun traceAsyncSection( name: String, block: suspend () -> T, @@ -70,6 +118,35 @@ private fun throwIfScanCancelled(isCancelled: () -> Boolean) { if (isCancelled()) throw ScanCancelledException() } +internal fun canReuseActiveLocalGeneration( + full: Boolean, + previousSource: CatalogSourceEntity?, + files: List, + existingByPath: Map, +): Boolean { + if ( + full || + previousSource?.activeGenerationId == null || + previousSource.artistCreditVersion < CURRENT_ARTIST_CREDIT_VERSION || + files.size != existingByPath.size + ) { + return false + } + + val seen = HashSet(files.size) + return files.all { file -> + if (!seen.add(file.uri)) return@all false + val existing = existingByPath[file.uri] ?: return@all false + existing.mtime == file.lastModified && existing.size == file.size + } +} + +private data class StagedLocalTrack( + val row: TrackEntity?, + val failed: Boolean, + val metadataChanged: Boolean, +) + private data class RemoteSyncHandle( val syncId: String, val sourceKey: String, @@ -395,8 +472,11 @@ class AstraLibraryRepository private constructor( onProgress: (phase: String, processed: Int, total: Int, folderName: String) -> Unit, isCancelled: () -> Boolean = { false }, ): NativeScanResult { + val timing = LocalScanTiming(folderId) initialize() + timing.beginLockWait() return catalogWriterMutex.withLock { + timing.acquiredLock() val userDao = requireUser().userDao() val folder = userDao.getFolder(folderId) ?: error("Folder $folderId does not exist") val database = requireCatalog() @@ -417,28 +497,58 @@ class AstraLibraryRepository private constructor( updatedAt = startedAt, ), ) - dao.insertGeneration( - ScanGenerationEntity( - id = generationId, - sourceKey = sourceKey, - state = "staging", - startedAt = startedAt, - ), - ) userDao.updateFolderScanState(folderId, folder.lastScannedAt, "scanning", null) updateOperationalStatus(LibraryStatus.SCANNING) try { throwIfScanCancelled(isCancelled) onProgress("discovering", 0, 0, folder.displayName) + val discoveryStarted = timing.now() val files = discover(folder.treeUri) + timing.discoveryMs = timing.elapsedMs(discoveryStarted) + timing.files = files.size throwIfScanCancelled(isCancelled) onProgress("discovering", files.size, files.size, folder.displayName) + val comparisonStarted = timing.now() val existing = dao.getActiveTrackEntitiesForSource(sourceKey) val existingByPath = existing.associateBy(TrackEntity::path) val seenPaths = files.mapTo(hashSetOf(), LocalAudioFile::uri) val removed = existing.count { it.path !in seenPaths } + val canReuse = canReuseActiveLocalGeneration( + full = full, + previousSource = previousSource, + files = files, + existingByPath = existingByPath, + ) + timing.comparisonMs = timing.elapsedMs(comparisonStarted) + throwIfScanCancelled(isCancelled) + + if (canReuse) { + val revision = dao.getRevision() + userDao.updateFolderScanState(folderId, System.currentTimeMillis(), "ready", null) + scheduleSnapshot() + refreshReadyStatus() + timing.outcome = "unchanged" + return@withLock NativeScanResult( + added = 0, + updated = 0, + removed = 0, + errors = 0, + total = files.size, + revision = revision, + ) + } + + val extractionAndStagingStarted = timing.now() + dao.insertGeneration( + ScanGenerationEntity( + id = generationId, + sourceKey = sourceKey, + state = "staging", + startedAt = startedAt, + ), + ) var added = 0 var updated = 0 var errors = 0 @@ -456,34 +566,42 @@ class AstraLibraryRepository private constructor( old.mtime == file.lastModified && old.size == file.size if (unchanged) { - old!!.copy( - id = 0, - generationId = generationId, - sourceKey = sourceKey, - titleSortKey = SortKeys.forText(old.title), - artistSortKey = SortKeys.forText(old.artist), - albumSortKey = SortKeys.forText(old.album), - fileNameSortKey = SortKeys.forText(old.fileName), - sectionLabel = SortKeys.sectionLabel(old.title), - ) to false + StagedLocalTrack( + row = old!!.copy( + id = 0, + generationId = generationId, + sourceKey = sourceKey, + titleSortKey = SortKeys.forText(old.title), + artistSortKey = SortKeys.forText(old.artist), + albumSortKey = SortKeys.forText(old.album), + fileNameSortKey = SortKeys.forText(old.fileName), + sectionLabel = SortKeys.sectionLabel(old.title), + ), + failed = false, + metadataChanged = false, + ) } else { val metadata = extract(file) throwIfScanCancelled(isCancelled) if (!metadata.ok) { - if (old != null) { - old.copy(id = 0, generationId = generationId, sourceKey = sourceKey) to true - } else { - null to true - } + StagedLocalTrack( + row = old?.copy(id = 0, generationId = generationId, sourceKey = sourceKey), + failed = true, + metadataChanged = false, + ) } else { - trackFromMetadata( - generationId = generationId, - sourceKey = sourceKey, - folderId = folderId, - file = file, - metadata = metadata, - addedAt = old?.addedAt ?: startedAt, - ) to false + StagedLocalTrack( + row = trackFromMetadata( + generationId = generationId, + sourceKey = sourceKey, + folderId = folderId, + file = file, + metadata = metadata, + addedAt = old?.addedAt ?: startedAt, + ), + failed = false, + metadataChanged = true, + ) } } } @@ -491,21 +609,25 @@ class AstraLibraryRepository private constructor( } throwIfScanCancelled(isCancelled) val insertRows = ArrayList(rows.size) - for ((row, failed) in rows) { - if (failed) errors += 1 - if (row == null) continue + for (staged in rows) { + if (staged.failed) errors += 1 + val row = staged.row ?: continue insertRows += row - if (existingByPath.containsKey(row.path)) updated += if (failed) 0 else 1 else added += 1 + if (staged.metadataChanged) { + if (existingByPath.containsKey(row.path)) updated += 1 else added += 1 + } } if (insertRows.isNotEmpty()) dao.putTracks(insertRows) processed += batch.size onProgress("extracting", processed, files.size, folder.displayName) } + timing.extractionAndStagingMs = timing.elapsedMs(extractionAndStagingStarted) throwIfScanCancelled(isCancelled) val prospective = dao.getProspectiveTracks(sourceKey, generationId) val nextRevision = dao.getRevision() + 1 onProgress("indexing", prospective.size, prospective.size, folder.displayName) + val readModelStarted = timing.now() val readModels = withContext(Dispatchers.Default) { CatalogReadModelBuilder.build( prospective, @@ -513,9 +635,11 @@ class AstraLibraryRepository private constructor( userDao.getFolders().associateBy(FolderEntity::id), ) } + timing.readModelMs = timing.elapsedMs(readModelStarted) // Publishing is one Room transaction. Honour cancellation immediately before // it starts; once inside, let it finish so the active catalog stays coherent. throwIfScanCancelled(isCancelled) + val publishStarted = timing.now() val revision = dao.publishGeneration( sourceKey = sourceKey, generationId = generationId, @@ -533,6 +657,8 @@ class AstraLibraryRepository private constructor( scheduleSnapshot() refreshReadyStatus() for (listener in catalogListeners) listener(revision) + timing.publishMs = timing.elapsedMs(publishStarted) + timing.outcome = "published" NativeScanResult( added = added, updated = updated, @@ -542,6 +668,7 @@ class AstraLibraryRepository private constructor( revision = revision, ) } catch (_: ScanCancelledException) { + timing.outcome = "cancelled" dao.deleteGenerationTracks(generationId) dao.deleteGeneration(generationId) userDao.updateFolderScanState( @@ -562,6 +689,7 @@ class AstraLibraryRepository private constructor( cancelled = true, ) } catch (error: Throwable) { + timing.outcome = "failed" runCatching { dao.deleteGenerationTracks(generationId) dao.setGenerationState( @@ -586,8 +714,11 @@ class AstraLibraryRepository private constructor( recoveryNotice = currentStatus.recoveryNotice, ) scheduleSnapshot() + timing.logIfDebuggable(applicationContext) throw error } + }.also { + timing.logIfDebuggable(applicationContext) } } diff --git a/modules/astra-library-scanner/android/src/test/java/expo/modules/astralibraryscanner/data/LocalScanSnapshotTest.kt b/modules/astra-library-scanner/android/src/test/java/expo/modules/astralibraryscanner/data/LocalScanSnapshotTest.kt new file mode 100644 index 0000000..773291f --- /dev/null +++ b/modules/astra-library-scanner/android/src/test/java/expo/modules/astralibraryscanner/data/LocalScanSnapshotTest.kt @@ -0,0 +1,175 @@ +package expo.modules.astralibraryscanner.data + +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class LocalScanSnapshotTest { + @Test + fun identicalSnapshotCanReuseActiveGenerationRegardlessOfDiscoveryOrder() { + val existing = listOf( + track("content://music/a.flac", size = 100, mtime = 10), + track("content://music/b.flac", size = 200, mtime = 20), + ).associateBy(TrackEntity::path) + val files = listOf( + file("content://music/b.flac", size = 200, mtime = 20), + file("content://music/a.flac", size = 100, mtime = 10), + ) + + assertTrue(canReuseActiveLocalGeneration(false, activeSource(), files, existing)) + } + + @Test + fun additionsAndRemovalsRequirePublishing() { + val existing = listOf( + track("content://music/a.flac", size = 100, mtime = 10), + track("content://music/b.flac", size = 200, mtime = 20), + ).associateBy(TrackEntity::path) + + assertFalse( + canReuseActiveLocalGeneration( + false, + activeSource(), + listOf(file("content://music/a.flac", size = 100, mtime = 10)), + existing, + ), + ) + assertFalse( + canReuseActiveLocalGeneration( + false, + activeSource(), + listOf( + file("content://music/a.flac", size = 100, mtime = 10), + file("content://music/b.flac", size = 200, mtime = 20), + file("content://music/c.flac", size = 300, mtime = 30), + ), + existing, + ), + ) + } + + @Test + fun sizeOrModificationChangesRequirePublishing() { + val path = "content://music/a.flac" + val existing = mapOf(path to track(path, size = 100, mtime = 10)) + + assertFalse( + canReuseActiveLocalGeneration( + false, + activeSource(), + listOf(file(path, size = 101, mtime = 10)), + existing, + ), + ) + assertFalse( + canReuseActiveLocalGeneration( + false, + activeSource(), + listOf(file(path, size = 100, mtime = 11)), + existing, + ), + ) + } + + @Test + fun matchingNullableSizesCanReuseActiveGeneration() { + val path = "content://music/unknown-size.flac" + val existing = mapOf(path to track(path, size = null, mtime = 0)) + + assertTrue( + canReuseActiveLocalGeneration( + false, + activeSource(), + listOf(file(path, size = null, mtime = 0)), + existing, + ), + ) + } + + @Test + fun duplicateDiscoveredUrisRequirePublishing() { + val first = "content://music/a.flac" + val second = "content://music/b.flac" + val existing = listOf( + track(first, size = 100, mtime = 10), + track(second, size = 100, mtime = 10), + ).associateBy(TrackEntity::path) + + assertFalse( + canReuseActiveLocalGeneration( + false, + activeSource(), + listOf(file(first, size = 100, mtime = 10), file(first, size = 100, mtime = 10)), + existing, + ), + ) + } + + @Test + fun fullMissingAndStaleGenerationsCannotBeReused() { + val path = "content://music/a.flac" + val files = listOf(file(path, size = 100, mtime = 10)) + val existing = mapOf(path to track(path, size = 100, mtime = 10)) + + assertFalse(canReuseActiveLocalGeneration(true, activeSource(), files, existing)) + assertFalse( + canReuseActiveLocalGeneration( + false, + activeSource().copy(activeGenerationId = null), + files, + existing, + ), + ) + assertFalse( + canReuseActiveLocalGeneration( + false, + activeSource().copy(artistCreditVersion = LEGACY_ARTIST_CREDIT_VERSION), + files, + existing, + ), + ) + } + + private fun activeSource(): CatalogSourceEntity = CatalogSourceEntity( + sourceKey = "local:1", + sourceType = "local", + sourceId = 1, + activeGenerationId = "active", + updatedAt = 1, + artistCreditVersion = CURRENT_ARTIST_CREDIT_VERSION, + ) + + private fun file(path: String, size: Long?, mtime: Long): LocalAudioFile = LocalAudioFile( + uri = path, + name = path.substringAfterLast('/'), + size = size, + lastModified = mtime, + mimeType = "audio/flac", + parentUri = "content://music", + coverUri = null, + ) + + private fun track(path: String, size: Long?, mtime: Long): TrackEntity = TrackEntity( + generationId = "active", + sourceKey = "local:1", + path = path, + folderId = 1, + title = path.substringAfterLast('/'), + artist = "Artist", + album = "Album", + albumIdentityKey = "album", + format = "FLAC", + fileName = path.substringAfterLast('/'), + size = size, + mtime = mtime, + addedAt = 1, + modifiedAt = mtime, + titleSortKey = "title", + artistSortKey = "artist", + albumSortKey = "album", + fileNameSortKey = "file", + discSort = 0, + trackSort = 0, + sectionLabel = "A", + ) +}