From 639e7cf7f0cd94b071ea2b31fd93f24f35bb8e5e Mon Sep 17 00:00:00 2001 From: phobos665 Date: Wed, 15 Jul 2026 12:54:02 +0100 Subject: [PATCH 1/7] fix(): performance enhancements for fetching/sorting/filtering and rendering steam apps. More to come. --- .../app/gamenative/db/dao/SteamLicenseDao.kt | 4 + .../app/gamenative/service/SteamService.kt | 11 +- .../gamenative/ui/model/LibraryViewModel.kt | 243 ++++++++++++------ 3 files changed, 171 insertions(+), 87 deletions(-) diff --git a/app/src/main/java/app/gamenative/db/dao/SteamLicenseDao.kt b/app/src/main/java/app/gamenative/db/dao/SteamLicenseDao.kt index dc921fd7dc..6923aa2e95 100644 --- a/app/src/main/java/app/gamenative/db/dao/SteamLicenseDao.kt +++ b/app/src/main/java/app/gamenative/db/dao/SteamLicenseDao.kt @@ -7,6 +7,7 @@ import androidx.room.Query import androidx.room.Transaction import androidx.room.Update import app.gamenative.data.SteamLicense +import app.gamenative.data.SteamLicenseForPics import kotlin.math.min val SQLITE_MAX_VARS = 999 @@ -29,6 +30,9 @@ interface SteamLicenseDao { @Query("SELECT * FROM steam_license") suspend fun getAllLicenses(): List + @Query("SELECT packageId, access_token FROM steam_license") + suspend fun getLicensesForPics(): List + @Query("SELECT * FROM steam_license WHERE packageId = :packageId") suspend fun findLicense(packageId: Int): SteamLicense? diff --git a/app/src/main/java/app/gamenative/service/SteamService.kt b/app/src/main/java/app/gamenative/service/SteamService.kt index 7852b5b90a..a5a2f53581 100644 --- a/app/src/main/java/app/gamenative/service/SteamService.kt +++ b/app/src/main/java/app/gamenative/service/SteamService.kt @@ -27,6 +27,7 @@ import app.gamenative.data.GameSource import app.gamenative.data.LaunchInfo import app.gamenative.data.OwnedGames import app.gamenative.data.PostSyncInfo +import app.gamenative.data.SteamLicenseForPics import app.gamenative.data.SteamApp import app.gamenative.data.SteamControllerConfigDetail import app.gamenative.data.SteamFriend @@ -4137,7 +4138,7 @@ class SteamService : Service(), IChallengeUrlChanged { // Chunk the input to reduce memory pressures for very large items. licenses = callback.licenseList cachedLicenseDao.deleteAll() - callback.licenseList.chunked(500).forEach { chunk -> + callback.licenseList.chunked(MAX_PICS_BUFFER).forEach { chunk -> cachedLicenseDao.insertAll( chunk.map { license -> CachedLicense(licenseJson = LicenseSerializer.serializeLicense(license)) @@ -4176,22 +4177,26 @@ class SteamService : Service(), IChallengeUrlChanged { if (licensesToAdd.isNotEmpty()) { Timber.i("Adding ${licensesToAdd.size} licenses") - licensesToAdd.chunked(500).forEach { chunk -> + licensesToAdd.chunked(MAX_PICS_BUFFER).forEach { chunk -> licenseDao.insertAll(chunk) } } + Timber.i("Finished adding ${licensesToAdd.size} licenses") + val licensesToRemove = licenseDao.findStaleLicences( packageIds = callback.licenseList.map { it.packageID }, ) + if (licensesToRemove.isNotEmpty()) { Timber.i("Removing ${licensesToRemove.size} (stale) licenses") val packageIds = licensesToRemove.map { it.packageId } licenseDao.deleteStaleLicenses(packageIds) } + Timber.i("Getting licenses from DB") // Get PICS information with the current license database. - licenseDao.getAllLicenses() + licenseDao.getLicensesForPics() .map { PICSRequest(it.packageId, it.accessToken) } .chunked(MAX_PICS_BUFFER) .forEach { chunk -> diff --git a/app/src/main/java/app/gamenative/ui/model/LibraryViewModel.kt b/app/src/main/java/app/gamenative/ui/model/LibraryViewModel.kt index 884332a57d..c621a90f47 100644 --- a/app/src/main/java/app/gamenative/ui/model/LibraryViewModel.kt +++ b/app/src/main/java/app/gamenative/ui/model/LibraryViewModel.kt @@ -59,6 +59,7 @@ import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.qualifiers.ApplicationContext import java.io.File import java.util.EnumSet +import java.util.concurrent.ConcurrentHashMap import javax.inject.Inject import kotlin.math.max import kotlin.math.min @@ -69,6 +70,7 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.ensureActive import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.flatMapLatest import kotlinx.coroutines.flow.map @@ -113,6 +115,20 @@ class LibraryViewModel @Inject constructor( @Volatile private var paginationCurrentPage: Int = 0 @Volatile private var lastPageInCurrentFilter: Int = 0 + + // Quick cache to check if we need to refresh + @Volatile private var cachedFilteredLibrary: List? = null + + // in-flight filter job to avoid duplicate operations + private var filterJob: Job? = null + + // in-flight pagination slice job; cancelled by newer page changes and by full rebuilds + // so a slow slice can't overwrite the UI with a stale page or pre-rebuild data + private var pageJob: Job? = null + + // Check size of cache to avoid duplication + private val steamSizeCache = ConcurrentHashMap() + // Complete and unfiltered app list private var appList: List = emptyList() private var gogGameList: List = emptyList() @@ -192,6 +208,7 @@ class LibraryViewModel @Inject constructor( // Check if the list has actually changed before triggering a re-filter if (appList != apps) { appList = apps + steamSizeCache.clear() onFilterApps(paginationCurrentPage) } } @@ -446,7 +463,86 @@ class LibraryViewModel @Inject constructor( // Amount to change by var toPage = max(0, paginationCurrentPage + pageIncrement) toPage = min(toPage, lastPageInCurrentFilter) - onFilterApps(toPage) + + // Pagination only reveals more of the already filtered + sorted list, so slice the + // cached result instead of re-running the whole filter pipeline. + val cached = cachedFilteredLibrary + if (cached == null) { + onFilterApps(toPage) + return + } + if (toPage == paginationCurrentPage) { + return + } + paginationCurrentPage = toPage + synchronized(this) { + pageJob?.cancel() + pageJob = viewModelScope.launch(Dispatchers.Default) { + val pagedList = applyPagination(cached, toPage, _state.value) + // Don't commit if a newer page change or a full rebuild superseded this job + ensureActive() + fetchCompatibilityForPage(pagedList.map { it.name }) + _state.update { + it.copy( + appInfoList = pagedList, + currentPaginationPage = toPage + 1, // visual display is not 0 indexed + ) + } + } + } + } + + /** + * Slices [combined] up to and including [page], prepending the recommendation item when + * applicable. Shared by the full filter pipeline and the pagination fast path. + */ + private fun applyPagination(combined: List, page: Int, currentState: LibraryState): List { + val pageSize = PrefManager.itemsPerPage + val endIndex = min((page + 1) * pageSize, combined.size) + var pagedList = combined.take(endIndex) + + // Prepend the hero (featured > recommendation) as first item on ALL tab when + // enabled and not searching. + val featured = cachedFeatured + val rec = cachedRecommendation + if (PrefManager.showRecommendations + && currentState.currentTab == LibraryTab.ALL + && currentState.searchQuery.isEmpty() + ) { + val heroItem = when { + featured != null -> LibraryItem( + index = -1, + appId = "FEATURED_${featured.campaignId}", + name = featured.title, + heroImageUrl = featured.heroImageUrl, + headerImageUrl = featured.heroImageUrl, + capsuleImageUrl = featured.capsuleImageUrl ?: featured.heroImageUrl, + iconHash = featured.iconUrl ?: featured.capsuleImageUrl ?: featured.heroImageUrl, + isRecommended = true, + isFeatured = true, + recommendedGameId = featured.campaignId, + recSource = "hero", + gameSource = GameSource.STEAM, + ) + rec != null -> LibraryItem( + index = -1, + appId = "RECOMMENDED_${rec.id}", + name = rec.name, + heroImageUrl = rec.heroImageUrl, + capsuleImageUrl = rec.capsuleImageUrl, + iconHash = rec.iconUrl ?: rec.capsuleImageUrl, + isRecommended = true, + recommendedGameId = rec.id, + recSource = "hero", + gameSource = GameSource.STEAM, + ) + else -> null + } + if (heroItem != null) { + pagedList = listOf(heroItem) + pagedList.map { it.copy(index = it.index + 1) } + } + } + return pagedList } fun onRefresh() { @@ -457,6 +553,7 @@ class LibraryViewModel @Inject constructor( GameCompatibilityCache.clear() DeviceGameStatsCache.clear() GpuGameStatsCache.clear() + steamSizeCache.clear() try { val newApps = SteamService.refreshOwnedGamesFromServer() @@ -563,9 +660,14 @@ class LibraryViewModel @Inject constructor( return true } - private fun onFilterApps(paginationPage: Int = 0): Job { + private fun onFilterApps(paginationPage: Int = 0): Job = synchronized(this) { Timber.tag("LibraryViewModel").d("onFilterApps - appList.size: ${appList.size}, isFirstLoad: $isFirstLoad") - return viewModelScope.launch(Dispatchers.IO) { + // Invalidate cache and current running jobs, including any in-flight pagination + // slice that would otherwise commit results from the pre-rebuild list + cachedFilteredLibrary = null + filterJob?.cancel() + pageJob?.cancel() + viewModelScope.launch(Dispatchers.IO) { _state.update { it.copy(isLoading = true) } val currentState = _state.value @@ -655,35 +757,42 @@ class LibraryViewModel @Inject constructor( .asSequence() .filter { item -> passesCompatibleFilter(item.name) } .filter { item -> passesStatsFilters(currentState, GameSource.STEAM, item.name) } - .sortedWith( - compareByDescending { - downloadDirectorySet.contains(SteamService.getAppDirName(it)) - }.thenBy { it.name.lowercase() }, - ) .toList() // Map Steam apps to UI items - data class LibraryEntry(val item: LibraryItem, val isInstalled: Boolean, val lastPlayed: Long = 0L) + data class LibraryEntry(val item: LibraryItem, val isInstalled: Boolean, val lastPlayed: Long = 0L) { + // Precomputed once so sort comparators don't allocate a lowercase copy per comparison + val sortName: String = item.name.lowercase() + } fun lastPlayedFor(appId: String): Long = playHistoryByAppId[appId] ?: 0L val licensedDepotMap = SteamService.buildLicensedDepotMap(filteredSteamApps) + // Single query for installed branches instead of one blocking lookup per installed app + val installedBranchByAppId = SteamService.getAllInstalledApps() + .orEmpty() + .associate { it.id to it.branch } + + ensureActive() + // Added this to avoid duplicate from custom imported steam game val steamEntriesAppIds = mutableSetOf() val steamEntries: List = filteredSteamApps.map { item -> val isInstalled = downloadDirectorySet.contains(SteamService.getAppDirName(item)) val installedBranch = if (isInstalled) { - SteamService.getInstalledApp(item.id)?.branch ?: "public" + installedBranchByAppId[item.id] ?: "public" } else { "public" } // base-game size: ownedDlc=emptyMap excludes DLC depots - val licensedDepots = licensedDepotMap[item.id] - val resolved = SteamService.resolveDownloadableDepots(item.depots, "", emptyMap(), licensedDepots) - val totalSizeBytes = resolved.values.sumOf { depot -> - depot.manifests[installedBranch]?.size ?: depot.manifests.values.firstOrNull()?.size ?: 0L + val totalSizeBytes = steamSizeCache.getOrPut("${item.id}:$installedBranch") { + val licensedDepots = licensedDepotMap[item.id] + val resolved = SteamService.resolveDownloadableDepots(item.depots, "", emptyMap(), licensedDepots) + resolved.values.sumOf { depot -> + depot.manifests[installedBranch]?.size ?: depot.manifests.values.firstOrNull()?.size ?: 0L + } } // Move appId here @@ -874,6 +983,12 @@ class LibraryViewModel @Inject constructor( // ALL tab uses user preferences, other tabs override with their presets // Use captured currentState (not _state.value) to avoid TOCTOU race val currentTab = currentState.currentTab + + // Credential checks hit storage; do each once per run instead of per usage below + val hasGOGCredentials = GOGService.hasStoredCredentials(context) + val hasEpicCredentials = EpicService.hasStoredCredentials(context) + val hasAmazonCredentials = AmazonService.hasStoredCredentials(context) + val includeSteam = if (currentTab == app.gamenative.ui.enums.LibraryTab.ALL) { currentState.showSteamInLibrary } else { @@ -889,59 +1004,60 @@ class LibraryViewModel @Inject constructor( currentState.showGOGInLibrary } else { currentTab.showGoG - }) && GOGService.hasStoredCredentials(context) + }) && hasGOGCredentials val includeEpic = (if (currentTab == app.gamenative.ui.enums.LibraryTab.ALL) { currentState.showEpicInLibrary } else { currentTab.showEpic - }) && EpicService.hasStoredCredentials(context) + }) && hasEpicCredentials val includeAmazon = (if (currentTab == app.gamenative.ui.enums.LibraryTab.ALL) { currentState.showAmazonInLibrary } else { currentTab.showAmazon - }) && AmazonService.hasStoredCredentials(context) + }) && hasAmazonCredentials // Combine both lists and apply sort option val sortComparator: Comparator = when (currentState.currentSortOption) { SortOption.INSTALLED_FIRST -> compareBy { entry -> if (entry.isInstalled) 0 else 1 - }.thenBy { it.item.name.lowercase() } + }.thenBy { it.sortName } - SortOption.NAME_ASC -> compareBy { it.item.name.lowercase() } + SortOption.NAME_ASC -> compareBy { it.sortName } - SortOption.NAME_DESC -> compareByDescending { it.item.name.lowercase() } + SortOption.NAME_DESC -> compareByDescending { it.sortName } SortOption.RECENTLY_PLAYED -> LibrarySortUtils.recentlyPlayedComparator( - name = { it.item.name }, + name = { it.sortName }, isInstalled = { it.isInstalled }, lastPlayed = { it.lastPlayed }, ) SortOption.SIZE_SMALLEST -> compareBy { it.item.sizeBytes } - .thenBy { it.item.name.lowercase() } + .thenBy { it.sortName } SortOption.SIZE_LARGEST -> compareByDescending { it.item.sizeBytes } - .thenBy { it.item.name.lowercase() } + .thenBy { it.sortName } SortOption.FPS_HIGH -> compareByDescending { currentState.statsFor(it.item)?.fps ?: -1 - }.thenBy { it.item.name.lowercase() } + }.thenBy { it.sortName } SortOption.RUNS_HIGH -> compareByDescending { currentState.statsFor(it.item)?.runsGpu ?: -1 - }.thenBy { it.item.name.lowercase() } + }.thenBy { it.sortName } SortOption.REVIEWS_HIGH -> compareByDescending { currentState.statsFor(it.item)?.reviewsDevice ?: -1 - }.thenBy { it.item.name.lowercase() } + }.thenBy { it.sortName } SortOption.REVIEWS_GPU_HIGH -> compareByDescending { currentState.statsFor(it.item)?.reviewsGpu ?: -1 - }.thenBy { it.item.name.lowercase() } + }.thenBy { it.sortName } } + ensureActive() // A Steam collection can only contain Steam apps, so when one is selected the non-Steam // sources can't match it — keep them out of the combined list (and their tab counts). val steamCollectionSelected = allowedSteamAppIds != null @@ -958,57 +1074,8 @@ class LibraryViewModel @Inject constructor( // Total count for the current filter val totalFound = combined.size - - // Determine how many pages and slice the list for incremental loading val pageSize = PrefManager.itemsPerPage - // Update internal pagination state - paginationCurrentPage = paginationPage - lastPageInCurrentFilter = if (totalFound == 0) 0 else (totalFound - 1) / pageSize - // Calculate how many items to show: (pagesLoaded * pageSize) - val endIndex = min((paginationPage + 1) * pageSize, totalFound) - var pagedList = combined.take(endIndex) - - // Prepend the hero (featured > recommendation) as first item on ALL tab when - // enabled and not searching. - val featured = cachedFeatured - val rec = cachedRecommendation - if (PrefManager.showRecommendations - && currentTab == LibraryTab.ALL - && currentState.searchQuery.isEmpty() - ) { - val heroItem = when { - featured != null -> LibraryItem( - index = -1, - appId = "FEATURED_${featured.campaignId}", - name = featured.title, - heroImageUrl = featured.heroImageUrl, - headerImageUrl = featured.heroImageUrl, - capsuleImageUrl = featured.capsuleImageUrl ?: featured.heroImageUrl, - iconHash = featured.iconUrl ?: featured.capsuleImageUrl ?: featured.heroImageUrl, - isRecommended = true, - isFeatured = true, - recommendedGameId = featured.campaignId, - recSource = "hero", - gameSource = GameSource.STEAM, - ) - rec != null -> LibraryItem( - index = -1, - appId = "RECOMMENDED_${rec.id}", - name = rec.name, - heroImageUrl = rec.heroImageUrl, - capsuleImageUrl = rec.capsuleImageUrl, - iconHash = rec.iconUrl ?: rec.capsuleImageUrl, - isRecommended = true, - recommendedGameId = rec.id, - recSource = "hero", - gameSource = GameSource.STEAM, - ) - else -> null - } - if (heroItem != null) { - pagedList = listOf(heroItem) + pagedList.map { it.copy(index = it.index + 1) } - } - } + val pagedList = applyPagination(combined, paginationPage, currentState) Timber.tag("LibraryViewModel").d("Filtered list size (with Custom Games): $totalFound") @@ -1016,6 +1083,14 @@ class LibraryViewModel @Inject constructor( isFirstLoad = false } + // Don't commit results if a newer filter request has superseded this run + ensureActive() + + // Update internal pagination state + paginationCurrentPage = paginationPage + lastPageInCurrentFilter = if (totalFound == 0) 0 else (totalFound - 1) / pageSize + cachedFilteredLibrary = combined + // Fetch compatibility for current page games fetchCompatibilityForPage(pagedList.map { it.name }) @@ -1030,18 +1105,18 @@ class LibraryViewModel @Inject constructor( // Use user prefs + auth state only (not current tab) so badges stay stable across tab switches allCount = (if (currentState.showSteamInLibrary) steamEntries.size else 0) + (if (currentState.showCustomGamesInLibrary) customEntries.size else 0) + - (if (currentState.showGOGInLibrary && GOGService.hasStoredCredentials(context)) gogEntries.size else 0) + - (if (currentState.showEpicInLibrary && EpicService.hasStoredCredentials(context)) epicEntries.size else 0) + - (if (currentState.showAmazonInLibrary && AmazonService.hasStoredCredentials(context)) amazonEntries.size else 0), + (if (currentState.showGOGInLibrary && hasGOGCredentials) gogEntries.size else 0) + + (if (currentState.showEpicInLibrary && hasEpicCredentials) epicEntries.size else 0) + + (if (currentState.showAmazonInLibrary && hasAmazonCredentials) amazonEntries.size else 0), steamCount = if (currentState.showSteamInLibrary) steamEntries.size else 0, - gogCount = if (currentState.showGOGInLibrary && GOGService.hasStoredCredentials(context)) gogEntries.size else 0, - epicCount = if (currentState.showEpicInLibrary && EpicService.hasStoredCredentials(context)) epicEntries.size else 0, - amazonCount = if (currentState.showAmazonInLibrary && AmazonService.hasStoredCredentials(context)) amazonEntries.size else 0, + gogCount = if (currentState.showGOGInLibrary && hasGOGCredentials) gogEntries.size else 0, + epicCount = if (currentState.showEpicInLibrary && hasEpicCredentials) epicEntries.size else 0, + amazonCount = if (currentState.showAmazonInLibrary && hasAmazonCredentials) amazonEntries.size else 0, localCount = if (currentState.showCustomGamesInLibrary) customEntries.size else 0, steamCollectionCounts = steamCollectionCounts, ) } - } + }.also { filterJob = it } } /** From 2365c43b71907dd6b9459b75c9735820903b292e Mon Sep 17 00:00:00 2001 From: phobos665 Date: Wed, 15 Jul 2026 10:13:55 +0100 Subject: [PATCH 2/7] fix(): fix issue with job timing out or erroring out and breaking pics jobs. --- .../app/gamenative/service/SteamService.kt | 52 +++++++++++++++---- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/app/src/main/java/app/gamenative/service/SteamService.kt b/app/src/main/java/app/gamenative/service/SteamService.kt index a5a2f53581..0b883efa4c 100644 --- a/app/src/main/java/app/gamenative/service/SteamService.kt +++ b/app/src/main/java/app/gamenative/service/SteamService.kt @@ -310,6 +310,10 @@ class SteamService : Service(), IChallengeUrlChanged { companion object { const val MAX_PICS_BUFFER = 256 + // Bulk PICS requests (large libraries) can take longer than AsyncJob's 10s default + // to get a first response; give them more headroom before treating them as timed out. + const val PICS_JOB_TIMEOUT_MS = 30_000L + const val MAX_RETRY_ATTEMPTS = 20 const val INVALID_APP_ID: Int = Int.MAX_VALUE @@ -4332,10 +4336,12 @@ class SteamService : Service(), IChallengeUrlChanged { val steamApps = instance?._steamApps ?: return@collect try { - val callback = steamApps.picsGetProductInfo( + val job = steamApps.picsGetProductInfo( apps = appRequests, packages = emptyList(), - ).await() + ) + job.timeout = PICS_JOB_TIMEOUT_MS + val callback = job.await() callback.results.forEachIndexed { index, picsCallback -> Timber.d( @@ -4384,6 +4390,13 @@ class SteamService : Service(), IChallengeUrlChanged { } } } + } catch (e: CancellationException) { + // A timed-out AsyncJobMultiple with zero results calls future.cancel() + // rather than failing the future, so this looks like coroutine + // cancellation. Only rethrow if this collector's own job was cancelled; + // otherwise skip the batch so later batches on the channel still run. + ensureActive() + Timber.w("PICS product info request timed out for ${appRequests.size} app(s); skipping batch") } catch (e: AsyncJobFailedException) { Timber.w("Could not get PICS product info $e") } @@ -4401,10 +4414,24 @@ class SteamService : Service(), IChallengeUrlChanged { if (!isLoggedIn) return@collect val steamApps = instance?._steamApps ?: return@collect - val callback = steamApps.picsGetProductInfo( - apps = emptyList(), - packages = packageRequests, - ).await() + val callback = try { + val job = steamApps.picsGetProductInfo( + apps = emptyList(), + packages = packageRequests, + ) + job.timeout = PICS_JOB_TIMEOUT_MS + job.await() + } catch (e: CancellationException) { + // See the app-channel collector above: a timed-out AsyncJobMultiple + // cancels its future instead of failing it, so only rethrow if this + // collector's own job was actually cancelled. + ensureActive() + Timber.w("PICS product info request timed out for ${packageRequests.size} package(s); skipping batch") + return@collect + } catch (e: AsyncJobFailedException) { + Timber.w("Could not get PICS product info for packages $e") + return@collect + } callback.results.forEach { picsCallback -> // Don't race the queue. @@ -4482,11 +4509,12 @@ class SteamService : Service(), IChallengeUrlChanged { } try { - // TODO: This could be an issue. (Stalling) - steamApps.picsGetAccessTokens( + val tokensJob = steamApps.picsGetAccessTokens( appIds = queue, packageIds = emptyList(), - ).await() + ) + tokensJob.timeout = PICS_JOB_TIMEOUT_MS + tokensJob.await() .appTokens .forEach { (key, value) -> appTokens[key] = value @@ -4500,6 +4528,12 @@ class SteamService : Service(), IChallengeUrlChanged { Timber.d("bufferedPICSGetProductInfo: Queueing ${chunk.size} for PICS") appPicsChannel.send(chunk) } + } catch (e: CancellationException) { + // See the app-channel collector above: a timed-out AsyncJobSingle + // cancels its future instead of failing it, so only rethrow if this + // collector's own job was actually cancelled. + ensureActive() + Timber.w("PICS access token request timed out for ${queue.size} app(s); skipping batch") } catch (e: AsyncJobFailedException) { Timber.w("Could not get PICS product info $e") } From 12d441b74fa69ce99659ea72eab5ea48ce9d69d2 Mon Sep 17 00:00:00 2001 From: phobos665 Date: Wed, 15 Jul 2026 18:30:43 +0100 Subject: [PATCH 3/7] fix(): Performance changes for loading and storing steamApps, licences as well as sorting/filtering etc. --- .../app/gamenative/data/SteamAppPicsMeta.kt | 17 +++ .../gamenative/data/SteamLicenseForPics.kt | 10 ++ .../java/app/gamenative/db/dao/SteamAppDao.kt | 55 +++++++- .../app/gamenative/service/SteamService.kt | 122 +++++++++++++----- 4 files changed, 174 insertions(+), 30 deletions(-) create mode 100644 app/src/main/java/app/gamenative/data/SteamAppPicsMeta.kt create mode 100644 app/src/main/java/app/gamenative/data/SteamLicenseForPics.kt diff --git a/app/src/main/java/app/gamenative/data/SteamAppPicsMeta.kt b/app/src/main/java/app/gamenative/data/SteamAppPicsMeta.kt new file mode 100644 index 0000000000..6da30b886f --- /dev/null +++ b/app/src/main/java/app/gamenative/data/SteamAppPicsMeta.kt @@ -0,0 +1,17 @@ +package app.gamenative.data + +import androidx.room.ColumnInfo + +// Slimmed-down projection of the columns the PICS collectors actually read when +// deciding whether an app needs (re)processing. Avoids pulling the full SteamApp +// row — including the large depots/config/UFS JSON blobs — into memory just to +// compare change numbers for tens of thousands of apps. +data class SteamAppPicsMeta( + val id: Int, + @ColumnInfo("package_id") + val packageId: Int, + @ColumnInfo("last_change_number") + val lastChangeNumber: Int, + @ColumnInfo("ufs_parse_version") + val ufsParseVersion: Int, +) diff --git a/app/src/main/java/app/gamenative/data/SteamLicenseForPics.kt b/app/src/main/java/app/gamenative/data/SteamLicenseForPics.kt new file mode 100644 index 0000000000..10fcb8f926 --- /dev/null +++ b/app/src/main/java/app/gamenative/data/SteamLicenseForPics.kt @@ -0,0 +1,10 @@ +package app.gamenative.data + +import androidx.room.ColumnInfo + +// Slimmed down type for retrieving licences for processing. +data class SteamLicenseForPics( + val packageId: Int, + @ColumnInfo("access_token") + val accessToken: Long, +) diff --git a/app/src/main/java/app/gamenative/db/dao/SteamAppDao.kt b/app/src/main/java/app/gamenative/db/dao/SteamAppDao.kt index ff9a1aff8a..877dc9ba25 100644 --- a/app/src/main/java/app/gamenative/db/dao/SteamAppDao.kt +++ b/app/src/main/java/app/gamenative/db/dao/SteamAppDao.kt @@ -7,7 +7,9 @@ import androidx.room.Query import androidx.room.Transaction import androidx.room.Update import app.gamenative.data.SteamApp +import app.gamenative.data.SteamAppPicsMeta import app.gamenative.service.SteamService.Companion.INVALID_PKG_ID +import kotlin.math.min import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.distinctUntilChanged @@ -199,5 +201,56 @@ interface SteamAppDao { suspend fun findSteamAppWithInstallDir(dirName: String): List @Query("SELECT * FROM steam_app WHERE id IN (:appIds)") - suspend fun findSteamAppWithAppIds(appIds: List): List + suspend fun _findSteamAppWithAppIds(appIds: List): List + + // batched to stay under SQLite's 999 bind-variable limit + @Transaction + suspend fun findSteamAppWithAppIds(appIds: List): List { + if (appIds.isEmpty()) return emptyList() + val results = mutableListOf() + for (chunkStart in appIds.indices step SQLITE_MAX_VARS) { + val chunkEnd = min(chunkStart + SQLITE_MAX_VARS, appIds.size) + results += _findSteamAppWithAppIds(appIds.subList(chunkStart, chunkEnd)) + } + return results + } + + @Query( + "SELECT id, package_id, last_change_number, ufs_parse_version " + + "FROM steam_app WHERE id IN (:appIds)", + ) + suspend fun _findAppPicsMeta(appIds: List): List + + @Query("UPDATE steam_app SET package_id = :packageId WHERE id IN (:appIds)") + suspend fun _updatePackageIdForApps(packageId: Int, appIds: List) + + /** + * Reassigns package ownership for a set of apps in one statement per chunk, + * instead of read-modify-writing each full SteamApp row. Batched to stay + * under SQLite's 999-parameter ceiling. + */ + @Transaction + suspend fun updatePackageIdForApps(packageId: Int, appIds: List) { + if (appIds.isEmpty()) return + for (chunkStart in appIds.indices step SQLITE_MAX_VARS) { + val chunkEnd = min(chunkStart + SQLITE_MAX_VARS, appIds.size) + _updatePackageIdForApps(packageId, appIds.subList(chunkStart, chunkEnd)) + } + } + + /** + * Lightweight projection of the columns the PICS collectors read to decide + * whether an app needs (re)processing. Batched to stay under SQLite's + * 999-parameter ceiling; avoids deserializing the full row/blobs per app. + */ + @Transaction + suspend fun findAppPicsMeta(appIds: List): List { + if (appIds.isEmpty()) return emptyList() + val results = mutableListOf() + for (chunkStart in appIds.indices step SQLITE_MAX_VARS) { + val chunkEnd = min(chunkStart + SQLITE_MAX_VARS, appIds.size) + results += _findAppPicsMeta(appIds.subList(chunkStart, chunkEnd)) + } + return results + } } diff --git a/app/src/main/java/app/gamenative/service/SteamService.kt b/app/src/main/java/app/gamenative/service/SteamService.kt index 0b883efa4c..a38caec31d 100644 --- a/app/src/main/java/app/gamenative/service/SteamService.kt +++ b/app/src/main/java/app/gamenative/service/SteamService.kt @@ -4351,10 +4351,30 @@ class SteamService : Service(), IChallengeUrlChanged { ) ensureActive() + + // Batch the DB reads that the per-app decision needs, instead of + // issuing two point queries (findApp + findLicense) per app. For a + // 30K-game library the per-app path was ~512 queries per 256-app + // batch, each findApp deserializing the full depots/config/UFS blob + // just to compare a change number. Here we do two batched reads: + // a slim metadata projection keyed by appId, and the licenses keyed + // by packageId, then decide everything in memory. + val appMetaById = appDao + .findAppPicsMeta(picsCallback.apps.values.map { it.id }) + .associateBy { it.id } + val licensesByPkgId = licenseDao + .findLicenses( + appMetaById.values + .map { it.packageId } + .filter { it != INVALID_PKG_ID } + .distinct(), + ) + .associateBy { it.packageId } + val steamAppsMap = picsCallback.apps.values.mapNotNull { app -> - val appFromDb = appDao.findApp(app.id) - val packageId = appFromDb?.packageId ?: INVALID_PKG_ID - val packageFromDb = if (packageId != INVALID_PKG_ID) licenseDao.findLicense(packageId) else null + val appMeta = appMetaById[app.id] + val packageId = appMeta?.packageId ?: INVALID_PKG_ID + val packageFromDb = if (packageId != INVALID_PKG_ID) licensesByPkgId[packageId] else null val ownerAccountId = packageFromDb?.ownerAccountId ?: emptyList() // Apps with -1 for the ownerAccountId should be added. @@ -4362,9 +4382,9 @@ class SteamService : Service(), IChallengeUrlChanged { // TODO maybe apps with -1 for the ownerAccountId can be stripped with necessities and name. - val ufsParseVersionOutdated = appFromDb != null && appFromDb.ufsParseVersion < CURRENT_UFS_PARSE_VERSION + val ufsParseVersionOutdated = appMeta != null && appMeta.ufsParseVersion < CURRENT_UFS_PARSE_VERSION - if (app.changeNumber != appFromDb?.lastChangeNumber || ufsParseVersionOutdated) { + if (app.changeNumber != appMeta?.lastChangeNumber || ufsParseVersionOutdated) { val newApp = app.keyValues.generateSteamApp().copy( packageId = packageId, ownerAccountId = ownerAccountId, @@ -4447,10 +4467,33 @@ class SteamService : Service(), IChallengeUrlChanged { // To fix that we (a) process user-owned packages last so they win the // last-write-wins assignment within this batch and (b) refuse to downgrade an // existing user-owned packageId across batches. + // + // For large libraries the per-app path used to issue a findApp (full-blob + // read) plus an insert/update per app, per package. Instead we now batch the + // reads (one slim metadata projection for every referenced app, one license + // lookup) and resolve the winning packageId per app in memory, then flush + // inserts/updates grouped by target package at the end of the batch. val accountId = userSteamId?.accountID?.toInt() + + // Every appId referenced by this callback's packages. + val referencedAppIds = picsCallback.packages.values + .flatMap { pkg -> pkg.keyValues["appids"].children.map { it.asInteger() } } + .distinct() + + // Existing packageId per app (absent => app not yet in the DB). + val originalPkgById: Map = appDao + .findAppPicsMeta(referencedAppIds) + .associate { it.id to it.packageId } + + // Licenses for the callback's packages AND for any package an existing app + // already points at (needed for the cross-batch downgrade guard below). val packageLicenses: Map = if (accountId != null) { - val packageIds = picsCallback.packages.values.map { it.id } - licenseDao.findLicenses(packageIds).associateBy { it.packageId } + val lookupIds = ( + picsCallback.packages.values.map { it.id } + originalPkgById.values + ) + .filter { it != INVALID_PKG_ID } + .distinct() + licenseDao.findLicenses(lookupIds).associateBy { it.packageId } } else { emptyMap() } @@ -4469,8 +4512,23 @@ class SteamService : Service(), IChallengeUrlChanged { return if (expired) 1 else 2 } + fun licenseRank(pkgId: Int): Int { + if (accountId == null || pkgId == INVALID_PKG_ID) return 0 + val license = packageLicenses[pkgId] + return when { + license == null -> 0 + !license.ownerAccountId.contains(accountId) -> 0 + ELicenseFlags.Expired in license.licenseFlags -> 1 + else -> 2 + } + } + val orderedPackages = picsCallback.packages.values.sortedBy { pkgRank(it.id) } + // Current winning packageId per app; seeded with the DB state and mutated + // in memory as packages are applied, so later packages see earlier wins. + val assignedPkgById = HashMap(originalPkgById) + orderedPackages.forEach { pkg -> val appIds = pkg.keyValues["appids"].children.map { it.asInteger() } licenseDao.updateApps(pkg.id, appIds) @@ -4478,34 +4536,40 @@ class SteamService : Service(), IChallengeUrlChanged { val depotIds = pkg.keyValues["depotids"].children.map { it.asInteger() } licenseDao.updateDepots(pkg.id, depotIds) - // Insert a stub row (or update) of SteamApps to the database. + // Resolve the winning packageId for each app (last-write-wins, but a + // user-owned assignment refuses to be downgraded by a lesser package). appIds.forEach { appid -> - val existing = appDao.findApp(appid) - if (existing == null) { - appDao.insert(SteamApp(id = appid, packageId = pkg.id)) - return@forEach + val current = assignedPkgById[appid] + when { + // Not in the DB and not yet assigned this batch: first writer wins. + current == null -> assignedPkgById[appid] = pkg.id + current == pkg.id -> Unit + licenseRank(current) > pkgRank(pkg.id) -> Unit + else -> assignedPkgById[appid] = pkg.id } - if (existing.packageId == pkg.id) { - return@forEach - } - if (accountId != null && existing.packageId != INVALID_PKG_ID) { - val existingLicense = packageLicenses[existing.packageId] - ?: licenseDao.findLicense(existing.packageId) - val existingRank = when { - existingLicense == null -> 0 - !existingLicense.ownerAccountId.contains(accountId) -> 0 - ELicenseFlags.Expired in existingLicense.licenseFlags -> 1 - else -> 2 - } - if (existingRank > pkgRank(pkg.id)) { - return@forEach - } - } - appDao.update(existing.copy(packageId = pkg.id)) } queue.addAll(appIds) } + + // Flush: brand-new apps become stub inserts; existing apps whose winning + // package changed get a batched package_id update grouped by target package. + val stubs = mutableListOf() + val updatesByPkg = HashMap>() + assignedPkgById.forEach { (appId, pkgId) -> + val original = originalPkgById[appId] + if (original == null) { + stubs += SteamApp(id = appId, packageId = pkgId) + } else if (original != pkgId) { + updatesByPkg.getOrPut(pkgId) { mutableListOf() }.add(appId) + } + } + if (stubs.isNotEmpty()) { + appDao.insertAll(stubs) + } + updatesByPkg.forEach { (pkgId, appIds) -> + appDao.updatePackageIdForApps(pkgId, appIds) + } } try { From 65be94b8d75f8c9a97c6e6938d996e3650299922 Mon Sep 17 00:00:00 2001 From: phobos665 Date: Wed, 15 Jul 2026 20:36:55 +0100 Subject: [PATCH 4/7] fix(): performance improvements: --- .../app/gamenative/data/SteamLicenseStub.kt | 10 + .../app/gamenative/db/dao/SteamLicenseDao.kt | 5 + .../java/app/gamenative/di/DatabaseModule.kt | 31 +++ .../app/gamenative/service/SteamService.kt | 188 ++++++++++++------ 4 files changed, 170 insertions(+), 64 deletions(-) create mode 100644 app/src/main/java/app/gamenative/data/SteamLicenseStub.kt diff --git a/app/src/main/java/app/gamenative/data/SteamLicenseStub.kt b/app/src/main/java/app/gamenative/data/SteamLicenseStub.kt new file mode 100644 index 0000000000..cecc1c4fc5 --- /dev/null +++ b/app/src/main/java/app/gamenative/data/SteamLicenseStub.kt @@ -0,0 +1,10 @@ +package app.gamenative.data + +import androidx.room.ColumnInfo + +// Smallest possible stub for steam license comparisons +data class SteamLicenseStub( + val packageId: Int, + @ColumnInfo("last_change_number") val lastChangeNumber: Int, + @ColumnInfo("access_token") val accessToken: Long, +) diff --git a/app/src/main/java/app/gamenative/db/dao/SteamLicenseDao.kt b/app/src/main/java/app/gamenative/db/dao/SteamLicenseDao.kt index 6923aa2e95..8dc8823229 100644 --- a/app/src/main/java/app/gamenative/db/dao/SteamLicenseDao.kt +++ b/app/src/main/java/app/gamenative/db/dao/SteamLicenseDao.kt @@ -8,6 +8,7 @@ import androidx.room.Transaction import androidx.room.Update import app.gamenative.data.SteamLicense import app.gamenative.data.SteamLicenseForPics +import app.gamenative.data.SteamLicenseStub import kotlin.math.min val SQLITE_MAX_VARS = 999 @@ -30,6 +31,10 @@ interface SteamLicenseDao { @Query("SELECT * FROM steam_license") suspend fun getAllLicenses(): List + //lightweight difffs for steamlicences + @Query("SELECT packageId, last_change_number, access_token FROM steam_license") + suspend fun getLicenseStubs(): List + @Query("SELECT packageId, access_token FROM steam_license") suspend fun getLicensesForPics(): List diff --git a/app/src/main/java/app/gamenative/di/DatabaseModule.kt b/app/src/main/java/app/gamenative/di/DatabaseModule.kt index 446eec3651..b32c1293fc 100644 --- a/app/src/main/java/app/gamenative/di/DatabaseModule.kt +++ b/app/src/main/java/app/gamenative/di/DatabaseModule.kt @@ -2,6 +2,8 @@ package app.gamenative.di import android.content.Context import androidx.room.Room +import androidx.room.RoomDatabase +import androidx.sqlite.db.SupportSQLiteDatabase import app.gamenative.db.DATABASE_NAME import app.gamenative.db.PluviaDatabase import app.gamenative.db.dao.AppInfoDao @@ -19,6 +21,7 @@ import dagger.Provides import dagger.hilt.InstallIn import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.components.SingletonComponent +import java.util.concurrent.Executors import javax.inject.Singleton @InstallIn(SingletonComponent::class) @@ -36,6 +39,34 @@ class DatabaseModule { ROOM_MIGRATION_V23_to_V24, ) .fallbackToDestructiveMigration(true) + // Use unbounded thread pools for reads/writes so a long-running query (e.g. the + // library COUNT/paged load during a large PICS sync) can't starve the default + // single-threaded executors and deadlock the connection pool. SQLite still + // serializes writers, so this only removes the scheduling deadlock. + .setQueryExecutor(Executors.newCachedThreadPool()) + .setTransactionExecutor(Executors.newCachedThreadPool()) + .addCallback(object : RoomDatabase.Callback() { + override fun onOpen(db: SupportSQLiteDatabase) { + // These indexes are not declared in @Entity (which would require a schema + // migration) — they're created here so we can add them without a version bump + // while keeping upstream compatibility. IF NOT EXISTS makes them idempotent. + // + // dlc_for_app_id: the OWNED_APPS_WHERE DLC EXISTS arm joins back into + // steam_app on this column. Without an index, SQLite scans all rows per + // outer row that fails the direct-license check, turning the COUNT(*) query + // O(N²) and causing 100+ second runtimes that exhaust the connection pool. + db.execSQL( + "CREATE INDEX IF NOT EXISTS idx_steam_app_dlc_for_app_id " + + "ON steam_app(dlc_for_app_id)", + ) + // package_id: used in the WHERE clause to exclude INVALID_PKG_ID rows and + // as the lookup key for the direct-license EXISTS arm. + db.execSQL( + "CREATE INDEX IF NOT EXISTS idx_steam_app_package_id " + + "ON steam_app(package_id)", + ) + } + }) .build() } diff --git a/app/src/main/java/app/gamenative/service/SteamService.kt b/app/src/main/java/app/gamenative/service/SteamService.kt index a38caec31d..ae239ca927 100644 --- a/app/src/main/java/app/gamenative/service/SteamService.kt +++ b/app/src/main/java/app/gamenative/service/SteamService.kt @@ -278,6 +278,11 @@ class SteamService : Service(), IChallengeUrlChanged { private val pendingSyncFileLock = Any() private val pendingSyncFile by lazy { File(applicationContext.filesDir, "pending_achievement_sync.txt") } + // Debounce fields for onLicenseList — coalesces rapid callback bursts (e.g. bulk + // activations) into one processing run so parallel in-flight coroutines don't OOM. + private var licenseListDebounceJob: Job? = null + @Volatile private var pendingLicenseCallback: LicenseListCallback? = null + private val onEndProcess: (AndroidEvent.EndProcess) -> Unit = { Companion.stop() } @@ -583,9 +588,14 @@ class SteamService : Service(), IChallengeUrlChanged { * Get licenses from database for use with DepotDownloader */ suspend fun getLicensesFromDb(): List = withContext(Dispatchers.IO) { - val cached = instance?.cachedLicenseDao?.getAll() ?: return@withContext emptyList() - cached.mapNotNull { cachedLicense -> - LicenseSerializer.deserializeLicense(cachedLicense.licenseJson) + val inst = instance ?: return@withContext emptyList() + val cached = inst.cachedLicenseDao.getAll() + if (cached.isNotEmpty()) { + cached.mapNotNull { LicenseSerializer.deserializeLicense(it.licenseJson) } + } else { + // Return the licences from memory-cache. Will eventually be correct and sync + Timber.w("getLicensesFromDb: cachedLicenseDao empty, using in-memory licenses") + inst.licenses } } @@ -2955,7 +2965,14 @@ class SteamService : Service(), IChallengeUrlChanged { } suspend fun getOwnedGames(friendID: Long): List = withContext(Dispatchers.IO) { - instance?._unifiedFriends!!.getOwnedGames(friendID) + try { + instance?._unifiedFriends?.getOwnedGames(friendID) ?: emptyList() + } catch (e: Exception) { + // ensureActive() rethrows if *our* coroutine was genuinely cancelled + ensureActive() + Timber.w(e, "getOwnedGames($friendID) failed; returning empty") + emptyList() + } } // Add helper to detect if any downloads or cloud sync are in progress @@ -4133,80 +4150,123 @@ class SteamService : Service(), IChallengeUrlChanged { Timber.i("Received License List ${callback.result}, size: ${callback.licenseList.size}") - scope.launch { - db.withTransaction { - // Note: I assume with every launch we do, in fact, update the licenses for app the apps if we join or get removed - // from family sharing... We really can't test this as there is a 1-year cooldown. - // Then 'findStaleLicences' will find these now invalid items to remove. - - // Chunk the input to reduce memory pressures for very large items. - licenses = callback.licenseList - cachedLicenseDao.deleteAll() - callback.licenseList.chunked(MAX_PICS_BUFFER).forEach { chunk -> - cachedLicenseDao.insertAll( - chunk.map { license -> - CachedLicense(licenseJson = LicenseSerializer.serializeLicense(license)) - }, + // Coalesce rapid callbacks (e.g. bulk activations) so only the last in a burst triggers + // the expensive DB + PICS work, preventing parallel in-flight coroutines from OOMing. + pendingLicenseCallback = callback + licenseListDebounceJob?.cancel() + licenseListDebounceJob = scope.launch { + delay(5.seconds) + val pending = pendingLicenseCallback ?: return@launch + + // Set the in-memory field immediately — DepotDownloader uses this for auth during the + // current session; the DB writes below are for persistence across restarts. + licenses = pending.licenseList + + // --- CPU work BEFORE acquiring any write lock --- + // Previously this groupBy/map ran inside withTransaction, holding the write lock + // during expensive CPU work on the full (potentially 30k+) license list. + val licensesToAdd = pending.licenseList + .groupBy { it.packageID } + .map { licensesEntry -> + val preferred = licensesEntry.value.firstOrNull { + it.ownerAccountID == userSteamId?.accountID?.toInt() + } ?: licensesEntry.value.first() + SteamLicense( + packageId = licensesEntry.key, + lastChangeNumber = preferred.lastChangeNumber, + timeCreated = preferred.timeCreated, + timeNextProcess = preferred.timeNextProcess, + minuteLimit = preferred.minuteLimit, + minutesUsed = preferred.minutesUsed, + paymentMethod = preferred.paymentMethod, + licenseFlags = licensesEntry.value + .map { it.licenseFlags } + .reduceOrNull { first, second -> + val combined = EnumSet.copyOf(first) + combined.addAll(second) + combined + } ?: EnumSet.noneOf(ELicenseFlags::class.java), + purchaseCode = preferred.purchaseCode, + licenseType = preferred.licenseType, + territoryCode = preferred.territoryCode, + accessToken = preferred.accessToken, + ownerAccountId = licensesEntry.value.map { it.ownerAccountID }, + masterPackageID = preferred.masterPackageID, ) } - val licensesToAdd = callback.licenseList - .groupBy { it.packageID } - .map { licensesEntry -> - val preferred = licensesEntry.value.firstOrNull { - it.ownerAccountID == userSteamId?.accountID?.toInt() - } ?: licensesEntry.value.first() - SteamLicense( - packageId = licensesEntry.key, - lastChangeNumber = preferred.lastChangeNumber, - timeCreated = preferred.timeCreated, - timeNextProcess = preferred.timeNextProcess, - minuteLimit = preferred.minuteLimit, - minutesUsed = preferred.minutesUsed, - paymentMethod = preferred.paymentMethod, - licenseFlags = licensesEntry.value - .map { it.licenseFlags } - .reduceOrNull { first, second -> - val combined = EnumSet.copyOf(first) - combined.addAll(second) - combined - } ?: EnumSet.noneOf(ELicenseFlags::class.java), - purchaseCode = preferred.purchaseCode, - licenseType = preferred.licenseType, - territoryCode = preferred.territoryCode, - accessToken = preferred.accessToken, - ownerAccountId = licensesEntry.value.map { it.ownerAccountID }, // Read note above - masterPackageID = preferred.masterPackageID, - ) - } - if (licensesToAdd.isNotEmpty()) { - Timber.i("Adding ${licensesToAdd.size} licenses") - licensesToAdd.chunked(MAX_PICS_BUFFER).forEach { chunk -> - licenseDao.insertAll(chunk) - } - } + // --- Diff against the DB before acquiring the write lock --- + // getLicenseStubs() fetches only packageId + lastChangeNumber + accessToken, avoiding + // deserialization of the large app_ids/depot_ids lists for every row. + val existingStubs = licenseDao.getLicenseStubs().associateBy { it.packageId } + val incomingIds = licensesToAdd.mapTo(HashSet(licensesToAdd.size)) { it.packageId } - Timber.i("Finished adding ${licensesToAdd.size} licenses") + val newLicenses = licensesToAdd.filter { it.packageId !in existingStubs } + val changedLicenses = licensesToAdd.filter { pkg -> + val stub = existingStubs[pkg.packageId] + stub != null && stub.lastChangeNumber != pkg.lastChangeNumber + } + val staleIds = existingStubs.keys.filterNot { it in incomingIds } - val licensesToRemove = licenseDao.findStaleLicences( - packageIds = callback.licenseList.map { it.packageID }, - ) + Timber.i( + "onLicenseList diff: ${newLicenses.size} new, ${changedLicenses.size} changed, " + + "${staleIds.size} stale, " + + "${licensesToAdd.size - newLicenses.size - changedLicenses.size} unchanged", + ) - if (licensesToRemove.isNotEmpty()) { - Timber.i("Removing ${licensesToRemove.size} (stale) licenses") - val packageIds = licensesToRemove.map { it.packageId } - licenseDao.deleteStaleLicenses(packageIds) + // For changed licenses, read their existing PICS-derived columns (app_ids/depot_ids) so + // we can carry them forward — licensesToAdd is built from the callback, which doesn't + // carry those. We only fetch full rows for the small changed subset, not the whole list. + val mergedChangedLicenses = if (changedLicenses.isNotEmpty()) { + val existingPics = licenseDao.findLicenses(changedLicenses.map { it.packageId }) + .associateBy { it.packageId } + changedLicenses.map { updated -> + val pics = existingPics[updated.packageId] + updated.copy( + appIds = pics?.appIds ?: emptyList(), + depotIds = pics?.depotIds ?: emptyList(), + ) } + } else { + emptyList() + } - Timber.i("Getting licenses from DB") - // Get PICS information with the current license database. - licenseDao.getLicensesForPics() + // transaction - Only update based on new and changed licences + db.withTransaction { + val toWrite = newLicenses + mergedChangedLicenses + if (toWrite.isNotEmpty()) { + toWrite.chunked(MAX_PICS_BUFFER).forEach { licenseDao.insertAll(it) } + } + if (staleIds.isNotEmpty()) { + Timber.i("Removing ${staleIds.size} (stale) licenses") + licenseDao.deleteStaleLicenses(staleIds) + } + } + + // queue only new and changed packages for PICS to reduce db load. + val toQueue = newLicenses + changedLicenses + if (toQueue.isNotEmpty()) { + toQueue .map { PICSRequest(it.packageId, it.accessToken) } .chunked(MAX_PICS_BUFFER) .forEach { chunk -> Timber.d("onLicenseList: Queueing ${chunk.size} package(s) for PICS") packagePicsChannel.send(chunk) } + } else { + Timber.i("onLicenseList: no packages need PICS sync, skipping queue") + } + + // --- Transaction 2: raw license persistence for DepotDownloader --- + // JSON serialization of the full license list happens here (outside T1) so T1's lock + // window stays short. Chunked writes cap peak allocation and keep each lock window + // to milliseconds. Runs after PICS is queued so it doesn't delay the pipeline. + db.withTransaction { cachedLicenseDao.deleteAll() } + pending.licenseList.chunked(MAX_PICS_BUFFER).forEach { chunk -> + val cachedChunk = chunk.map { license -> + CachedLicense(licenseJson = LicenseSerializer.serializeLicense(license)) + } + db.withTransaction { cachedLicenseDao.insertAll(cachedChunk) } } } } From 47ce1d5ab61dbe8b23ff80f7931dad64b6921e52 Mon Sep 17 00:00:00 2001 From: Daniel Joyce Date: Wed, 15 Jul 2026 20:58:45 +0100 Subject: [PATCH 5/7] fix(): Adjust comments to be more semantic. --- .../java/app/gamenative/data/SteamAppPicsMeta.kt | 5 +---- .../main/java/app/gamenative/db/dao/SteamAppDao.kt | 12 ++---------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/app/src/main/java/app/gamenative/data/SteamAppPicsMeta.kt b/app/src/main/java/app/gamenative/data/SteamAppPicsMeta.kt index 6da30b886f..075dfa81c9 100644 --- a/app/src/main/java/app/gamenative/data/SteamAppPicsMeta.kt +++ b/app/src/main/java/app/gamenative/data/SteamAppPicsMeta.kt @@ -2,10 +2,7 @@ package app.gamenative.data import androidx.room.ColumnInfo -// Slimmed-down projection of the columns the PICS collectors actually read when -// deciding whether an app needs (re)processing. Avoids pulling the full SteamApp -// row — including the large depots/config/UFS JSON blobs — into memory just to -// compare change numbers for tens of thousands of apps. +// Slimmed-down projection of the columns the PICS collectors actually read data class SteamAppPicsMeta( val id: Int, @ColumnInfo("package_id") diff --git a/app/src/main/java/app/gamenative/db/dao/SteamAppDao.kt b/app/src/main/java/app/gamenative/db/dao/SteamAppDao.kt index 877dc9ba25..bcbc3f7847 100644 --- a/app/src/main/java/app/gamenative/db/dao/SteamAppDao.kt +++ b/app/src/main/java/app/gamenative/db/dao/SteamAppDao.kt @@ -224,11 +224,7 @@ interface SteamAppDao { @Query("UPDATE steam_app SET package_id = :packageId WHERE id IN (:appIds)") suspend fun _updatePackageIdForApps(packageId: Int, appIds: List) - /** - * Reassigns package ownership for a set of apps in one statement per chunk, - * instead of read-modify-writing each full SteamApp row. Batched to stay - * under SQLite's 999-parameter ceiling. - */ + // Reassigns package ownership for a set of apps in one statement per chunk @Transaction suspend fun updatePackageIdForApps(packageId: Int, appIds: List) { if (appIds.isEmpty()) return @@ -238,11 +234,7 @@ interface SteamAppDao { } } - /** - * Lightweight projection of the columns the PICS collectors read to decide - * whether an app needs (re)processing. Batched to stay under SQLite's - * 999-parameter ceiling; avoids deserializing the full row/blobs per app. - */ + // Lightweight projection of the columns the PICS collectors read to decide whether an app needs (re)processing @Transaction suspend fun findAppPicsMeta(appIds: List): List { if (appIds.isEmpty()) return emptyList() From fc3721d678a4b924af0187343a2c0499e3fec420 Mon Sep 17 00:00:00 2001 From: Daniel Joyce Date: Wed, 15 Jul 2026 21:04:45 +0100 Subject: [PATCH 6/7] fix(): adjust comments --- .../main/java/app/gamenative/di/DatabaseModule.kt | 12 +----------- .../main/java/app/gamenative/service/SteamService.kt | 9 ++------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/app/src/main/java/app/gamenative/di/DatabaseModule.kt b/app/src/main/java/app/gamenative/di/DatabaseModule.kt index b32c1293fc..b8091c98ad 100644 --- a/app/src/main/java/app/gamenative/di/DatabaseModule.kt +++ b/app/src/main/java/app/gamenative/di/DatabaseModule.kt @@ -42,25 +42,15 @@ class DatabaseModule { // Use unbounded thread pools for reads/writes so a long-running query (e.g. the // library COUNT/paged load during a large PICS sync) can't starve the default // single-threaded executors and deadlock the connection pool. SQLite still - // serializes writers, so this only removes the scheduling deadlock. + // serializes writers, so this only removes the scheduling deadlock .setQueryExecutor(Executors.newCachedThreadPool()) .setTransactionExecutor(Executors.newCachedThreadPool()) .addCallback(object : RoomDatabase.Callback() { override fun onOpen(db: SupportSQLiteDatabase) { - // These indexes are not declared in @Entity (which would require a schema - // migration) — they're created here so we can add them without a version bump - // while keeping upstream compatibility. IF NOT EXISTS makes them idempotent. - // - // dlc_for_app_id: the OWNED_APPS_WHERE DLC EXISTS arm joins back into - // steam_app on this column. Without an index, SQLite scans all rows per - // outer row that fails the direct-license check, turning the COUNT(*) query - // O(N²) and causing 100+ second runtimes that exhaust the connection pool. db.execSQL( "CREATE INDEX IF NOT EXISTS idx_steam_app_dlc_for_app_id " + "ON steam_app(dlc_for_app_id)", ) - // package_id: used in the WHERE clause to exclude INVALID_PKG_ID rows and - // as the lookup key for the direct-license EXISTS arm. db.execSQL( "CREATE INDEX IF NOT EXISTS idx_steam_app_package_id " + "ON steam_app(package_id)", diff --git a/app/src/main/java/app/gamenative/service/SteamService.kt b/app/src/main/java/app/gamenative/service/SteamService.kt index ae239ca927..698e3e998f 100644 --- a/app/src/main/java/app/gamenative/service/SteamService.kt +++ b/app/src/main/java/app/gamenative/service/SteamService.kt @@ -4162,9 +4162,6 @@ class SteamService : Service(), IChallengeUrlChanged { // current session; the DB writes below are for persistence across restarts. licenses = pending.licenseList - // --- CPU work BEFORE acquiring any write lock --- - // Previously this groupBy/map ran inside withTransaction, holding the write lock - // during expensive CPU work on the full (potentially 30k+) license list. val licensesToAdd = pending.licenseList .groupBy { it.packageID } .map { licensesEntry -> @@ -4195,9 +4192,8 @@ class SteamService : Service(), IChallengeUrlChanged { ) } - // --- Diff against the DB before acquiring the write lock --- - // getLicenseStubs() fetches only packageId + lastChangeNumber + accessToken, avoiding - // deserialization of the large app_ids/depot_ids lists for every row. + + // Compare the new and changed licences so we only update/add what we need val existingStubs = licenseDao.getLicenseStubs().associateBy { it.packageId } val incomingIds = licensesToAdd.mapTo(HashSet(licensesToAdd.size)) { it.packageId } @@ -4257,7 +4253,6 @@ class SteamService : Service(), IChallengeUrlChanged { Timber.i("onLicenseList: no packages need PICS sync, skipping queue") } - // --- Transaction 2: raw license persistence for DepotDownloader --- // JSON serialization of the full license list happens here (outside T1) so T1's lock // window stays short. Chunked writes cap peak allocation and keep each lock window // to milliseconds. Runs after PICS is queued so it doesn't delay the pipeline. From d62837fe4aa4d23d31a0d535a008e268d207ccae Mon Sep 17 00:00:00 2001 From: Daniel Joyce Date: Wed, 15 Jul 2026 21:35:46 +0100 Subject: [PATCH 7/7] fix(): Feedback adjustments - Need to test more. --- .../java/app/gamenative/db/dao/SteamAppDao.kt | 8 +- .../java/app/gamenative/di/DatabaseModule.kt | 10 +- .../app/gamenative/service/SteamService.kt | 245 ++++++++++-------- 3 files changed, 150 insertions(+), 113 deletions(-) diff --git a/app/src/main/java/app/gamenative/db/dao/SteamAppDao.kt b/app/src/main/java/app/gamenative/db/dao/SteamAppDao.kt index bcbc3f7847..c2cb243332 100644 --- a/app/src/main/java/app/gamenative/db/dao/SteamAppDao.kt +++ b/app/src/main/java/app/gamenative/db/dao/SteamAppDao.kt @@ -224,12 +224,14 @@ interface SteamAppDao { @Query("UPDATE steam_app SET package_id = :packageId WHERE id IN (:appIds)") suspend fun _updatePackageIdForApps(packageId: Int, appIds: List) - // Reassigns package ownership for a set of apps in one statement per chunk + // Reassigns package ownership for a set of apps in one statement per chunk. + // The query binds :packageId (1 var) plus each element of :appIds, so the chunk + // size is SQLITE_MAX_VARS - 1 to stay within the 999-variable limit. @Transaction suspend fun updatePackageIdForApps(packageId: Int, appIds: List) { if (appIds.isEmpty()) return - for (chunkStart in appIds.indices step SQLITE_MAX_VARS) { - val chunkEnd = min(chunkStart + SQLITE_MAX_VARS, appIds.size) + for (chunkStart in appIds.indices step SQLITE_MAX_VARS - 1) { + val chunkEnd = min(chunkStart + SQLITE_MAX_VARS - 1, appIds.size) _updatePackageIdForApps(packageId, appIds.subList(chunkStart, chunkEnd)) } } diff --git a/app/src/main/java/app/gamenative/di/DatabaseModule.kt b/app/src/main/java/app/gamenative/di/DatabaseModule.kt index b8091c98ad..f45ee7fab4 100644 --- a/app/src/main/java/app/gamenative/di/DatabaseModule.kt +++ b/app/src/main/java/app/gamenative/di/DatabaseModule.kt @@ -39,11 +39,11 @@ class DatabaseModule { ROOM_MIGRATION_V23_to_V24, ) .fallbackToDestructiveMigration(true) - // Use unbounded thread pools for reads/writes so a long-running query (e.g. the - // library COUNT/paged load during a large PICS sync) can't starve the default - // single-threaded executors and deadlock the connection pool. SQLite still - // serializes writers, so this only removes the scheduling deadlock - .setQueryExecutor(Executors.newCachedThreadPool()) + // Bound the query thread pool so PICS-sync bursts can't spin up an unbounded + // number of threads. Size to at least 4 threads so short queries aren't blocked + // behind long-running ones; the transaction executor stays unbounded so writers + // (which serialise at the SQLite level) don't deadlock waiting for a thread. + .setQueryExecutor(Executors.newFixedThreadPool(maxOf(4, Runtime.getRuntime().availableProcessors() * 2))) .setTransactionExecutor(Executors.newCachedThreadPool()) .addCallback(object : RoomDatabase.Callback() { override fun onOpen(db: SupportSQLiteDatabase) { diff --git a/app/src/main/java/app/gamenative/service/SteamService.kt b/app/src/main/java/app/gamenative/service/SteamService.kt index 698e3e998f..3ad38e361d 100644 --- a/app/src/main/java/app/gamenative/service/SteamService.kt +++ b/app/src/main/java/app/gamenative/service/SteamService.kt @@ -319,6 +319,10 @@ class SteamService : Service(), IChallengeUrlChanged { // to get a first response; give them more headroom before treating them as timed out. const val PICS_JOB_TIMEOUT_MS = 30_000L + // Timed-out PICS batches are retried up to this many times with linear backoff. + const val MAX_PICS_BATCH_RETRIES = 3 + const val PICS_RETRY_BACKOFF_MS = 5_000L + const val MAX_RETRY_ATTEMPTS = 20 const val INVALID_APP_ID: Int = Int.MAX_VALUE @@ -2927,6 +2931,12 @@ class SteamService : Service(), IChallengeUrlChanged { fun clearDatabase(clearCloudSyncState: Boolean = false) { with(instance!!) { + // Cancel any pending debounced license processing so it cannot repopulate + // license tokens after logout. + licenseListDebounceJob?.cancel() + licenseListDebounceJob = null + pendingLicenseCallback = null + licenses = emptyList() scope.launch { db.withTransaction { appDao.deleteAll() @@ -2935,6 +2945,7 @@ class SteamService : Service(), IChallengeUrlChanged { fileChangeListsDao.deleteAll() } licenseDao.deleteAll() + cachedLicenseDao.deleteAll() encryptedAppTicketDao.deleteAll() downloadingAppInfoDao.deleteAll() steamUnlockedBranchDao.deleteAll() @@ -4150,6 +4161,10 @@ class SteamService : Service(), IChallengeUrlChanged { Timber.i("Received License List ${callback.result}, size: ${callback.licenseList.size}") + // Set the in-memory field immediately — DepotDownloader uses this for auth during the + // current session; the DB writes below are for persistence across restarts. + licenses = callback.licenseList + // Coalesce rapid callbacks (e.g. bulk activations) so only the last in a burst triggers // the expensive DB + PICS work, preventing parallel in-flight coroutines from OOMing. pendingLicenseCallback = callback @@ -4158,10 +4173,6 @@ class SteamService : Service(), IChallengeUrlChanged { delay(5.seconds) val pending = pendingLicenseCallback ?: return@launch - // Set the in-memory field immediately — DepotDownloader uses this for auth during the - // current session; the DB writes below are for persistence across restarts. - licenses = pending.licenseList - val licensesToAdd = pending.licenseList .groupBy { it.packageID } .map { licensesEntry -> @@ -4200,7 +4211,7 @@ class SteamService : Service(), IChallengeUrlChanged { val newLicenses = licensesToAdd.filter { it.packageId !in existingStubs } val changedLicenses = licensesToAdd.filter { pkg -> val stub = existingStubs[pkg.packageId] - stub != null && stub.lastChangeNumber != pkg.lastChangeNumber + stub != null && (stub.lastChangeNumber != pkg.lastChangeNumber || stub.accessToken != pkg.accessToken) } val staleIds = existingStubs.keys.filterNot { it in incomingIds } @@ -4254,14 +4265,17 @@ class SteamService : Service(), IChallengeUrlChanged { } // JSON serialization of the full license list happens here (outside T1) so T1's lock - // window stays short. Chunked writes cap peak allocation and keep each lock window - // to milliseconds. Runs after PICS is queued so it doesn't delay the pipeline. - db.withTransaction { cachedLicenseDao.deleteAll() } - pending.licenseList.chunked(MAX_PICS_BUFFER).forEach { chunk -> - val cachedChunk = chunk.map { license -> - CachedLicense(licenseJson = LicenseSerializer.serializeLicense(license)) + // window stays short. A single wrapping transaction ensures getLicensesFromDb() sees + // either the previous complete snapshot or the new one, with no empty window between. + // Chunked writes cap peak allocation while keeping the entire replace atomic. + db.withTransaction { + cachedLicenseDao.deleteAll() + pending.licenseList.chunked(MAX_PICS_BUFFER).forEach { chunk -> + val cachedChunk = chunk.map { license -> + CachedLicense(licenseJson = LicenseSerializer.serializeLicense(license)) + } + cachedLicenseDao.insertAll(cachedChunk) } - db.withTransaction { cachedLicenseDao.insertAll(cachedChunk) } } } } @@ -4390,90 +4404,101 @@ class SteamService : Service(), IChallengeUrlChanged { if (!isLoggedIn) return@collect val steamApps = instance?._steamApps ?: return@collect - try { - val job = steamApps.picsGetProductInfo( - apps = appRequests, - packages = emptyList(), - ) - job.timeout = PICS_JOB_TIMEOUT_MS - val callback = job.await() - - callback.results.forEachIndexed { index, picsCallback -> - Timber.d( - "onPicsProduct: ${index + 1} of ${callback.results.size}" + - "\n\tReceived PICS result of ${picsCallback.apps.size} app(s)." + - "\n\tReceived PICS result of ${picsCallback.packages.size} package(s).", + var picsAttempt = 0 + while (picsAttempt <= MAX_PICS_BATCH_RETRIES) { + try { + val job = steamApps.picsGetProductInfo( + apps = appRequests, + packages = emptyList(), ) - - ensureActive() - - // Batch the DB reads that the per-app decision needs, instead of - // issuing two point queries (findApp + findLicense) per app. For a - // 30K-game library the per-app path was ~512 queries per 256-app - // batch, each findApp deserializing the full depots/config/UFS blob - // just to compare a change number. Here we do two batched reads: - // a slim metadata projection keyed by appId, and the licenses keyed - // by packageId, then decide everything in memory. - val appMetaById = appDao - .findAppPicsMeta(picsCallback.apps.values.map { it.id }) - .associateBy { it.id } - val licensesByPkgId = licenseDao - .findLicenses( - appMetaById.values - .map { it.packageId } - .filter { it != INVALID_PKG_ID } - .distinct(), + job.timeout = PICS_JOB_TIMEOUT_MS + val callback = job.await() + + callback.results.forEachIndexed { index, picsCallback -> + Timber.d( + "onPicsProduct: ${index + 1} of ${callback.results.size}" + + "\n\tReceived PICS result of ${picsCallback.apps.size} app(s)." + + "\n\tReceived PICS result of ${picsCallback.packages.size} package(s).", ) - .associateBy { it.packageId } - val steamAppsMap = picsCallback.apps.values.mapNotNull { app -> - val appMeta = appMetaById[app.id] - val packageId = appMeta?.packageId ?: INVALID_PKG_ID - val packageFromDb = if (packageId != INVALID_PKG_ID) licensesByPkgId[packageId] else null - val ownerAccountId = packageFromDb?.ownerAccountId ?: emptyList() + ensureActive() + + // Batch the DB reads that the per-app decision needs, instead of + // issuing two point queries (findApp + findLicense) per app. For a + // 30K-game library the per-app path was ~512 queries per 256-app + // batch, each findApp deserializing the full depots/config/UFS blob + // just to compare a change number. Here we do two batched reads: + // a slim metadata projection keyed by appId, and the licenses keyed + // by packageId, then decide everything in memory. + val appMetaById = appDao + .findAppPicsMeta(picsCallback.apps.values.map { it.id }) + .associateBy { it.id } + val licensesByPkgId = licenseDao + .findLicenses( + appMetaById.values + .map { it.packageId } + .filter { it != INVALID_PKG_ID } + .distinct(), + ) + .associateBy { it.packageId } + + val steamAppsMap = picsCallback.apps.values.mapNotNull { app -> + val appMeta = appMetaById[app.id] + val packageId = appMeta?.packageId ?: INVALID_PKG_ID + val packageFromDb = if (packageId != INVALID_PKG_ID) licensesByPkgId[packageId] else null + val ownerAccountId = packageFromDb?.ownerAccountId ?: emptyList() - // Apps with -1 for the ownerAccountId should be added. - // This can help with friend game names. + // Apps with -1 for the ownerAccountId should be added. + // This can help with friend game names. - // TODO maybe apps with -1 for the ownerAccountId can be stripped with necessities and name. + // TODO maybe apps with -1 for the ownerAccountId can be stripped with necessities and name. - val ufsParseVersionOutdated = appMeta != null && appMeta.ufsParseVersion < CURRENT_UFS_PARSE_VERSION + val ufsParseVersionOutdated = appMeta != null && appMeta.ufsParseVersion < CURRENT_UFS_PARSE_VERSION - if (app.changeNumber != appMeta?.lastChangeNumber || ufsParseVersionOutdated) { - val newApp = app.keyValues.generateSteamApp().copy( - packageId = packageId, - ownerAccountId = ownerAccountId, - receivedPICS = true, - lastChangeNumber = app.changeNumber, - licenseFlags = packageFromDb?.licenseFlags ?: EnumSet.noneOf(ELicenseFlags::class.java), - ) - if (ufsParseVersionOutdated && newApp.ufs.saveFilePatterns.any { it.uploadRoot != it.root || it.uploadPath != it.path }) { - // UFS path logic changed and this app has rootoverrides: store 0 to force one - // full cloud query while preserving the local sync snapshot. - changeNumbersDao.insert(app.id, 0L) + if (app.changeNumber != appMeta?.lastChangeNumber || ufsParseVersionOutdated) { + val newApp = app.keyValues.generateSteamApp().copy( + packageId = packageId, + ownerAccountId = ownerAccountId, + receivedPICS = true, + lastChangeNumber = app.changeNumber, + licenseFlags = packageFromDb?.licenseFlags ?: EnumSet.noneOf(ELicenseFlags::class.java), + ) + if (ufsParseVersionOutdated && newApp.ufs.saveFilePatterns.any { it.uploadRoot != it.root || it.uploadPath != it.path }) { + // UFS path logic changed and this app has rootoverrides: store 0 to force one + // full cloud query while preserving the local sync snapshot. + changeNumbersDao.insert(app.id, 0L) + } + newApp + } else { + null } - newApp - } else { - null } - } - if (steamAppsMap.isNotEmpty()) { - Timber.i("Inserting ${steamAppsMap.size} PICS apps to database") - db.withTransaction { - appDao.insertAll(steamAppsMap) + if (steamAppsMap.isNotEmpty()) { + Timber.i("Inserting ${steamAppsMap.size} PICS apps to database") + db.withTransaction { + appDao.insertAll(steamAppsMap) + } } } + break + } catch (e: CancellationException) { + // A timed-out AsyncJobMultiple with zero results calls future.cancel() + // rather than failing the future, so this looks like coroutine + // cancellation. Only rethrow if this collector's own job was cancelled; + // otherwise retry with backoff until the limit is reached. + ensureActive() + picsAttempt++ + if (picsAttempt > MAX_PICS_BATCH_RETRIES) { + Timber.w("PICS product info request timed out for ${appRequests.size} app(s); max retries exceeded, dropping batch") + } else { + Timber.w("PICS product info timed out for ${appRequests.size} app(s); retry $picsAttempt of $MAX_PICS_BATCH_RETRIES") + delay(picsAttempt.toLong() * PICS_RETRY_BACKOFF_MS) + } + } catch (e: AsyncJobFailedException) { + Timber.w("Could not get PICS product info $e") + break } - } catch (e: CancellationException) { - // A timed-out AsyncJobMultiple with zero results calls future.cancel() - // rather than failing the future, so this looks like coroutine - // cancellation. Only rethrow if this collector's own job was cancelled; - // otherwise skip the batch so later batches on the channel still run. - ensureActive() - Timber.w("PICS product info request timed out for ${appRequests.size} app(s); skipping batch") - } catch (e: AsyncJobFailedException) { - Timber.w("Could not get PICS product info $e") } } } @@ -4489,29 +4514,20 @@ class SteamService : Service(), IChallengeUrlChanged { if (!isLoggedIn) return@collect val steamApps = instance?._steamApps ?: return@collect - val callback = try { - val job = steamApps.picsGetProductInfo( - apps = emptyList(), - packages = packageRequests, - ) - job.timeout = PICS_JOB_TIMEOUT_MS - job.await() - } catch (e: CancellationException) { - // See the app-channel collector above: a timed-out AsyncJobMultiple - // cancels its future instead of failing it, so only rethrow if this - // collector's own job was actually cancelled. - ensureActive() - Timber.w("PICS product info request timed out for ${packageRequests.size} package(s); skipping batch") - return@collect - } catch (e: AsyncJobFailedException) { - Timber.w("Could not get PICS product info for packages $e") - return@collect - } + var picsAttempt = 0 + while (picsAttempt <= MAX_PICS_BATCH_RETRIES) { + try { + val job = steamApps.picsGetProductInfo( + apps = emptyList(), + packages = packageRequests, + ) + job.timeout = PICS_JOB_TIMEOUT_MS + val callback = job.await() - callback.results.forEach { picsCallback -> - // Don't race the queue. - if (!isLoggedIn) return@collect - val queue = Collections.synchronizedList(mutableListOf()) + callback.results.forEach { picsCallback -> + // Don't race the queue. + if (!isLoggedIn) return@collect + val queue = Collections.synchronizedList(mutableListOf()) db.withTransaction { // When the same app appears in multiple packages (e.g. user owns the game and @@ -4656,6 +4672,25 @@ class SteamService : Service(), IChallengeUrlChanged { } catch (e: AsyncJobFailedException) { Timber.w("Could not get PICS product info $e") } + } + break + } catch (e: CancellationException) { + // See the app-channel collector above: a timed-out AsyncJobMultiple + // cancels its future instead of failing it, so only rethrow if this + // collector's own job was actually cancelled; otherwise retry with + // backoff up to the limit so failed metadata requests can be retried. + ensureActive() + picsAttempt++ + if (picsAttempt > MAX_PICS_BATCH_RETRIES) { + Timber.w("PICS product info request timed out for ${packageRequests.size} package(s); max retries exceeded, dropping batch") + } else { + Timber.w("PICS product info timed out for ${packageRequests.size} package(s); retry $picsAttempt of $MAX_PICS_BATCH_RETRIES") + delay(picsAttempt.toLong() * PICS_RETRY_BACKOFF_MS) + } + } catch (e: AsyncJobFailedException) { + Timber.w("Could not get PICS product info for packages $e") + break + } } } }