Refresh HTTP cache on miss for exact-version restore (#3116)

src/NuGet.Core/NuGet.Commands/RestoreCommand/SourceRepositoryDependencyProvider.cs

@@ -4,7 +4,9 @@
 #nullable disable
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -36,10 +38,34 @@ public class SourceRepositoryDependencyProvider : IRemoteDependencyProvider
         private bool _ignoreWarning;
         private bool _isFallbackFolderSource;
         private bool _useLegacyAssetTargetFallbackBehavior;
+        private readonly bool _refreshHttpCacheOnMissEnabled;
 
         private readonly TaskResultCache<LibraryRangeCacheKey, LibraryDependencyInfo> _dependencyInfoCache = new();
         private readonly TaskResultCache<LibraryRange, LibraryIdentity> _libraryMatchCache = new();
 
+        // Refresh-on-miss coordination, scoped to this provider instance. Because the provider is
+        // cached and shared across all projects in a restore operation (see RestoreCommandProvidersCache),
+        // these collections are effectively operation-wide. See https://github.com/NuGet/Home/issues/3116.
+
+        // Package ids for which the HTTP cache has already been refreshed once during this operation.
+        // Used to guarantee at most one refresh-on-miss per id per operation.
+        private readonly ConcurrentDictionary<string, byte> _idsRefreshedOnMiss = new(StringComparer.OrdinalIgnoreCase);
+
+        // Package ids whose version list was already fetched with a fresh cache during this operation.
+        // A later miss for such an id must not trigger another refresh: the data is already authoritative.
+        private readonly ConcurrentDictionary<string, byte> _idsFetchedThisOperation = new(StringComparer.OrdinalIgnoreCase);
+
+        // Opt-out for refresh-on-miss. When set to "false"/"0" (case-insensitive), NuGet will not refresh
+        // the HTTP cache when the cached versions list does not satisfy an exact requested version.
+        private const string RefreshHttpCacheOnMissEnvVar = "NUGET_HTTP_CACHE_REFRESH_ON_MISS";
+
+        internal static bool IsRefreshOnMissEnabled(IEnvironmentVariableReader environmentVariableReader)
+        {
+            string value = environmentVariableReader.GetEnvironmentVariable(RefreshHttpCacheOnMissEnvVar);
+            return !string.Equals(value, "false", StringComparison.OrdinalIgnoreCase)
+                && !string.Equals(value, "0", StringComparison.Ordinal);
+        }
+
         // Limiting concurrent requests to limit the amount of files open at a time.
         private readonly static SemaphoreSlim _throttle = GetThrottleSemaphoreSlim(EnvironmentVariableWrapper.Instance);
         internal static SemaphoreSlim GetThrottleSemaphoreSlim(IEnvironmentVariableReader env)
@@ -138,6 +164,7 @@ internal SourceRepositoryDependencyProvider(
             _packageFileCache = fileCache;
             _isFallbackFolderSource = isFallbackFolderSource;
             _useLegacyAssetTargetFallbackBehavior = MSBuildStringUtility.IsTrue(environmentVariableReader.GetEnvironmentVariable("NUGET_USE_LEGACY_ASSET_TARGET_FALLBACK_DEPENDENCY_RESOLUTION"));
+            _refreshHttpCacheOnMissEnabled = IsRefreshOnMissEnabled(environmentVariableReader);
         }
 
         /// <summary>
@@ -238,6 +265,75 @@ private async Task<LibraryIdentity> FindLibraryCoreAsync(
         {
             await EnsureResource(cancellationToken);
 
+            string id = libraryRange.Name;
+
+            LibraryIdentity result = await FindLibraryFromFeedAsync(libraryRange, cacheContext, logger, cancellationToken);
+
+            // If this lookup already consulted a fresh cache (an explicit refresh-on-miss, --no-cache,
+            // or the existing download-retry path), record the id so a later miss for the same id does
+            // not trigger another, redundant refresh during this operation.
+            if (cacheContext.RefreshMemoryCache)
+            {
+                _idsFetchedThisOperation[id] = 0;
+            }
+
+            if (result != null)
+            {
+                return result;
+            }
+
+            // Refresh-on-miss: the cached versions list for this HTTP source did not contain the
+            // requested exact version. Refresh the HTTP cache once per id per restore operation and
+            // retry before declaring the package unresolved. This eliminates spurious NU1102 failures
+            // for the publish-then-consume scenario. See https://github.com/NuGet/Home/issues/3116.
+            if (ShouldRefreshHttpCacheOnMiss(libraryRange, cacheContext)
+                && !_idsFetchedThisOperation.ContainsKey(id)
+                && _idsRefreshedOnMiss.TryAdd(id, 0))
+            {
+                logger.LogMinimal(string.Format(
+                    CultureInfo.CurrentCulture,
+                    Strings.Log_RefreshingHttpCacheOnMiss,
+                    id,
+                    libraryRange.VersionRange.ToString()));
+
+                SourceCacheContext refreshedCacheContext = cacheContext.WithRefreshCacheTrue();
+
+                result = await FindLibraryFromFeedAsync(libraryRange, refreshedCacheContext, logger, cancellationToken);
+
+                _idsFetchedThisOperation[id] = 0;
+            }
+
+            return result;
+        }
+
+        /// <summary>
+        /// Determines whether a cache miss for the given <paramref name="libraryRange" /> should trigger a
+        /// one-time HTTP cache refresh. Only exact (non-floating, min-inclusive) version requests against an
+        /// HTTP source qualify: those are the ones where "the cache says the version doesn't exist" is
+        /// unambiguous. Floating ranges may be legitimately satisfied by an older cached version.
+        /// </summary>
+        private bool ShouldRefreshHttpCacheOnMiss(LibraryRange libraryRange, SourceCacheContext cacheContext)
+        {
+            if (!_refreshHttpCacheOnMissEnabled
+                || !IsHttp
+                || cacheContext.RefreshMemoryCache)
+            {
+                return false;
+            }
+
+            VersionRange versionRange = libraryRange.VersionRange;
+            return versionRange != null
+                && !versionRange.IsFloating
+                && versionRange.IsMinInclusive
+                && versionRange.MinVersion != null;
+        }
+
+        private async Task<LibraryIdentity> FindLibraryFromFeedAsync(
+            LibraryRange libraryRange,
+            SourceCacheContext cacheContext,
+            ILogger logger,
+            CancellationToken cancellationToken)
+        {
             if (libraryRange.VersionRange?.MinVersion != null && libraryRange.VersionRange.IsMinInclusive && !libraryRange.VersionRange.IsFloating)
             {
                 // first check if the exact min version exist then simply return that

src/NuGet.Core/NuGet.Commands/Strings.resx

@@ -142,6 +142,10 @@
   <data name="Log_ResolvingConflicts" xml:space="preserve">
     <value>Resolving conflicts for {0}...</value>
   </data>
+  <data name="Log_RefreshingHttpCacheOnMiss" xml:space="preserve">
+    <value>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</value>
+    <comment>{0} - Package id, {1} - version range string</comment>
+  </data>
   <data name="Log_RestoringPackages" xml:space="preserve">
     <value>Restoring packages for {0}...</value>
   </data>

src/NuGet.Core/NuGet.Commands/xlf/Strings.cs.xlf

@@ -884,6 +884,11 @@ Upgradujte svou sadu .NET SDK nebo odeberte RestoreUseLegacyDependencyResolver,
         <target state="translated">Čte se soubor projektu {0}.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">Nejde vyhovět konfliktním žádostem pro {0}: {1} Architektura: {2}</target>

src/NuGet.Core/NuGet.Commands/xlf/Strings.de.xlf

@@ -884,6 +884,11 @@ Aktualisieren Sie Ihr .NET SDK oder entfernen Sie RestoreUseLegacyDependencyReso
         <target state="translated">Projektdatei "{0}" wird gelesen.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">Die einen Konflikt verursachenden Anforderungen für "{0}" können nicht erfüllt werden: {1} Framework: {2}</target>

src/NuGet.Core/NuGet.Commands/xlf/Strings.es.xlf

@@ -884,6 +884,11 @@ Actualice el SDK de .NET o quite RestoreUseLegacyDependencyResolver para usar es
         <target state="translated">Leyendo el archivo del proyecto {0}.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">No se pueden satisfacer las solicitudes en conflicto para "{0}": {1} marco de trabajo {2}</target>

src/NuGet.Core/NuGet.Commands/xlf/Strings.fr.xlf

@@ -884,6 +884,11 @@ Mettez à niveau votre Kit de développement logiciel (SDK) .NET ou supprimez Re
         <target state="translated">Lecture du fichier projet {0}.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">Impossible de satisfaire les requêtes en conflit pour '{0}' : {1} Framework : {2}</target>

src/NuGet.Core/NuGet.Commands/xlf/Strings.it.xlf

@@ -884,6 +884,11 @@ Per usare questa funzionalità, aggiornare .NET SDK o rimuovere RestoreUseLegacy
         <target state="translated">Lettura del file del progetto {0}.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">Non è possibile soddisfare le richieste in conflitto per '{0}': {1}. Framework: {2}</target>

src/NuGet.Core/NuGet.Commands/xlf/Strings.ja.xlf

@@ -884,6 +884,11 @@ Upgrade your .NET SDK or remove RestoreUseLegacyDependencyResolver to use this f
         <target state="translated">プロジェクト ファイル {0} を読み取っています。</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">'{0}' の競合する要求を満たすことができません: {1} フレームワーク: {2}</target>

src/NuGet.Core/NuGet.Commands/xlf/Strings.ko.xlf

@@ -884,6 +884,11 @@ Upgrade your .NET SDK or remove RestoreUseLegacyDependencyResolver to use this f
         <target state="translated">프로젝트 파일 {0}을(를) 읽는 중입니다.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">'{0}'에 대해 충돌하는 요청을 충족할 수 없습니다. {1} 프레임워크: {2}</target>

src/NuGet.Core/NuGet.Commands/xlf/Strings.pl.xlf

@@ -884,6 +884,11 @@ Uaktualnij zestaw .NET SDK lub usuń RestoreUseLegacyDependencyResolver, aby kor
         <target state="translated">Odczytywanie pliku projektu {0}.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">Nie można zrealizować żądań będących w konflikcie dla elementu „{0}”: {1}, struktura: {2}</target>

src/NuGet.Core/NuGet.Commands/xlf/Strings.pt-BR.xlf

@@ -884,6 +884,11 @@ Atualize o SDK do .NET ou remova RestoreUseLegacyDependencyResolver para usar es
         <target state="translated">Lendo o arquivo de projeto {0}.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">Não foi possível satisfazer às solicitações conflitantes de '{0}': {1} Estrutura: {2}</target>

src/NuGet.Core/NuGet.Commands/xlf/Strings.ru.xlf

@@ -884,6 +884,11 @@ Upgrade your .NET SDK or remove RestoreUseLegacyDependencyResolver to use this f
         <target state="translated">Чтение файла проекта {0}.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">Не удалось удовлетворить конфликтующие запросы для "{0}": {1}. Платформа: {2}</target>

src/NuGet.Core/NuGet.Commands/xlf/Strings.tr.xlf

@@ -884,6 +884,11 @@ Bu özelliği kullanmak için .NET SDK'nizi yükseltin veya RestoreUseLegacyDepe
         <target state="translated">{0} adlı proje dosyası okunuyor.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">'{0}' için çakışan istekler giderilemiyor: {1} Çerçevesi: {2}</target>

src/NuGet.Core/NuGet.Commands/xlf/Strings.zh-Hans.xlf

@@ -884,6 +884,11 @@ Upgrade your .NET SDK or remove RestoreUseLegacyDependencyResolver to use this f
         <target state="translated">正在读取项目文件 {0}。</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">无法满足“{0}”的互相冲突的请求: {1} 框架: {2}</target>

src/NuGet.Core/NuGet.Commands/xlf/Strings.zh-Hant.xlf

@@ -884,6 +884,11 @@ Upgrade your .NET SDK or remove RestoreUseLegacyDependencyResolver to use this f
         <target state="translated">正在讀取專案檔 {0}。</target>
         <note />
       </trans-unit>
+      <trans-unit id="Log_RefreshingHttpCacheOnMiss">
+        <source>Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</source>
+        <target state="new">Cached versions for '{0}' did not contain a version satisfying '{1}'; refreshing the HTTP cache once before failing.</target>
+        <note>{0} - Package id, {1} - version range string</note>
+      </trans-unit>
       <trans-unit id="Log_ResolverConflict">
         <source>Unable to satisfy conflicting requests for '{0}': {1} Framework: {2}</source>
         <target state="translated">無法滿足 '{0}' 的衝突要求: {1} 架構: {2}</target>