C#: Support replaces-base via the DependabotProxy.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Immutable;
 using System.Collections.Generic;
 using System.IO;
 using System.Security.Cryptography.X509Certificates;
@@ -14,13 +15,39 @@ public class DependabotProxy : IDependabotProxy
         /// <summary>
         /// Represents configurations for package registries.
         /// </summary>
-        /// <param name="Type">The type of package registry.</param>
-        /// <param name="URL">The URL of the package registry.</param>
-        public record class RegistryConfig(string Type, string URL);
+        public class RegistryConfig
+        {
+            /// <summary>
+            /// The type of the package registry.
+            /// </summary>
+            public string Type { get; init; } = "";
+
+            /// <summary>
+            /// The URL of the package registry.
+            /// </summary>
+            public string URL { get; init; } = "";
+
+            /// <summary>
+            /// A boolean indicating whether this registry replaces the base registry.
+            /// </summary>
+            [JsonProperty("replaces-base")]
+            public bool ReplacesBase { get; init; } = false;
+        };
 
         public string Address { get; }
 
-        public HashSet<string> RegistryURLs { get; } = [];
+        /// <summary>
+        /// A dictionary mapping registry URLs to a boolean indicating whether they replace the base registry.
+        /// </summary>
+        private readonly Dictionary<string, bool> registryMapping = [];
+
+        private ImmutableHashSet<string>? registryURLs;
+        public ImmutableHashSet<string> RegistryURLs =>
+            registryURLs ??= registryMapping.Keys.ToImmutableHashSet();
+
+        private ImmutableHashSet<string>? registryBaseURLs;
+        public ImmutableHashSet<string> RegistryBaseURLs =>
+            registryBaseURLs ??= registryMapping.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToImmutableHashSet();
 
         public string? CertificatePath { get; private set; }
 
@@ -65,7 +92,7 @@ private DependabotProxy(IDependabotProxyConfiguration config, ILogger logger, Te
                             }
 
                             logger.LogInfo($"Found private registry at '{registry.URL}'");
-                            RegistryURLs.Add(registry.URL);
+                            registryMapping.AddOrUpdateToLatest(registry.URL, registry.ReplacesBase);
                         }
                     }
                 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -56,7 +56,6 @@ internal static class EnvironmentVariableNames
 
         /// <summary>
         /// Specifies the NuGet feeds to use for fallback NuGet dependency fetching. The value is a space-separated list of feed URLs.
-        /// The default value is `https://api.nuget.org/v3/index.json`.
         /// </summary>
         public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs

@@ -10,13 +10,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     internal sealed partial class FeedManager : IDisposable
     {
-        internal const string PublicNugetOrgFeed = "https://api.nuget.org/v3/index.json";
+        private const string PublicNugetOrg = "nuget.org";
+        private const string PublicDotNugetOrg = $".{PublicNugetOrg}";
+        internal const string PublicApiNugetOrgFeed = $"https://api{PublicDotNugetOrg}/v3/index.json";
 
         private readonly ILogger logger;
         private readonly IDotNet dotnet;
         private readonly IFileProvider fileProvider;
         private readonly DependencyDirectory emptyPackageDirectory;
         private readonly ImmutableHashSet<string> privateRegistryFeeds;
+        private readonly bool hasPrivateRegistryBaseFeeds;
+        private readonly ImmutableHashSet<string> privateRegistryBaseFeeds;
         private readonly IFeedManagerIO feedManagerIo;
 
         /// <summary>
@@ -72,14 +76,33 @@ internal sealed partial class FeedManager : IDisposable
         /// </summary>
         public ImmutableHashSet<string> ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value;
 
+        private readonly Lazy<ImmutableHashSet<string>> lazyReachableDefaultFeeds;
+
+        /// <summary>
+        /// Gets the list of default NuGet feeds that are configured in the environment.
+        /// This is either the public NuGet feed or a set of feeds specified by the environment.
+        /// </summary>
+        public ImmutableHashSet<string> DefaultFeeds { get; init; }
+
+        /// <summary>
+        /// Gets the list of reachable default NuGet feeds.
+        /// </summary>
+        public ImmutableHashSet<string> ReachableDefaultFeeds => lazyReachableDefaultFeeds.Value;
+
         public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider, IFeedManagerIO feedManagerIo)
         {
             this.logger = logger;
             this.dotnet = dotnet;
             this.fileProvider = fileProvider;
             this.feedManagerIo = feedManagerIo;
-            privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
+            privateRegistryFeeds = dependabotProxy?.RegistryURLs ?? [];
             HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0;
+            privateRegistryBaseFeeds = dependabotProxy?.RegistryBaseURLs ?? [];
+            hasPrivateRegistryBaseFeeds = privateRegistryBaseFeeds.Count > 0;
+
+            DefaultFeeds = hasPrivateRegistryBaseFeeds
+                ? privateRegistryBaseFeeds
+                : [PublicApiNugetOrgFeed];
             emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);
 
             lazyExplicitFeeds = new Lazy<ImmutableHashSet<string>>(GetExplicitFeeds);
@@ -96,13 +119,28 @@ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotP
                 var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
                 return reachableFallbackFeeds.ToImmutableHashSet();
             });
+            lazyReachableDefaultFeeds = new Lazy<ImmutableHashSet<string>>(() => CheckSpecifiedFeeds(DefaultFeeds));
         }
 
         public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider)
             : this(logger, dotnet, dependabotProxy, fileProvider, new FeedManagerIO(logger, dependabotProxy))
         {
         }
 
+        private bool IsNugetOrgFeed(string url)
+        {
+            try
+            {
+                var uri = new Uri(url);
+                return uri.Host.EndsWith(PublicDotNugetOrg, StringComparison.InvariantCultureIgnoreCase) ||
+                    string.Equals(uri.Host, PublicNugetOrg, StringComparison.InvariantCultureIgnoreCase);
+            }
+            catch (UriFormatException)
+            {
+                return false;
+            }
+        }
+
         private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
         {
             var results = getNugetFeeds();
@@ -124,10 +162,18 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
                     continue;
                 }
 
-                if (!string.IsNullOrWhiteSpace(url))
+                if (hasPrivateRegistryBaseFeeds && IsNugetOrgFeed(url))
                 {
-                    yield return url;
+                    // Use private registry base feeds.
+                    foreach (var feed in privateRegistryBaseFeeds)
+                    {
+                        logger.LogDebug($"Using private registry base feed '{feed}'.");
+                        yield return feed;
+                    }
+                    continue;
                 }
+
+                yield return url;
             }
         }
 
@@ -266,22 +312,6 @@ private ImmutableHashSet<string> CheckSpecifiedFeeds(ImmutableHashSet<string> fe
             return reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet();
         }
 
-        /// <summary>
-        /// Return true if the default NuGet feed is reachable, false otherwise.
-        /// If the reachability check is disabled, this method will always return true.
-        /// </summary>
-        /// <returns>True if the default NuGet feed is reachable, false otherwise.</returns>
-        public bool IsDefaultFeedReachable()
-        {
-            if (CheckNugetFeedResponsiveness)
-            {
-                var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
-                return feedManagerIo.IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount);
-            }
-
-            return true;
-        }
-
         /// <summary>
         /// Tests which of the feeds given by <paramref name="feedsToCheck"/> are reachable.
         /// </summary>
@@ -315,8 +345,8 @@ private List<string> GetReachableFallbackNugetFeeds()
             var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
             if (fallbackFeeds.Count == 0)
             {
-                fallbackFeeds.Add(PublicNugetOrgFeed);
-                logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}");
+                fallbackFeeds.UnionWith(DefaultFeeds);
+                logger.LogInfo($"No fallback NuGet feeds specified. Adding default feeds: {string.Join(", ", DefaultFeeds.OrderBy(f => f))}");
 
                 var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
                 logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs

@@ -1,5 +1,5 @@
 using System;
-using System.Collections.Generic;
+using System.Collections.Immutable;
 using System.Security.Cryptography.X509Certificates;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -14,7 +14,12 @@ public interface IDependabotProxy : IDisposable
         /// <summary>
         /// The URLs of package registries that are configured for the proxy.
         /// </summary>
-        HashSet<string> RegistryURLs { get; }
+        ImmutableHashSet<string> RegistryURLs { get; }
+
+        /// <summary>
+        /// The URLs of package registries that replace the base registry.
+        /// </summary>
+        ImmutableHashSet<string> RegistryBaseURLs { get; }
 
         /// <summary>
         /// The path to the temporary file where the certificate is stored.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -460,7 +460,7 @@ private bool TryRestorePackageManually(string package, List<string> nugetSources
                 return true;
             }
 
-            if (!feedManager.CheckNugetFeedResponsiveness && res.HasNugetPackageSourceError && nugetSources.Count > 0)
+            if (!feedManager.CheckNugetFeedResponsiveness && !feedManager.HasPrivateRegistryFeeds && res.HasNugetPackageSourceError && nugetSources.Count > 0)
             {
                 logger.LogDebug($"Trying to restore '{package}' without explicitly providing NuGet sources.");
                 // Restore could not be completed because the listed source is unavailable. Try without an explicit restore source argument.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs

@@ -67,10 +67,6 @@ private class NugetExeWrapper : IPackagesConfigRestore
 
             private bool IsWindows => SystemBuildActions.Instance.IsWindows();
 
-            private bool? isDefaultFeedReachable;
-            private bool IsDefaultFeedReachable =>
-                isDefaultFeedReachable ??= feedManager.IsDefaultFeedReachable();
-
             /// <summary>
             /// Create the package manager for a specified source tree.
             /// </summary>
@@ -169,15 +165,18 @@ private bool TryRestoreNugetPackage(string packagesConfig)
 
                 List<string> sourcesArgument = [];
                 var feedsToUse = feedManager.FeedsToUse(packagesConfig).ToList();
-                var useDefaultFeed = feedsToUse.Count == 0 && IsDefaultFeedReachable;
+                var defaultFeeds = feedManager.CheckNugetFeedResponsiveness
+                    ? feedManager.ReachableDefaultFeeds
+                    : feedManager.DefaultFeeds;
+                var useDefaultFeeds = feedsToUse.Count == 0 && defaultFeeds.Count > 0;
 
                 // Explicitly construct the sources to be used for the restore command when checking feed
-                // responsiveness, using private registries, or falling back to nuget.org.
-                if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeed)
+                // responsiveness, using private registries, or falling back to default feeds.
+                if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeeds)
                 {
-                    if (useDefaultFeed)
+                    if (useDefaultFeeds)
                     {
-                        feedsToUse.Add(FeedManager.PublicNugetOrgFeed);
+                        feedsToUse.AddRange(defaultFeeds);
                     }
                     var restoreFeeds = feedManager.RestoreFeeds(feedsToUse);
                     sourcesArgument = restoreFeeds.SelectMany<string, string>(feed => ["-Source", feed]).ToList();

csharp/extractor/Semmle.Extraction.Tests/DependabotProxy.cs

@@ -135,7 +135,8 @@ public void TestDependabotRegistryUrls1()
 
             // Verify
             Assert.NotNull(proxy);
-            Assert.Equal([], proxy.RegistryURLs);
+            Assert.Empty(proxy.RegistryURLs);
+            Assert.Empty(proxy.RegistryBaseURLs);
         }
 
         [Fact]
@@ -158,6 +159,7 @@ public void TestDependabotRegistryUrls2()
             Assert.Equal([
                 "https://nuget.pkg.github.com/org/index.json"
             ], proxy.RegistryURLs);
+            Assert.Empty(proxy.RegistryBaseURLs);
         }
 
         [Fact]
@@ -180,6 +182,33 @@ public void TestDependabotRegistryUrls3()
             Assert.Equal([
                 "https://example.com/org/index.json"
             ], proxy.RegistryURLs);
+            Assert.Empty(proxy.RegistryBaseURLs);
+        }
+
+        [Fact]
+        public void TestDependabotReplacesBase1()
+        {
+            // Setup
+            var config = new DependabotConfigurationStub
+            {
+                Port = "8080",
+                Host = "localhost",
+                RegistryURLs = "[ { \"type\": \"nuget_feed\", \"url\": \"https://example.com/org/index.json\", \"replaces-base\": true }, { \"type\": \"nuget_feed\", \"url\": \"https://example2.com/org/index.json\", \"replaces-base\": false } ]"
+            };
+
+            // Execute
+            using var tempWorkingDirectory = MakeTemporaryDirectory();
+            using var proxy = DependabotProxy.Make(config, new LoggerStub(), new DiagnosticsWriterStub(), tempWorkingDirectory);
+
+            // Verify
+            Assert.NotNull(proxy);
+            Assert.Equal([
+                "https://example.com/org/index.json",
+                "https://example2.com/org/index.json"
+            ], proxy.RegistryURLs);
+            Assert.Equal([
+                "https://example.com/org/index.json",
+            ], proxy.RegistryBaseURLs);
         }
     }
 }