From d2215eb130844be04027a8d50d82fc9c2bcb70b2 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 2 Sep 2026 11:11:13 -0400 Subject: [PATCH 1/4] feat: Route GVFS endpoints to dedicated cache servers Context: The microsoft/git GVFS helper supports endpoint-specific cache servers so cache infrastructure can be migrated independently. VFS for Git previously sent every protocol request to one global URL. Justification: Use the same gvfs..cache-server keys and clone option names as Scalar. Keeping endpoint preferences on CacheServerInfo centralizes precedence and lets mount-time cache resolution preserve the configured routes. Implementation: Load, persist, and validate overrides for prefetch, object GET, object POST, and sizes requests. Add matching clone options, retain the global cache as the default, preserve overrides while resolving cache identity, and cover configuration, CLI parsing, and mount resolution. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- GVFS/FastFetch/FastFetchVerb.cs | 4 +- .../GvfsMainCliTests.cs | 18 +++- GVFS/GVFS.Common/GVFSConstants.cs | 4 + GVFS/GVFS.Common/Http/CacheServerInfo.cs | 71 +++++++++++++- GVFS/GVFS.Common/Http/CacheServerResolver.cs | 37 +++++++- .../Http/GitObjectsHttpRequestor.cs | 6 +- GVFS/GVFS.Mount/InProcessMount.cs | 5 +- .../Common/CacheServerResolverTests.cs | 94 ++++++++++++++++++- GVFS/GVFS/CommandLine/CloneVerb.cs | 38 ++++++++ GVFS/GVFS/CommandLine/GVFSVerb.cs | 3 +- GVFS/GVFS/CommandLine/PrefetchVerb.cs | 4 +- 11 files changed, 272 insertions(+), 12 deletions(-) diff --git a/GVFS/FastFetch/FastFetchVerb.cs b/GVFS/FastFetch/FastFetchVerb.cs index 737b31ffe3..cf6add3aad 100644 --- a/GVFS/FastFetch/FastFetchVerb.cs +++ b/GVFS/FastFetch/FastFetchVerb.cs @@ -237,7 +237,9 @@ private int ExecuteWithExitCode() string fastfetchLogFile = Enlistment.GetNewLogFileName(enlistment.FastFetchLogRoot, "fastfetch"); tracer.AddLogFileEventListener(fastfetchLogFile, EventLevel.Informational, Keywords.Any); - CacheServerInfo cacheServer = new CacheServerInfo(this.GetRemoteUrl(enlistment), null); + CacheServerInfo cacheServer = string.IsNullOrWhiteSpace(this.CacheServerUrl) + ? CacheServerResolver.GetCacheServerFromConfig(enlistment) + : new CacheServerInfo(this.GetRemoteUrl(enlistment), null); tracer.WriteStartEvent( enlistment.PrimaryEnlistmentRoot, diff --git a/GVFS/GVFS.CommandLine.Tests/GvfsMainCliTests.cs b/GVFS/GVFS.CommandLine.Tests/GvfsMainCliTests.cs index 4eb360808e..27e117d216 100644 --- a/GVFS/GVFS.CommandLine.Tests/GvfsMainCliTests.cs +++ b/GVFS/GVFS.CommandLine.Tests/GvfsMainCliTests.cs @@ -215,6 +215,10 @@ public void Clone_FullCommandLine_ParsesCorrectly() { "clone", "https://example.com/repo", @"C:\Users\test\repo", "--cache-server-url", "https://cache.test", + "--prefetch-cache-server-url", "https://prefetch-cache.test", + "--get-cache-server-url", "https://get-cache.test", + "--post-cache-server-url", "https://post-cache.test", + "--sizes-cache-server-url", "https://sizes-cache.test", "-b", "develop", "--single-branch", "--no-mount", @@ -342,7 +346,19 @@ public void Repair_FullCommandLine_ParsesCorrectly() [Test] public void Clone_HasAllExpectedOptions() { - var expected = new[] { "--cache-server-url", "--branch", "--single-branch", "--no-mount", "--no-prefetch", "--local-cache-path" }; + var expected = new[] + { + "--cache-server-url", + "--prefetch-cache-server-url", + "--get-cache-server-url", + "--post-cache-server-url", + "--sizes-cache-server-url", + "--branch", + "--single-branch", + "--no-mount", + "--no-prefetch", + "--local-cache-path", + }; foreach (var optName in expected) { Assert.That(FindOptionOnCommand("clone", optName), Is.Not.Null, diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 143b59e693..088d771957 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -33,6 +33,10 @@ public static class GitConfig public const string MountId = GVFSPrefix + "mount-id"; public const string EnlistmentId = GVFSPrefix + "enlistment-id"; public const string CacheServer = GVFSPrefix + "cache-server"; + public const string PrefetchCacheServer = GVFSPrefix + "prefetch.cache-server"; + public const string GetCacheServer = GVFSPrefix + "get.cache-server"; + public const string PostCacheServer = GVFSPrefix + "post.cache-server"; + public const string SizesCacheServer = GVFSPrefix + "sizes.cache-server"; public const string DeprecatedCacheEndpointSuffix = ".cache-server-url"; public const string HooksPrefix = GitConfig.GVFSPrefix + "clone.default-"; public const string GVFSTelemetryId = GitConfig.GVFSPrefix + "telemetry-id"; diff --git a/GVFS/GVFS.Common/Http/CacheServerInfo.cs b/GVFS/GVFS.Common/Http/CacheServerInfo.cs index 0ec929b0dc..33e1b61e70 100644 --- a/GVFS/GVFS.Common/Http/CacheServerInfo.cs +++ b/GVFS/GVFS.Common/Http/CacheServerInfo.cs @@ -11,27 +11,89 @@ public class CacheServerInfo [JsonConstructor] public CacheServerInfo(string url, string name, bool globalDefault = false) + : this(url, name, globalDefault, null, null, null, null) + { + } + + public CacheServerInfo( + string url, + string name, + bool globalDefault, + string prefetchCacheServerUrl, + string getCacheServerUrl, + string postCacheServerUrl, + string sizesCacheServerUrl) { this.Url = url; this.Name = name; this.GlobalDefault = globalDefault; + this.PrefetchCacheServerUrl = prefetchCacheServerUrl; + this.GetCacheServerUrl = getCacheServerUrl; + this.PostCacheServerUrl = postCacheServerUrl; + this.SizesCacheServerUrl = sizesCacheServerUrl; if (this.Url != null) { this.ObjectsEndpointUrl = this.Url + ObjectsEndpointSuffix; - this.PrefetchEndpointUrl = this.Url + PrefetchEndpointSuffix; - this.SizesEndpointUrl = this.Url + SizesEndpointSuffix; } + + this.PrefetchEndpointUrl = GetEndpointUrl(prefetchCacheServerUrl ?? this.Url, PrefetchEndpointSuffix); + this.ObjectsGetEndpointUrl = GetEndpointUrl(getCacheServerUrl ?? this.Url, ObjectsEndpointSuffix); + this.ObjectsPostEndpointUrl = GetEndpointUrl(postCacheServerUrl ?? this.Url, ObjectsEndpointSuffix); + this.SizesEndpointUrl = GetEndpointUrl(sizesCacheServerUrl ?? this.Url, SizesEndpointSuffix); } public string Url { get; } public string Name { get; } public bool GlobalDefault { get; } + [JsonIgnore] + public string PrefetchCacheServerUrl { get; } + + [JsonIgnore] + public string GetCacheServerUrl { get; } + + [JsonIgnore] + public string PostCacheServerUrl { get; } + + [JsonIgnore] + public string SizesCacheServerUrl { get; } + public string ObjectsEndpointUrl { get; } public string PrefetchEndpointUrl { get; } public string SizesEndpointUrl { get; } + [JsonIgnore] + public string ObjectsGetEndpointUrl { get; } + + [JsonIgnore] + public string ObjectsPostEndpointUrl { get; } + + public CacheServerInfo WithEndpointOverrides( + string prefetchCacheServerUrl, + string getCacheServerUrl, + string postCacheServerUrl, + string sizesCacheServerUrl) + { + return new CacheServerInfo( + this.Url, + this.Name, + this.GlobalDefault, + prefetchCacheServerUrl, + getCacheServerUrl, + postCacheServerUrl, + sizesCacheServerUrl); + } + + public CacheServerInfo WithEndpointOverridesFrom(CacheServerInfo cacheServer) + { + return this.WithEndpointOverrides( + cacheServer.PrefetchCacheServerUrl, + cacheServer.GetCacheServerUrl, + cacheServer.PostCacheServerUrl, + cacheServer.SizesCacheServerUrl); + } + public bool HasValidUrl() { return Uri.IsWellFormedUriString(this.Url, UriKind.Absolute); @@ -64,5 +126,10 @@ public static class ReservedNames public const string Default = "Default"; public const string UserDefined = "User Defined"; } + + private static string GetEndpointUrl(string cacheServerUrl, string endpointSuffix) + { + return cacheServerUrl == null ? null : cacheServerUrl + endpointSuffix; + } } } diff --git a/GVFS/GVFS.Common/Http/CacheServerResolver.cs b/GVFS/GVFS.Common/Http/CacheServerResolver.cs index bc1df9727b..26037c0a75 100644 --- a/GVFS/GVFS.Common/Http/CacheServerResolver.cs +++ b/GVFS/GVFS.Common/Http/CacheServerResolver.cs @@ -20,10 +20,16 @@ public CacheServerResolver( public static CacheServerInfo GetCacheServerFromConfig(Enlistment enlistment) { + GitProcess git = enlistment.CreateGitProcess(); string url = GetUrlFromConfig(enlistment); return new CacheServerInfo( url, - url == enlistment.RepoUrl ? CacheServerInfo.ReservedNames.None : null); + url == enlistment.RepoUrl ? CacheServerInfo.ReservedNames.None : null, + globalDefault: false, + GetValueFromConfig(git, GVFSConstants.GitConfig.PrefetchCacheServer, localOnly: true), + GetValueFromConfig(git, GVFSConstants.GitConfig.GetCacheServer, localOnly: true), + GetValueFromConfig(git, GVFSConstants.GitConfig.PostCacheServer, localOnly: true), + GetValueFromConfig(git, GVFSConstants.GitConfig.SizesCacheServer, localOnly: true)); } public static string GetUrlFromConfig(Enlistment enlistment) @@ -129,6 +135,22 @@ public bool TrySaveUrlToLocalConfig(CacheServerInfo cache, out string error) return result.ExitCodeIsSuccess; } + public bool TrySaveEndpointUrlsToLocalConfig(CacheServerInfo cache, out string error) + { + GitProcess git = this.enlistment.CreateGitProcess(); + + if (!TrySaveEndpointUrl(git, GVFSConstants.GitConfig.PrefetchCacheServer, cache.PrefetchCacheServerUrl, out error) || + !TrySaveEndpointUrl(git, GVFSConstants.GitConfig.GetCacheServer, cache.GetCacheServerUrl, out error) || + !TrySaveEndpointUrl(git, GVFSConstants.GitConfig.PostCacheServer, cache.PostCacheServerUrl, out error) || + !TrySaveEndpointUrl(git, GVFSConstants.GitConfig.SizesCacheServer, cache.SizesCacheServerUrl, out error)) + { + return false; + } + + error = null; + return true; + } + private static string GetValueFromConfig(GitProcess git, string configName, bool localOnly) { GitProcess.ConfigResult result = @@ -144,6 +166,19 @@ private static string GetValueFromConfig(GitProcess git, string configName, bool return value; } + private static bool TrySaveEndpointUrl(GitProcess git, string configName, string url, out string error) + { + error = null; + if (url == null) + { + return true; + } + + GitProcess.Result result = git.SetInLocalConfig(configName, url, replaceAll: true); + error = result.Errors; + return result.ExitCodeIsSuccess; + } + private static string GetDeprecatedCacheConfigSettingName(Enlistment enlistment) { string sectionUrl = diff --git a/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs b/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs index 2cdffcb8da..45e4ba8c4f 100644 --- a/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs @@ -149,7 +149,7 @@ public virtual RetryWrapper.InvocationResult TryDownloadLoo onSuccess, eArgs => this.HandleDownloadAndSaveObjectError(retryOnFailure, requestId, eArgs), HttpMethod.Get, - new Uri(this.CacheServer.ObjectsEndpointUrl + "/" + objectId), + new Uri(this.CacheServer.ObjectsGetEndpointUrl + "/" + objectId), cancellationToken, requestBody: null, acceptType: null, @@ -170,7 +170,7 @@ public virtual RetryWrapper.InvocationResult TryDownloadObj onSuccess, onFailure, HttpMethod.Post, - new Uri(this.CacheServer.ObjectsEndpointUrl), + new Uri(this.CacheServer.ObjectsPostEndpointUrl), CancellationToken.None, () => this.ObjectIdsJsonGenerator(requestId, objectIdGenerator), preferBatchedLooseObjects ? CustomLooseObjectsHeader : null); @@ -204,7 +204,7 @@ public virtual RetryWrapper.InvocationResult TryDownloadObj onSuccess, onFailure, HttpMethod.Post, - new Uri(this.CacheServer.ObjectsEndpointUrl), + new Uri(this.CacheServer.ObjectsPostEndpointUrl), CancellationToken.None, objectIdsJson, preferBatchedLooseObjects ? CustomLooseObjectsHeader : null); diff --git a/GVFS/GVFS.Mount/InProcessMount.cs b/GVFS/GVFS.Mount/InProcessMount.cs index 629be66138..0a5c930b5f 100644 --- a/GVFS/GVFS.Mount/InProcessMount.cs +++ b/GVFS/GVFS.Mount/InProcessMount.cs @@ -353,7 +353,10 @@ private void MountWithLockAcquired(EventLevel verbosity, Keywords keywords) this.mountProgressMessage = "Resolving cache server"; CacheServerResolver cacheServerResolver = new CacheServerResolver(this.tracer, this.enlistment); - this.cacheServer = cacheServerResolver.ResolveNameFromRemote(this.cacheServer.Url, serverGVFSConfig); + CacheServerInfo cacheServerFromConfig = this.cacheServer; + this.cacheServer = cacheServerResolver + .ResolveNameFromRemote(cacheServerFromConfig.Url, serverGVFSConfig) + .WithEndpointOverridesFrom(cacheServerFromConfig); this.tracer.RelatedEvent( EventLevel.Informational, diff --git a/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs b/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs index 852ecb908a..651e05343b 100644 --- a/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs +++ b/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs @@ -13,6 +13,10 @@ public class CacheServerResolverTests { private const string CacheServerUrl = "https://cache/server"; private const string CacheServerName = "TestCacheServer"; + private const string PrefetchCacheServerUrl = "https://prefetch-cache/server"; + private const string GetCacheServerUrl = "https://get-cache/server"; + private const string PostCacheServerUrl = "https://post-cache/server"; + private const string SizesCacheServerUrl = "https://sizes-cache/server"; [TestCase] public void CanGetCacheServerFromNewConfig() @@ -43,6 +47,76 @@ public void CanGetCacheServerWithNoConfig() CacheServerResolver.GetUrlFromConfig(enlistment).ShouldEqual(enlistment.RepoUrl); } + [TestCase] + public void EndpointSpecificCacheServersOverrideGlobalCacheServer() + { + MockGVFSEnlistment enlistment = this.CreateEnlistment( + CacheServerUrl, + prefetchCacheServerUrl: PrefetchCacheServerUrl, + getCacheServerUrl: GetCacheServerUrl, + postCacheServerUrl: PostCacheServerUrl, + sizesCacheServerUrl: SizesCacheServerUrl); + + CacheServerInfo cacheServer = CacheServerResolver.GetCacheServerFromConfig(enlistment); + + cacheServer.PrefetchEndpointUrl.ShouldEqual(PrefetchCacheServerUrl + "/gvfs/prefetch"); + cacheServer.ObjectsGetEndpointUrl.ShouldEqual(GetCacheServerUrl + "/gvfs/objects"); + cacheServer.ObjectsPostEndpointUrl.ShouldEqual(PostCacheServerUrl + "/gvfs/objects"); + cacheServer.SizesEndpointUrl.ShouldEqual(SizesCacheServerUrl + "/gvfs/sizes"); + } + + [TestCase] + public void EndpointSpecificCacheServersFallBackToGlobalCacheServer() + { + CacheServerInfo cacheServer = CacheServerResolver.GetCacheServerFromConfig(this.CreateEnlistment(CacheServerUrl)); + + cacheServer.PrefetchEndpointUrl.ShouldEqual(CacheServerUrl + "/gvfs/prefetch"); + cacheServer.ObjectsGetEndpointUrl.ShouldEqual(CacheServerUrl + "/gvfs/objects"); + cacheServer.ObjectsPostEndpointUrl.ShouldEqual(CacheServerUrl + "/gvfs/objects"); + cacheServer.SizesEndpointUrl.ShouldEqual(CacheServerUrl + "/gvfs/sizes"); + } + + [TestCase] + public void EndpointSpecificCacheServersArePreservedWhenGlobalCacheServerIsResolved() + { + CacheServerInfo configuredCacheServer = new CacheServerInfo(CacheServerUrl, CacheServerName) + .WithEndpointOverrides(PrefetchCacheServerUrl, GetCacheServerUrl, PostCacheServerUrl, SizesCacheServerUrl); + CacheServerInfo resolvedCacheServer = new CacheServerInfo("https://resolved-cache/server", "ResolvedCache") + .WithEndpointOverridesFrom(configuredCacheServer); + + resolvedCacheServer.PrefetchCacheServerUrl.ShouldEqual(PrefetchCacheServerUrl); + resolvedCacheServer.GetCacheServerUrl.ShouldEqual(GetCacheServerUrl); + resolvedCacheServer.PostCacheServerUrl.ShouldEqual(PostCacheServerUrl); + resolvedCacheServer.SizesCacheServerUrl.ShouldEqual(SizesCacheServerUrl); + resolvedCacheServer.HasValidUrl().ShouldEqual(true); + } + + [TestCase] + public void CanSaveEndpointSpecificCacheServers() + { + MockGVFSEnlistment enlistment = this.CreateEnlistment(); + MockGitProcess git = (MockGitProcess)enlistment.CreateGitProcess(); + git.SetExpectedCommandResult( + "config --local --replace-all \"gvfs.prefetch.cache-server\" \"https://prefetch-cache/server\"", + () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.SuccessCode)); + git.SetExpectedCommandResult( + "config --local --replace-all \"gvfs.get.cache-server\" \"https://get-cache/server\"", + () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.SuccessCode)); + git.SetExpectedCommandResult( + "config --local --replace-all \"gvfs.post.cache-server\" \"https://post-cache/server\"", + () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.SuccessCode)); + git.SetExpectedCommandResult( + "config --local --replace-all \"gvfs.sizes.cache-server\" \"https://sizes-cache/server\"", + () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.SuccessCode)); + + CacheServerInfo cacheServer = new CacheServerInfo(CacheServerUrl, CacheServerName) + .WithEndpointOverrides(PrefetchCacheServerUrl, GetCacheServerUrl, PostCacheServerUrl, SizesCacheServerUrl); + + new CacheServerResolver(new MockTracer(), enlistment) + .TrySaveEndpointUrlsToLocalConfig(cacheServer, out string error) + .ShouldEqual(true, error); + } + [TestCase] public void CanResolveUrlForKnownName() { @@ -190,7 +264,13 @@ private void ValidateIsNone(Enlistment enlistment, CacheServerInfo cacheServer) cacheServer.Name.ShouldEqual(CacheServerInfo.ReservedNames.None); } - private MockGVFSEnlistment CreateEnlistment(string newConfigValue = null, string oldConfigValue = null) + private MockGVFSEnlistment CreateEnlistment( + string newConfigValue = null, + string oldConfigValue = null, + string prefetchCacheServerUrl = null, + string getCacheServerUrl = null, + string postCacheServerUrl = null, + string sizesCacheServerUrl = null) { MockGitProcess gitProcess = new MockGitProcess(); gitProcess.SetExpectedCommandResult( @@ -199,6 +279,18 @@ private MockGVFSEnlistment CreateEnlistment(string newConfigValue = null, string gitProcess.SetExpectedCommandResult( "config gvfs.mock:..repourl.cache-server-url", () => new GitProcess.Result(oldConfigValue ?? string.Empty, string.Empty, oldConfigValue != null ? GitProcess.Result.SuccessCode : GitProcess.Result.GenericFailureCode)); + gitProcess.SetExpectedCommandResult( + "config --local gvfs.prefetch.cache-server", + () => new GitProcess.Result(prefetchCacheServerUrl ?? string.Empty, string.Empty, prefetchCacheServerUrl != null ? GitProcess.Result.SuccessCode : GitProcess.Result.GenericFailureCode)); + gitProcess.SetExpectedCommandResult( + "config --local gvfs.get.cache-server", + () => new GitProcess.Result(getCacheServerUrl ?? string.Empty, string.Empty, getCacheServerUrl != null ? GitProcess.Result.SuccessCode : GitProcess.Result.GenericFailureCode)); + gitProcess.SetExpectedCommandResult( + "config --local gvfs.post.cache-server", + () => new GitProcess.Result(postCacheServerUrl ?? string.Empty, string.Empty, postCacheServerUrl != null ? GitProcess.Result.SuccessCode : GitProcess.Result.GenericFailureCode)); + gitProcess.SetExpectedCommandResult( + "config --local gvfs.sizes.cache-server", + () => new GitProcess.Result(sizesCacheServerUrl ?? string.Empty, string.Empty, sizesCacheServerUrl != null ? GitProcess.Result.SuccessCode : GitProcess.Result.GenericFailureCode)); return new MockGVFSEnlistment(gitProcess); } diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 1277355ab6..370f40b30a 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -24,6 +24,14 @@ public class CloneVerb : GVFSVerb public string CacheServerUrl { get; set; } + public string PrefetchCacheServerUrl { get; set; } + + public string GetCacheServerUrl { get; set; } + + public string PostCacheServerUrl { get; set; } + + public string SizesCacheServerUrl { get; set; } + public string Branch { get; set; } public bool SingleBranch { get; set; } @@ -56,6 +64,18 @@ public static System.CommandLine.Command CreateCommand() System.CommandLine.Option cacheServerOption = new System.CommandLine.Option("--cache-server-url") { Description = "The url or friendly name of the cache server" }; cmd.Add(cacheServerOption); + System.CommandLine.Option prefetchCacheServerOption = new System.CommandLine.Option("--prefetch-cache-server-url") { Description = "The cache server URL for the prefetch endpoint" }; + cmd.Add(prefetchCacheServerOption); + + System.CommandLine.Option getCacheServerOption = new System.CommandLine.Option("--get-cache-server-url") { Description = "The cache server URL for the objects GET endpoint" }; + cmd.Add(getCacheServerOption); + + System.CommandLine.Option postCacheServerOption = new System.CommandLine.Option("--post-cache-server-url") { Description = "The cache server URL for the objects POST endpoint" }; + cmd.Add(postCacheServerOption); + + System.CommandLine.Option sizesCacheServerOption = new System.CommandLine.Option("--sizes-cache-server-url") { Description = "The cache server URL for the sizes endpoint" }; + cmd.Add(sizesCacheServerOption); + System.CommandLine.Option branchOption = new System.CommandLine.Option("--branch", new[] { "-b" }) { Description = "Branch to checkout after clone" }; cmd.Add(branchOption); @@ -86,6 +106,10 @@ public static System.CommandLine.Command CreateCommand() } verb.CacheServerUrl = result.GetValue(cacheServerOption); + verb.PrefetchCacheServerUrl = result.GetValue(prefetchCacheServerOption); + verb.GetCacheServerUrl = result.GetValue(getCacheServerOption); + verb.PostCacheServerUrl = result.GetValue(postCacheServerOption); + verb.SizesCacheServerUrl = result.GetValue(sizesCacheServerOption); verb.Branch = result.GetValue(branchOption); verb.SingleBranch = result.GetValue(singleBranchOption); verb.NoMount = result.GetValue(noMountOption); @@ -144,6 +168,10 @@ public override void Execute() this.CheckKernelDriverSupported(normalizedEnlistmentRootPath); this.CheckNotInsideExistingRepo(normalizedEnlistmentRootPath); this.BlockEmptyCacheServerUrl(this.CacheServerUrl); + this.BlockEmptyCacheServerUrl(this.PrefetchCacheServerUrl); + this.BlockEmptyCacheServerUrl(this.GetCacheServerUrl); + this.BlockEmptyCacheServerUrl(this.PostCacheServerUrl); + this.BlockEmptyCacheServerUrl(this.SizesCacheServerUrl); try { @@ -231,6 +259,11 @@ public override void Execute() } cacheServer = this.ResolveCacheServer(tracer, cacheServer, cacheServerResolver, serverGVFSConfig); + cacheServer = cacheServer.WithEndpointOverrides( + this.PrefetchCacheServerUrl, + this.GetCacheServerUrl, + this.PostCacheServerUrl, + this.SizesCacheServerUrl); this.ValidateClientVersions(tracer, enlistment, serverGVFSConfig, showWarnings: true); @@ -643,6 +676,11 @@ private Result CreateClone( return new Result("Unable to configure cache server: " + errorMessage); } + if (!cacheServerResolver.TrySaveEndpointUrlsToLocalConfig(objectRequestor.CacheServer, out errorMessage)) + { + return new Result("Unable to configure endpoint-specific cache servers: " + errorMessage); + } + GitProcess git = new GitProcess(enlistment); string originBranchName = "origin/" + branch; GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName); diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 51b693578d..84a4c678bf 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -475,6 +475,7 @@ protected CacheServerInfo ResolveCacheServer( resolvedCacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServer.Url, serverGVFSConfig); } + resolvedCacheServer = resolvedCacheServer.WithEndpointOverridesFrom(cacheServer); this.Output.WriteLine("Using cache server: " + resolvedCacheServer); return resolvedCacheServer; } @@ -526,7 +527,7 @@ protected bool TryDownloadCommit( if (!gitObjects.TryDownloadCommit(commitId)) { - error = "Could not download commit " + commitId + " from: " + Uri.EscapeDataString(objectRequestor.CacheServer.ObjectsEndpointUrl); + error = "Could not download commit " + commitId + " from: " + Uri.EscapeDataString(objectRequestor.CacheServer.ObjectsPostEndpointUrl); return false; } diff --git a/GVFS/GVFS/CommandLine/PrefetchVerb.cs b/GVFS/GVFS/CommandLine/PrefetchVerb.cs index 6fa0d91f42..f82f4c0898 100644 --- a/GVFS/GVFS/CommandLine/PrefetchVerb.cs +++ b/GVFS/GVFS/CommandLine/PrefetchVerb.cs @@ -340,7 +340,9 @@ private void InitializeServerConnection( CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); - resolvedCacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerFromConfig.Url, serverGVFSConfig); + resolvedCacheServer = cacheServerResolver + .ResolveNameFromRemote(cacheServerFromConfig.Url, serverGVFSConfig) + .WithEndpointOverridesFrom(cacheServerFromConfig); if (!this.SkipVersionCheck) { From d423ed1e208a38e1db7d39ed77f7ba29f72527ff Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 2 Sep 2026 11:11:59 -0400 Subject: [PATCH 2/4] fix: Fall back safely from dedicated cache endpoints Context: Endpoint-specific cache servers are preferences above the global cache, but failures previously terminated requests instead of using the healthy fallback route. Early fallback handling also charged abandoned attempts to the process-wide circuit breaker, confused cancellation with transport failure, and exposed excess URI data in telemetry. Justification: Treat route failover separately from transient retry accounting. Cancellation remains control flow, local processing errors stay on the active route, and network-body failures alone can move a request to the global cache. Authority-only metadata preserves diagnostics without exposing credentials or request details. Implementation: Fall back prefetch, object GET, object POST, and sizes requests through the global cache, with sizes retaining its final origin fallback. Track response-stream failures, preserve circuit-breaker budget across route transitions, propagate cancellation unchanged, validate endpoint URLs, and emit redacted fallback telemetry. Add focused coverage for HTTP, transport, body-read, local-write, cancellation, telemetry, and terminal failures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../GvfsMainCliTests.cs | 20 + GVFS/GVFS.Common/Git/GitObjects.cs | 12 +- GVFS/GVFS.Common/Http/CacheServerInfo.cs | 15 +- GVFS/GVFS.Common/Http/CacheServerResolver.cs | 23 +- .../Http/GitEndPointResponseData.cs | 91 +++- .../Http/GitObjectsHttpRequestor.cs | 353 +++++++++++-- GVFS/GVFS.Common/Http/HttpRequestor.cs | 9 +- GVFS/GVFS.Common/RetryWrapper.cs | 10 +- .../Common/CacheServerResolverTests.cs | 14 + .../Http/GitObjectsHttpRequestorTests.cs | 485 ++++++++++++++++++ .../GVFS.UnitTests/Http/HttpRequestorTests.cs | 11 +- GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs | 8 + GVFS/GVFS/CommandLine/CloneVerb.cs | 29 ++ 13 files changed, 1020 insertions(+), 60 deletions(-) create mode 100644 GVFS/GVFS.UnitTests/Http/GitObjectsHttpRequestorTests.cs diff --git a/GVFS/GVFS.CommandLine.Tests/GvfsMainCliTests.cs b/GVFS/GVFS.CommandLine.Tests/GvfsMainCliTests.cs index 27e117d216..5cb6be2407 100644 --- a/GVFS/GVFS.CommandLine.Tests/GvfsMainCliTests.cs +++ b/GVFS/GVFS.CommandLine.Tests/GvfsMainCliTests.cs @@ -225,6 +225,26 @@ public void Clone_FullCommandLine_ParsesCorrectly() "--no-prefetch" }); Assert.That(parseResult.Errors, Is.Empty, "Full clone command should parse without errors"); + Assert.Multiple(() => + { + Assert.That(parseResult.GetValue((Option)FindOptionOnCommand("clone", "--cache-server-url")), Is.EqualTo("https://cache.test")); + Assert.That(parseResult.GetValue((Option)FindOptionOnCommand("clone", "--prefetch-cache-server-url")), Is.EqualTo("https://prefetch-cache.test")); + Assert.That(parseResult.GetValue((Option)FindOptionOnCommand("clone", "--get-cache-server-url")), Is.EqualTo("https://get-cache.test")); + Assert.That(parseResult.GetValue((Option)FindOptionOnCommand("clone", "--post-cache-server-url")), Is.EqualTo("https://post-cache.test")); + Assert.That(parseResult.GetValue((Option)FindOptionOnCommand("clone", "--sizes-cache-server-url")), Is.EqualTo("https://sizes-cache.test")); + }); + } + + [TestCase("--prefetch-cache-server-url")] + [TestCase("--get-cache-server-url")] + [TestCase("--post-cache-server-url")] + [TestCase("--sizes-cache-server-url")] + public void Clone_EndpointCacheServerUrl_RejectsInvalidUrl(string optionName) + { + var parseResult = rootCommand.Parse(new[] { "clone", "https://example.com/repo", optionName, "not-a-url" }); + + Assert.That(parseResult.Errors, Has.Count.EqualTo(1)); + Assert.That(parseResult.Errors[0].Message, Does.Contain("requires an absolute URL")); } [Test] diff --git a/GVFS/GVFS.Common/Git/GitObjects.cs b/GVFS/GVFS.Common/Git/GitObjects.cs index a9b0f2851a..19c1a0e608 100644 --- a/GVFS/GVFS.Common/Git/GitObjects.cs +++ b/GVFS/GVFS.Common/Git/GitObjects.cs @@ -180,6 +180,11 @@ public virtual bool TryDownloadPrefetchPacks(GitProcess gitProcess, long latestT "{0}?lastPackTimestamp={1}", this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl, latestTimestamp)), + fallbackEndPointGenerator: () => new Uri( + string.Format( + "{0}?lastPackTimestamp={1}", + this.GitObjectRequestor.CacheServer.GlobalPrefetchEndpointUrl, + latestTimestamp)), requestBodyGenerator: () => null, cancellationToken: CancellationToken.None, acceptType: new MediaTypeWithQualityHeaderValue(GVFSConstants.MediaTypes.PrefetchPackFilesAndIndexesMediaType)); @@ -188,18 +193,21 @@ public virtual bool TryDownloadPrefetchPacks(GitProcess gitProcess, long latestT if (!result.Succeeded) { + Uri requestUri = result.Result?.RequestUri + ?? new Uri(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl); + string requestAuthority = HttpRequestor.GetAuthorityForTelemetry(requestUri); if (result.Result != null && result.Result.HttpStatusCodeResult == HttpStatusCode.NotFound) { EventMetadata warning = CreateEventMetadata(); warning.Add(TracingConstants.MessageKey.WarningMessage, "The server does not support " + GVFSConstants.Endpoints.GVFSPrefetch); - warning.Add(nameof(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl), this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl); + warning.Add("PrefetchEndpointUrl", requestAuthority); activity.RelatedEvent(EventLevel.Warning, "CommandNotSupported", warning); } else { EventMetadata error = CreateEventMetadata(result.Error); error.Add("latestTimestamp", latestTimestamp); - error.Add(nameof(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl), this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl); + error.Add("PrefetchEndpointUrl", requestAuthority); activity.RelatedWarning(error, "DownloadPrefetchPacks failed.", Keywords.Telemetry); } } diff --git a/GVFS/GVFS.Common/Http/CacheServerInfo.cs b/GVFS/GVFS.Common/Http/CacheServerInfo.cs index 33e1b61e70..9df0399e42 100644 --- a/GVFS/GVFS.Common/Http/CacheServerInfo.cs +++ b/GVFS/GVFS.Common/Http/CacheServerInfo.cs @@ -37,6 +37,8 @@ public CacheServerInfo( this.ObjectsEndpointUrl = this.Url + ObjectsEndpointSuffix; } + this.GlobalPrefetchEndpointUrl = GetEndpointUrl(this.Url, PrefetchEndpointSuffix); + this.GlobalSizesEndpointUrl = GetEndpointUrl(this.Url, SizesEndpointSuffix); this.PrefetchEndpointUrl = GetEndpointUrl(prefetchCacheServerUrl ?? this.Url, PrefetchEndpointSuffix); this.ObjectsGetEndpointUrl = GetEndpointUrl(getCacheServerUrl ?? this.Url, ObjectsEndpointSuffix); this.ObjectsPostEndpointUrl = GetEndpointUrl(postCacheServerUrl ?? this.Url, ObjectsEndpointSuffix); @@ -69,6 +71,12 @@ public CacheServerInfo( [JsonIgnore] public string ObjectsPostEndpointUrl { get; } + [JsonIgnore] + public string GlobalPrefetchEndpointUrl { get; } + + [JsonIgnore] + public string GlobalSizesEndpointUrl { get; } + public CacheServerInfo WithEndpointOverrides( string prefetchCacheServerUrl, string getCacheServerUrl, @@ -96,7 +104,12 @@ public CacheServerInfo WithEndpointOverridesFrom(CacheServerInfo cacheServer) public bool HasValidUrl() { - return Uri.IsWellFormedUriString(this.Url, UriKind.Absolute); + return IsValidUrl(this.Url); + } + + public static bool IsValidUrl(string url) + { + return Uri.IsWellFormedUriString(url, UriKind.Absolute); } public bool IsNone(string repoUrl) diff --git a/GVFS/GVFS.Common/Http/CacheServerResolver.cs b/GVFS/GVFS.Common/Http/CacheServerResolver.cs index 26037c0a75..7a0c6e1a73 100644 --- a/GVFS/GVFS.Common/Http/CacheServerResolver.cs +++ b/GVFS/GVFS.Common/Http/CacheServerResolver.cs @@ -22,14 +22,18 @@ public static CacheServerInfo GetCacheServerFromConfig(Enlistment enlistment) { GitProcess git = enlistment.CreateGitProcess(); string url = GetUrlFromConfig(enlistment); + string prefetchCacheServerUrl = GetEndpointUrlFromConfig(git, GVFSConstants.GitConfig.PrefetchCacheServer); + string getCacheServerUrl = GetEndpointUrlFromConfig(git, GVFSConstants.GitConfig.GetCacheServer); + string postCacheServerUrl = GetEndpointUrlFromConfig(git, GVFSConstants.GitConfig.PostCacheServer); + string sizesCacheServerUrl = GetEndpointUrlFromConfig(git, GVFSConstants.GitConfig.SizesCacheServer); return new CacheServerInfo( url, url == enlistment.RepoUrl ? CacheServerInfo.ReservedNames.None : null, globalDefault: false, - GetValueFromConfig(git, GVFSConstants.GitConfig.PrefetchCacheServer, localOnly: true), - GetValueFromConfig(git, GVFSConstants.GitConfig.GetCacheServer, localOnly: true), - GetValueFromConfig(git, GVFSConstants.GitConfig.PostCacheServer, localOnly: true), - GetValueFromConfig(git, GVFSConstants.GitConfig.SizesCacheServer, localOnly: true)); + prefetchCacheServerUrl, + getCacheServerUrl, + postCacheServerUrl, + sizesCacheServerUrl); } public static string GetUrlFromConfig(Enlistment enlistment) @@ -166,6 +170,17 @@ private static string GetValueFromConfig(GitProcess git, string configName, bool return value; } + private static string GetEndpointUrlFromConfig(GitProcess git, string configName) + { + string url = GetValueFromConfig(git, configName, localOnly: true); + if (url != null && !CacheServerInfo.IsValidUrl(url)) + { + throw new InvalidRepoException($"Invalid value for {configName}: '{url}' is not an absolute URL."); + } + + return url; + } + private static bool TrySaveEndpointUrl(GitProcess git, string configName, string url, out string error) { error = null; diff --git a/GVFS/GVFS.Common/Http/GitEndPointResponseData.cs b/GVFS/GVFS.Common/Http/GitEndPointResponseData.cs index 0d450bf9d1..9eb6164219 100644 --- a/GVFS/GVFS.Common/Http/GitEndPointResponseData.cs +++ b/GVFS/GVFS.Common/Http/GitEndPointResponseData.cs @@ -4,6 +4,8 @@ using System.IO; using System.Net; using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; namespace GVFS.Common.Http { @@ -30,7 +32,7 @@ public GitEndPointResponseData(HttpStatusCode statusCode, Exception error, bool public GitEndPointResponseData(HttpStatusCode statusCode, string contentType, Stream responseStream, HttpResponseMessage message, Action onResponseDisposed) : this(statusCode, null, false, message, onResponseDisposed) { - this.Stream = responseStream; + this.Stream = responseStream == null ? null : new ReadErrorTrackingStream(responseStream); this.ContentType = MapContentType(contentType); } @@ -42,6 +44,11 @@ public GitEndPointResponseData(HttpStatusCode statusCode, string contentType, St public Stream Stream { get; private set; } + public bool StreamReadFailed + { + get { return this.Stream is ReadErrorTrackingStream trackingStream && trackingStream.ReadFailed; } + } + public bool HasErrors { get { return this.StatusCode != HttpStatusCode.OK; } @@ -70,7 +77,7 @@ public string RetryableReadToEnd() { return contentStreamReader.ReadToEnd(); } - catch (Exception ex) + catch (Exception ex) when (!(ex is OperationCanceledException)) { // All exceptions potentially from network should be retried throw new RetryableException("Exception while reading stream. See inner exception for details.", ex); @@ -99,7 +106,7 @@ public List RetryableReadAllLines() line = contentStreamReader.ReadLine(); } - catch (Exception ex) + catch (Exception ex) when (!(ex is OperationCanceledException)) { // All exceptions potentially from network should be retried throw new RetryableException("Exception while reading stream. See inner exception for details.", ex); @@ -159,5 +166,83 @@ private static GitObjectContentType MapContentType(string contentType) return GitObjectContentType.None; } } + + private sealed class ReadErrorTrackingStream : Stream + { + private readonly Stream innerStream; + + public ReadErrorTrackingStream(Stream innerStream) + { + this.innerStream = innerStream; + } + + public bool ReadFailed { get; private set; } + + public override bool CanRead => this.innerStream.CanRead; + + public override bool CanSeek => this.innerStream.CanSeek; + + public override bool CanWrite => this.innerStream.CanWrite; + + public override long Length => this.innerStream.Length; + + public override long Position + { + get { return this.innerStream.Position; } + set { this.innerStream.Position = value; } + } + + public override void Flush() => this.innerStream.Flush(); + + public override int Read(byte[] buffer, int offset, int count) => + this.TrackRead(() => this.innerStream.Read(buffer, offset, count)); + + public override int ReadByte() => this.TrackRead(this.innerStream.ReadByte); + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + this.TrackReadAsync(() => this.innerStream.ReadAsync(buffer, offset, count, cancellationToken)); + + public override long Seek(long offset, SeekOrigin origin) => this.innerStream.Seek(offset, origin); + + public override void SetLength(long value) => this.innerStream.SetLength(value); + + public override void Write(byte[] buffer, int offset, int count) => this.innerStream.Write(buffer, offset, count); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + this.innerStream.Dispose(); + } + + base.Dispose(disposing); + } + + private T TrackRead(Func read) + { + try + { + return read(); + } + catch (Exception ex) when (!(ex is OperationCanceledException)) + { + this.ReadFailed = true; + throw; + } + } + + private async Task TrackReadAsync(Func> read) + { + try + { + return await read().ConfigureAwait(false); + } + catch (Exception ex) when (!(ex is OperationCanceledException)) + { + this.ReadFailed = true; + throw; + } + } + } } } diff --git a/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs b/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs index 45e4ba8c4f..278351b8ee 100644 --- a/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs @@ -2,6 +2,7 @@ using GVFS.Common.Tracing; using System; using System.Collections.Generic; +using System.IO; using System.Text.Json.Serialization; using System.Linq; using System.Net; @@ -34,8 +35,12 @@ public virtual List QueryForFileSizes(IEnumerable objectI long requestId = HttpRequestor.GetNewRequestId(); string objectIdsJson = ToJsonList(objectIds); - Uri cacheServerEndpoint = new Uri(this.CacheServer.SizesEndpointUrl); + Uri preferredCacheServerEndpoint = new Uri(this.CacheServer.SizesEndpointUrl); + Uri globalCacheServerEndpoint = new Uri(this.CacheServer.GlobalSizesEndpointUrl); Uri originEndpoint = new Uri(this.enlistment.RepoUrl + GVFSConstants.Endpoints.GVFSSizes); + bool hasEndpointOverride = preferredCacheServerEndpoint != globalCacheServerEndpoint; + bool useGlobalCacheServer = !hasEndpointOverride; + bool useOrigin = this.nextCacheServerAttemptTime >= DateTime.Now; EventMetadata metadata = new EventMetadata(); metadata.Add("RequestId", requestId); @@ -51,38 +56,91 @@ public virtual List QueryForFileSizes(IEnumerable objectI this.Tracer.RelatedEvent(EventLevel.Informational, "QueryFileSizes", metadata, Keywords.Network); - RetryWrapper> retrier = new RetryWrapper>(this.RetryConfig.MaxAttempts, cancellationToken); + RetryWrapper> retrier = new RetryWrapper>( + this.RetryConfig.MaxAttempts + (hasEndpointOverride && !useOrigin ? 2 : 0), + cancellationToken); retrier.OnFailure += RetryWrapper>.StandardErrorHandler(this.Tracer, requestId, "QueryFileSizes"); RetryWrapper>.InvocationResult requestTask = retrier.Invoke( tryCount => { Uri gvfsEndpoint; - if (this.nextCacheServerAttemptTime < DateTime.Now) + if (useOrigin) + { + gvfsEndpoint = originEndpoint; + } + else if (useGlobalCacheServer) { - gvfsEndpoint = cacheServerEndpoint; + gvfsEndpoint = globalCacheServerEndpoint; } else { - gvfsEndpoint = originEndpoint; + gvfsEndpoint = preferredCacheServerEndpoint; } - using (GitEndPointResponseData response = this.SendRequest(requestId, gvfsEndpoint, HttpMethod.Post, objectIdsJson, cancellationToken)) + try { - if (response.StatusCode == HttpStatusCode.NotFound) + using (GitEndPointResponseData response = this.SendProtocolRequest(requestId, gvfsEndpoint, HttpMethod.Post, objectIdsJson, cancellationToken)) { - this.nextCacheServerAttemptTime = DateTime.Now.AddDays(1); - return new RetryWrapper>.CallbackResult(response.Error, true); + if (response.HasErrors && !useGlobalCacheServer && !useOrigin) + { + this.TraceCacheServerFallback( + requestId, + preferredCacheServerEndpoint, + globalCacheServerEndpoint, + "EndpointSpecific", + "Global"); + useGlobalCacheServer = true; + return new RetryWrapper>.CallbackResult( + response.Error, + shouldRetry: true, + result: null, + shouldRecordFailure: false); + } + + if (response.StatusCode == HttpStatusCode.NotFound) + { + if (!useOrigin) + { + this.TraceCacheServerFallback( + requestId, + globalCacheServerEndpoint, + originEndpoint, + "Global", + "Origin"); + } + + this.nextCacheServerAttemptTime = DateTime.Now.AddDays(1); + useOrigin = true; + return new RetryWrapper>.CallbackResult( + response.Error, + shouldRetry: true, + result: null, + shouldRecordFailure: false); + } + + if (response.HasErrors) + { + return new RetryWrapper>.CallbackResult(response.Error, response.ShouldRetry); + } + + string objectSizesString = response.RetryableReadToEnd(); + List objectSizes = GVFSJsonOptions.Deserialize>(objectSizesString); + return new RetryWrapper>.CallbackResult(objectSizes); } - - if (response.HasErrors) - { - return new RetryWrapper>.CallbackResult(response.Error, response.ShouldRetry); - } - - string objectSizesString = response.RetryableReadToEnd(); - List objectSizes = GVFSJsonOptions.Deserialize>(objectSizesString); - return new RetryWrapper>.CallbackResult(objectSizes); + } + catch (Exception e) when ( + (e is HttpRequestException || e is IOException || e is RetryableException) && + !useGlobalCacheServer && + !useOrigin) + { + this.TraceCacheServerFallback(requestId, preferredCacheServerEndpoint, globalCacheServerEndpoint, "EndpointSpecific", "Global"); + useGlobalCacheServer = true; + return new RetryWrapper>.CallbackResult( + e, + shouldRetry: true, + result: null, + shouldRecordFailure: false); } }); @@ -109,7 +167,7 @@ public virtual GitRefs QueryInfoRefs(string branch) RetryWrapper.InvocationResult output = retrier.Invoke( tryCount => { - using (GitEndPointResponseData response = this.SendRequest( + using (GitEndPointResponseData response = this.SendProtocolRequest( requestId, infoRefsEndpoint, HttpMethod.Get, @@ -150,6 +208,7 @@ public virtual RetryWrapper.InvocationResult TryDownloadLoo eArgs => this.HandleDownloadAndSaveObjectError(retryOnFailure, requestId, eArgs), HttpMethod.Get, new Uri(this.CacheServer.ObjectsGetEndpointUrl + "/" + objectId), + new Uri(this.CacheServer.ObjectsEndpointUrl + "/" + objectId), cancellationToken, requestBody: null, acceptType: null, @@ -170,10 +229,11 @@ public virtual RetryWrapper.InvocationResult TryDownloadObj onSuccess, onFailure, HttpMethod.Post, - new Uri(this.CacheServer.ObjectsPostEndpointUrl), - CancellationToken.None, - () => this.ObjectIdsJsonGenerator(requestId, objectIdGenerator), - preferBatchedLooseObjects ? CustomLooseObjectsHeader : null); + () => new Uri(this.CacheServer.ObjectsPostEndpointUrl), + requestBodyGenerator: () => this.ObjectIdsJsonGenerator(requestId, objectIdGenerator), + cancellationToken: CancellationToken.None, + acceptType: preferBatchedLooseObjects ? CustomLooseObjectsHeader : null, + fallbackEndPointGenerator: () => new Uri(this.CacheServer.ObjectsEndpointUrl)); } public virtual RetryWrapper.InvocationResult TryDownloadObjects( @@ -205,11 +265,37 @@ public virtual RetryWrapper.InvocationResult TryDownloadObj onFailure, HttpMethod.Post, new Uri(this.CacheServer.ObjectsPostEndpointUrl), + new Uri(this.CacheServer.ObjectsEndpointUrl), CancellationToken.None, objectIdsJson, preferBatchedLooseObjects ? CustomLooseObjectsHeader : null); } + public virtual RetryWrapper.InvocationResult TrySendProtocolRequest( + long requestId, + Func.CallbackResult> onSuccess, + Action.ErrorEventArgs> onFailure, + HttpMethod method, + Uri endPoint, + Uri fallbackEndPoint, + CancellationToken cancellationToken, + string requestBody = null, + MediaTypeWithQualityHeaderValue acceptType = null, + bool retryOnFailure = true) + { + return this.TrySendProtocolRequest( + requestId, + onSuccess, + onFailure, + method, + () => endPoint, + requestBodyGenerator: () => requestBody, + cancellationToken: cancellationToken, + acceptType: acceptType, + retryOnFailure: retryOnFailure, + fallbackEndPointGenerator: () => fallbackEndPoint); + } + public virtual RetryWrapper.InvocationResult TrySendProtocolRequest( long requestId, Func.CallbackResult> onSuccess, @@ -227,10 +313,11 @@ public virtual RetryWrapper.InvocationResult TrySendProtoco onFailure, method, endPoint, - cancellationToken, - () => requestBody, - acceptType, - retryOnFailure); + fallbackEndPoint: null, + cancellationToken: cancellationToken, + requestBody: requestBody, + acceptType: acceptType, + retryOnFailure: retryOnFailure); } public virtual RetryWrapper.InvocationResult TrySendProtocolRequest( @@ -250,10 +337,10 @@ public virtual RetryWrapper.InvocationResult TrySendProtoco onFailure, method, () => endPoint, - requestBodyGenerator, - cancellationToken, - acceptType, - retryOnFailure); + requestBodyGenerator: requestBodyGenerator, + cancellationToken: cancellationToken, + acceptType: acceptType, + retryOnFailure: retryOnFailure); } public virtual RetryWrapper.InvocationResult TrySendProtocolRequest( @@ -265,10 +352,16 @@ public virtual RetryWrapper.InvocationResult TrySendProtoco Func requestBodyGenerator, CancellationToken cancellationToken, MediaTypeWithQualityHeaderValue acceptType = null, - bool retryOnFailure = true) + bool retryOnFailure = true, + Func fallbackEndPointGenerator = null) { + Uri endPoint = endPointGenerator(); + Uri fallbackEndPoint = fallbackEndPointGenerator?.Invoke(); + bool hasFallbackEndPoint = fallbackEndPoint != null && endPoint != fallbackEndPoint; + bool useFallbackEndPoint = false; + RetryWrapper retrier = new RetryWrapper( - retryOnFailure ? this.RetryConfig.MaxAttempts : 1, + (retryOnFailure ? this.RetryConfig.MaxAttempts : 1) + (hasFallbackEndPoint ? 1 : 0), cancellationToken); if (onFailure != null) { @@ -278,24 +371,182 @@ public virtual RetryWrapper.InvocationResult TrySendProtoco return retrier.Invoke( tryCount => { - using (GitEndPointResponseData response = this.SendRequest( - requestId, - endPointGenerator(), - method, - requestBodyGenerator(), - cancellationToken, - acceptType)) + Uri requestEndPoint = useFallbackEndPoint ? fallbackEndPointGenerator() : endPointGenerator(); + GitEndPointResponseData response; + + try + { + response = this.SendProtocolRequest( + requestId, + requestEndPoint, + method, + requestBodyGenerator(), + cancellationToken, + acceptType); + } + catch (HttpRequestException e) + { + return this.HandleProtocolException( + requestId, + e, + requestEndPoint, + fallbackEndPoint, + hasFallbackEndPoint, + ref useFallbackEndPoint, + retryOnFailure); + } + catch (IOException e) + { + return this.HandleProtocolException( + requestId, + e, + requestEndPoint, + fallbackEndPoint, + hasFallbackEndPoint, + ref useFallbackEndPoint, + retryOnFailure); + } + catch (RetryableException e) + { + return this.HandleProtocolException( + requestId, + e, + requestEndPoint, + fallbackEndPoint, + hasFallbackEndPoint, + ref useFallbackEndPoint, + retryOnFailure); + } + + using (response) { if (response.HasErrors) { - return new RetryWrapper.CallbackResult(response.Error, response.ShouldRetry, new GitObjectTaskResult(response.StatusCode)); + bool shouldFallBack = hasFallbackEndPoint && !useFallbackEndPoint; + if (shouldFallBack) + { + this.TraceCacheServerFallback( + requestId, + requestEndPoint, + fallbackEndPoint, + "EndpointSpecific", + "Global"); + } + + useFallbackEndPoint |= shouldFallBack; + return new RetryWrapper.CallbackResult( + response.Error, + shouldFallBack || response.ShouldRetry, + new GitObjectTaskResult(response.StatusCode, requestEndPoint), + shouldRecordFailure: response.ShouldRetry && !shouldFallBack); } - return onSuccess(tryCount, response); + RetryWrapper.CallbackResult result; + try + { + result = onSuccess(tryCount, response); + } + catch (Exception e) + { + if (response.StreamReadFailed) + { + return this.HandleProtocolException( + requestId, + e, + requestEndPoint, + fallbackEndPoint, + hasFallbackEndPoint, + ref useFallbackEndPoint, + retryOnFailure); + } + + throw; + } + + if (result.HasErrors) + { + bool shouldFallBack = response.StreamReadFailed && hasFallbackEndPoint && !useFallbackEndPoint; + if (shouldFallBack) + { + this.TraceCacheServerFallback( + requestId, + requestEndPoint, + fallbackEndPoint, + "EndpointSpecific", + "Global"); + useFallbackEndPoint = true; + } + + GitObjectTaskResult requestResult = result.Result == null + ? new GitObjectTaskResult(success: false, requestEndPoint) + : result.Result.WithRequestUri(requestEndPoint); + return new RetryWrapper.CallbackResult( + result.Error, + shouldFallBack || result.ShouldRetry, + requestResult, + shouldRecordFailure: result.ShouldRecordFailure && !shouldFallBack); + } + + return result; } }); } + private RetryWrapper.CallbackResult HandleProtocolException( + long requestId, + Exception error, + Uri requestEndPoint, + Uri fallbackEndPoint, + bool hasFallbackEndPoint, + ref bool useFallbackEndPoint, + bool retryOnFailure) + { + bool shouldFallBack = hasFallbackEndPoint && !useFallbackEndPoint; + if (shouldFallBack) + { + this.TraceCacheServerFallback( + requestId, + requestEndPoint, + fallbackEndPoint, + "EndpointSpecific", + "Global"); + useFallbackEndPoint = true; + } + + return new RetryWrapper.CallbackResult( + error, + shouldFallBack || retryOnFailure, + new GitObjectTaskResult(success: false, requestEndPoint), + shouldRecordFailure: retryOnFailure && !shouldFallBack); + } + + private void TraceCacheServerFallback( + long requestId, + Uri source, + Uri target, + string sourceRoute, + string targetRoute) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("RequestId", requestId); + metadata.Add("SourceRoute", sourceRoute); + metadata.Add("SourceAuthority", GetAuthorityForTelemetry(source)); + metadata.Add("TargetRoute", targetRoute); + metadata.Add("TargetAuthority", GetAuthorityForTelemetry(target)); + this.Tracer.RelatedEvent(EventLevel.Informational, "CacheServerFallback", metadata, Keywords.Network | Keywords.Telemetry); + } + + protected virtual GitEndPointResponseData SendProtocolRequest( + long requestId, + Uri requestUri, + HttpMethod httpMethod, + string requestContent, + CancellationToken cancellationToken, + MediaTypeWithQualityHeaderValue acceptType = null) + { + return this.SendRequest(requestId, requestUri, httpMethod, requestContent, cancellationToken, acceptType); + } + private static string ToJsonList(IEnumerable strings) { return "[\"" + string.Join("\",\"", strings) + "\"]"; @@ -356,19 +607,29 @@ public GitObjectSize(string id, long size) public class GitObjectTaskResult { - public GitObjectTaskResult(bool success) + public GitObjectTaskResult(bool success, Uri requestUri = null) { this.Success = success; + this.RequestUri = requestUri; } - public GitObjectTaskResult(HttpStatusCode statusCode) - : this(statusCode == HttpStatusCode.OK) + public GitObjectTaskResult(HttpStatusCode statusCode, Uri requestUri = null) + : this(statusCode == HttpStatusCode.OK, requestUri) { this.HttpStatusCodeResult = statusCode; } public bool Success { get; } - public HttpStatusCode HttpStatusCodeResult { get; } + public HttpStatusCode HttpStatusCodeResult { get; private set; } + public Uri RequestUri { get; } + + public GitObjectTaskResult WithRequestUri(Uri requestUri) + { + return new GitObjectTaskResult(this.Success, requestUri) + { + HttpStatusCodeResult = this.HttpStatusCodeResult, + }; + } } } } \ No newline at end of file diff --git a/GVFS/GVFS.Common/Http/HttpRequestor.cs b/GVFS/GVFS.Common/Http/HttpRequestor.cs index 435e52c2b1..ee1558abf2 100644 --- a/GVFS/GVFS.Common/Http/HttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/HttpRequestor.cs @@ -312,11 +312,11 @@ protected GitEndPointResponseData SendRequest( } private static bool ShouldRetry(HttpStatusCode statusCode) - { + { // Retry timeout, Unauthorized, 429 (Too Many Requests), and 5xx errors int statusInt = (int)statusCode; if (statusCode == HttpStatusCode.RequestTimeout || - statusCode == HttpStatusCode.Unauthorized || + statusCode == HttpStatusCode.Unauthorized || statusInt == 429 || (statusInt >= 500 && statusInt < 600)) { @@ -378,6 +378,11 @@ internal static bool ShouldRejectCredentials(HttpStatusCode statusCode, string r return false; } + internal static string GetAuthorityForTelemetry(Uri uri) + { + return uri.Authority; + } + private static string GetSingleHeaderOrEmpty(HttpHeaders headers, string headerName) { IEnumerable values; diff --git a/GVFS/GVFS.Common/RetryWrapper.cs b/GVFS/GVFS.Common/RetryWrapper.cs index 4d6a0ccd84..4d56ccc1fa 100644 --- a/GVFS/GVFS.Common/RetryWrapper.cs +++ b/GVFS/GVFS.Common/RetryWrapper.cs @@ -88,7 +88,7 @@ public InvocationResult Invoke(Func toInvoke) CallbackResult result = toInvoke(tryCount); if (result.HasErrors) { - if (result.ShouldRetry) + if (result.ShouldRecordFailure) { RetryCircuitBreaker.RecordFailure(); } @@ -224,6 +224,7 @@ public CallbackResult(Exception error, bool shouldRetry) this.HasErrors = true; this.Error = error; this.ShouldRetry = shouldRetry; + this.ShouldRecordFailure = shouldRetry; } public CallbackResult(Exception error, bool shouldRetry, T result) @@ -232,9 +233,16 @@ public CallbackResult(Exception error, bool shouldRetry, T result) this.Result = result; } + public CallbackResult(Exception error, bool shouldRetry, T result, bool shouldRecordFailure) + : this(error, shouldRetry, result) + { + this.ShouldRecordFailure = shouldRecordFailure; + } + public bool HasErrors { get; } public Exception Error { get; } public bool ShouldRetry { get; } + public bool ShouldRecordFailure { get; } public T Result { get; } } } diff --git a/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs b/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs index 651e05343b..f338fedadf 100644 --- a/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs +++ b/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs @@ -91,6 +91,20 @@ public void EndpointSpecificCacheServersArePreservedWhenGlobalCacheServerIsResol resolvedCacheServer.HasValidUrl().ShouldEqual(true); } + [TestCase] + public void InvalidEndpointSpecificCacheServerIsRejected() + { + MockGVFSEnlistment enlistment = this.CreateEnlistment( + CacheServerUrl, + prefetchCacheServerUrl: "not-a-url"); + + InvalidRepoException exception = Assert.Throws( + () => CacheServerResolver.GetCacheServerFromConfig(enlistment)); + + exception.Message.ShouldContain(GVFSConstants.GitConfig.PrefetchCacheServer); + exception.Message.ShouldContain("not an absolute URL"); + } + [TestCase] public void CanSaveEndpointSpecificCacheServers() { diff --git a/GVFS/GVFS.UnitTests/Http/GitObjectsHttpRequestorTests.cs b/GVFS/GVFS.UnitTests/Http/GitObjectsHttpRequestorTests.cs new file mode 100644 index 0000000000..ee89a68ae2 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Http/GitObjectsHttpRequestorTests.cs @@ -0,0 +1,485 @@ +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Http; +using GVFS.Common.Tracing; +using GVFS.Tests.Should; +using GVFS.UnitTests.Mock.Common; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; + +namespace GVFS.UnitTests.Http +{ + [TestFixture] + public class GitObjectsHttpRequestorTests + { + private const string GlobalCacheServerUrl = "https://global-cache/server"; + private const string EndpointCacheServerUrl = "https://endpoint-cache/server"; + + [SetUp] + public void SetUp() + { + RetryCircuitBreaker.Reset(); + } + + [TearDown] + public void TearDown() + { + RetryCircuitBreaker.Reset(); + } + + [TestCase] + public void LooseObjectFallsBackToGlobalCacheServerWhenEndpointRequestFails() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + requestor.EnqueueResponse(HttpStatusCode.OK); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadLooseObject( + "0123456789abcdef", + retryOnFailure: false, + CancellationToken.None, + requestSource: "test", + onSuccess: SuccessfulRequest); + + result.Succeeded.ShouldEqual(true); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects/0123456789abcdef"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects/0123456789abcdef"); + requestor.TestTracer.RelatedEventNames.ShouldContain(name => name == "CacheServerFallback"); + requestor.TestTracer.RelatedEventKeywords.ShouldContain( + keywords => (keywords & Keywords.Telemetry) == Keywords.Telemetry); + } + + [TestCase] + public void BatchedObjectRequestFallsBackToGlobalCacheServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable, shouldRetry: true); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: SuccessfulRequest, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(false); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void PrefetchRequestFallsBackToGlobalCacheServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(HttpStatusCode.BadRequest); + requestor.EnqueueResponse(HttpStatusCode.OK); + + RetryWrapper.InvocationResult result = + requestor.TrySendProtocolRequest( + requestId: 1, + onSuccess: SuccessfulRequest, + onFailure: null, + method: HttpMethod.Get, + endPointGenerator: () => new Uri(EndpointCacheServerUrl + "/gvfs/prefetch?lastPackTimestamp=0"), + fallbackEndPointGenerator: () => new Uri(GlobalCacheServerUrl + "/gvfs/prefetch?lastPackTimestamp=0"), + requestBodyGenerator: () => null, + cancellationToken: CancellationToken.None); + + result.Succeeded.ShouldEqual(true); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/prefetch?lastPackTimestamp=0"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/prefetch?lastPackTimestamp=0"); + } + + [TestCase] + public void TransportExceptionFallsBackToGlobalCacheServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueException(new HttpRequestException("Test failure")); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: SuccessfulRequest, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(false); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void ResponseBodyReadFailureFallsBackToGlobalCacheServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(new ThrowingReadStream()); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: (tryCount, response) => + { + response.Stream.ReadByte(); + return SuccessfulRequest(tryCount, response); + }, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(false); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void ResponseBodyReadFailureReportedByHandlerFallsBackToGlobalCacheServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(new ThrowingReadStream()); + requestor.EnqueueResponse(HttpStatusCode.OK); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: (tryCount, response) => + { + try + { + response.Stream.ReadByte(); + return SuccessfulRequest(tryCount, response); + } + catch (IOException e) + { + return new RetryWrapper.CallbackResult( + e, + shouldRetry: true); + } + }, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(true); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void ResponseBodyCancellationIsNotRetriedOrReportedAsFallback() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 1); + requestor.EnqueueResponse(new CancelingReadStream()); + + Assert.Throws( + () => requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: (tryCount, response) => + { + response.RetryableReadToEnd(); + return SuccessfulRequest(tryCount, response); + }, + onFailure: null, + preferBatchedLooseObjects: false)); + + requestor.RequestUris.Count.ShouldEqual(1); + requestor.TestTracer.RelatedEventNames.ShouldNotContain(name => name == "CacheServerFallback"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void FallbackTelemetryExcludesCredentialsAndRequestPath() + { + const string CredentialedGlobalUrl = "https://global-user:global-secret@global-cache:8443/server"; + const string CredentialedEndpointUrl = "https://endpoint-user:endpoint-secret@endpoint-cache:9443/server"; + TestGitObjectsHttpRequestor requestor = this.CreateRequestor( + maxRetries: 0, + globalCacheServerUrl: CredentialedGlobalUrl, + endpointCacheServerUrl: CredentialedEndpointUrl); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + requestor.EnqueueResponse(HttpStatusCode.OK); + + requestor.TryDownloadLooseObject( + "0123456789abcdef", + retryOnFailure: false, + CancellationToken.None, + requestSource: "test", + onSuccess: SuccessfulRequest); + + int fallbackEventIndex = requestor.TestTracer.RelatedEventNames.IndexOf("CacheServerFallback"); + EventMetadata metadata = requestor.TestTracer.RelatedEventMetadata[fallbackEventIndex]; + metadata["SourceAuthority"].ShouldEqual("endpoint-cache:9443"); + metadata["TargetAuthority"].ShouldEqual("global-cache:8443"); + } + + [TestCase] + public void NoEndpointOverrideUsesNormalGlobalCacheRetries() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 1, endpointOverrides: false); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable, shouldRetry: true); + requestor.EnqueueResponse(HttpStatusCode.OK); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: SuccessfulRequest, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(true); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + requestor.TestTracer.RelatedEventNames.ShouldNotContain(name => name == "CacheServerFallback"); + } + + [TestCase] + public void TerminalFallbackFailureReportsGlobalCacheServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: SuccessfulRequest, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(false); + result.Attempts.ShouldEqual(2); + result.Result.HttpStatusCodeResult.ShouldEqual(HttpStatusCode.NotFound); + result.Result.RequestUri.AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void SuccessHandlerFailureRetriesTheEndpointSpecificServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 1); + requestor.EnqueueResponse(HttpStatusCode.OK); + requestor.EnqueueResponse(HttpStatusCode.OK); + int successHandlerCalls = 0; + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: (tryCount, response) => + { + if (++successHandlerCalls == 1) + { + throw new RetryableException("Local write failed"); + } + + return SuccessfulRequest(tryCount, response); + }, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(true); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects"); + requestor.TestTracer.RelatedEventNames.ShouldNotContain(name => name == "CacheServerFallback"); + } + + [TestCase] + public void SizesRequestFallsBackThroughGlobalCacheServerToOrigin() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + requestor.EnqueueResponse(HttpStatusCode.OK, "[]"); + + requestor.QueryForFileSizes(new[] { "0123456789abcdef" }, CancellationToken.None); + + requestor.RequestUris.Count.ShouldEqual(3); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/sizes"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/sizes"); + requestor.RequestUris[2].AbsoluteUri.ShouldEqual("mock://repourl/gvfs/sizes"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void SizesHttpFallbackDoesNotChargeCircuitBreaker() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable, shouldRetry: true); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable); + + requestor.QueryForFileSizes(new[] { "0123456789abcdef" }, CancellationToken.None); + + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/sizes"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/sizes"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void SizesTransportFallbackDoesNotChargeCircuitBreaker() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueException(new HttpRequestException("Test failure")); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable); + + requestor.QueryForFileSizes(new[] { "0123456789abcdef" }, CancellationToken.None); + + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/sizes"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/sizes"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + private static RetryWrapper.CallbackResult SuccessfulRequest( + int tryCount, + GitEndPointResponseData response) + { + return new RetryWrapper.CallbackResult( + new GitObjectsHttpRequestor.GitObjectTaskResult(true)); + } + + private TestGitObjectsHttpRequestor CreateRequestor( + int maxRetries, + bool endpointOverrides = true, + string globalCacheServerUrl = GlobalCacheServerUrl, + string endpointCacheServerUrl = EndpointCacheServerUrl) + { + CacheServerInfo cacheServer = new CacheServerInfo(globalCacheServerUrl, "global"); + if (endpointOverrides) + { + cacheServer = cacheServer.WithEndpointOverrides( + endpointCacheServerUrl, + endpointCacheServerUrl, + endpointCacheServerUrl, + endpointCacheServerUrl); + } + + return new TestGitObjectsHttpRequestor( + new MockGVFSEnlistment(), + cacheServer, + new RetryConfig(maxRetries)); + } + + private class TestGitObjectsHttpRequestor : GitObjectsHttpRequestor + { + private readonly Queue responses = new Queue(); + + public TestGitObjectsHttpRequestor( + Enlistment enlistment, + CacheServerInfo cacheServer, + RetryConfig retryConfig) + : this(new MockTracer(), enlistment, cacheServer, retryConfig) + { + } + + private TestGitObjectsHttpRequestor( + MockTracer tracer, + Enlistment enlistment, + CacheServerInfo cacheServer, + RetryConfig retryConfig) + : base(tracer, enlistment, cacheServer, retryConfig) + { + this.TestTracer = tracer; + this.RequestUris = new List(); + } + + public MockTracer TestTracer { get; } + public List RequestUris { get; } + + public void EnqueueResponse(HttpStatusCode statusCode, string body = "", bool shouldRetry = false) + { + this.responses.Enqueue(Tuple.Create(statusCode, body, shouldRetry)); + } + + public void EnqueueException(Exception exception) + { + this.responses.Enqueue(exception); + } + + public void EnqueueResponse(Stream stream) + { + this.responses.Enqueue(stream); + } + + protected override GitEndPointResponseData SendProtocolRequest( + long requestId, + Uri requestUri, + HttpMethod httpMethod, + string requestContent, + CancellationToken cancellationToken, + MediaTypeWithQualityHeaderValue acceptType = null) + { + this.RequestUris.Add(requestUri); + object nextResponse = this.responses.Dequeue(); + if (nextResponse is Exception exception) + { + throw exception; + } + + if (nextResponse is Stream stream) + { + return new GitEndPointResponseData( + HttpStatusCode.OK, + "application/json", + stream, + message: null, + onResponseDisposed: null); + } + + Tuple response = (Tuple)nextResponse; + + if (response.Item1 == HttpStatusCode.OK) + { + return new GitEndPointResponseData( + response.Item1, + "application/json", + new MemoryStream(Encoding.UTF8.GetBytes(response.Item2)), + message: null, + onResponseDisposed: null); + } + + return new GitEndPointResponseData( + response.Item1, + new GitObjectsHttpException(response.Item1, "Test failure"), + shouldRetry: response.Item3, + message: null, + onResponseDisposed: null); + } + } + + private class ThrowingReadStream : MemoryStream + { + public override int ReadByte() + { + throw new IOException("Response body read failed"); + } + } + + private class CancelingReadStream : MemoryStream + { + public override int Read(byte[] buffer, int offset, int count) + { + throw new OperationCanceledException("Response body read canceled"); + } + } + } +} diff --git a/GVFS/GVFS.UnitTests/Http/HttpRequestorTests.cs b/GVFS/GVFS.UnitTests/Http/HttpRequestorTests.cs index 33692dd03b..e81b81170d 100644 --- a/GVFS/GVFS.UnitTests/Http/HttpRequestorTests.cs +++ b/GVFS/GVFS.UnitTests/Http/HttpRequestorTests.cs @@ -1,7 +1,8 @@ -using System.Net; using GVFS.Common.Http; using GVFS.Tests.Should; using NUnit.Framework; +using System; +using System.Net; namespace GVFS.UnitTests.Http { @@ -75,5 +76,13 @@ public void CommonNonAuthStatusesDoNotRejectCredentials() HttpRequestor.ShouldRejectCredentials(HttpStatusCode.RequestTimeout, responseBody: null) .ShouldEqual(false, "A 408 must NOT reject credentials"); } + + [TestCase] + public void AuthorityForTelemetryExcludesCredentialsAndRequestPath() + { + Uri uri = new Uri("https://alice:secret@cache.example.com:8443/private/path?token=sensitive#fragment"); + + HttpRequestor.GetAuthorityForTelemetry(uri).ShouldEqual("cache.example.com:8443"); + } } } diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs index d933584e94..47ad66e295 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs @@ -17,6 +17,8 @@ public MockTracer() this.RelatedWarningEvents = new List(); this.RelatedErrorEvents = new List(); this.RelatedEventNames = new List(); + this.RelatedEventKeywords = new List(); + this.RelatedEventMetadata = new List(); } public MockTracer StartActivityTracer { get; private set; } @@ -29,6 +31,8 @@ public MockTracer() // Names of events reported via RelatedEvent (which, unlike RelatedInfo/Warning/Error, // do not otherwise get recorded). Lets tests assert a specific diagnostic event fired. public List RelatedEventNames { get; } + public List RelatedEventKeywords { get; } + public List RelatedEventMetadata { get; } public void WaitForRelatedEvent() { @@ -38,6 +42,8 @@ public void WaitForRelatedEvent() public void RelatedEvent(EventLevel error, string eventName, EventMetadata metadata) { this.RelatedEventNames.Add(eventName); + this.RelatedEventKeywords.Add(Keywords.None); + this.RelatedEventMetadata.Add(metadata); if (eventName == this.WaitRelatedEventName) { this.waitEvent.Set(); @@ -47,6 +53,8 @@ public void RelatedEvent(EventLevel error, string eventName, EventMetadata metad public void RelatedEvent(EventLevel error, string eventName, EventMetadata metadata, Keywords keyword) { this.RelatedEventNames.Add(eventName); + this.RelatedEventKeywords.Add(keyword); + this.RelatedEventMetadata.Add(metadata); if (eventName == this.WaitRelatedEventName) { this.waitEvent.Set(); diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 370f40b30a..2e9bb52762 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -65,15 +65,19 @@ public static System.CommandLine.Command CreateCommand() cmd.Add(cacheServerOption); System.CommandLine.Option prefetchCacheServerOption = new System.CommandLine.Option("--prefetch-cache-server-url") { Description = "The cache server URL for the prefetch endpoint" }; + AddEndpointCacheServerUrlValidator(prefetchCacheServerOption); cmd.Add(prefetchCacheServerOption); System.CommandLine.Option getCacheServerOption = new System.CommandLine.Option("--get-cache-server-url") { Description = "The cache server URL for the objects GET endpoint" }; + AddEndpointCacheServerUrlValidator(getCacheServerOption); cmd.Add(getCacheServerOption); System.CommandLine.Option postCacheServerOption = new System.CommandLine.Option("--post-cache-server-url") { Description = "The cache server URL for the objects POST endpoint" }; + AddEndpointCacheServerUrlValidator(postCacheServerOption); cmd.Add(postCacheServerOption); System.CommandLine.Option sizesCacheServerOption = new System.CommandLine.Option("--sizes-cache-server-url") { Description = "The cache server URL for the sizes endpoint" }; + AddEndpointCacheServerUrlValidator(sizesCacheServerOption); cmd.Add(sizesCacheServerOption); System.CommandLine.Option branchOption = new System.CommandLine.Option("--branch", new[] { "-b" }) { Description = "Branch to checkout after clone" }; @@ -131,6 +135,19 @@ public static System.CommandLine.Command CreateCommand() return cmd; } + private static void AddEndpointCacheServerUrlValidator(System.CommandLine.Option option) + { + option.Validators.Add( + result => + { + string url = result.GetValueOrDefault(); + if (url != null && !CacheServerInfo.IsValidUrl(url)) + { + result.AddError($"Option '{option.Name}' requires an absolute URL."); + } + }); + } + protected override string VerbName { get { return CloneVerbName; } @@ -172,6 +189,10 @@ public override void Execute() this.BlockEmptyCacheServerUrl(this.GetCacheServerUrl); this.BlockEmptyCacheServerUrl(this.PostCacheServerUrl); this.BlockEmptyCacheServerUrl(this.SizesCacheServerUrl); + this.BlockInvalidEndpointCacheServerUrl("--prefetch-cache-server-url", this.PrefetchCacheServerUrl); + this.BlockInvalidEndpointCacheServerUrl("--get-cache-server-url", this.GetCacheServerUrl); + this.BlockInvalidEndpointCacheServerUrl("--post-cache-server-url", this.PostCacheServerUrl); + this.BlockInvalidEndpointCacheServerUrl("--sizes-cache-server-url", this.SizesCacheServerUrl); try { @@ -629,6 +650,14 @@ private bool TryDetermineLocalCacheAndInitializePaths( return true; } + private void BlockInvalidEndpointCacheServerUrl(string optionName, string url) + { + if (url != null && !CacheServerInfo.IsValidUrl(url)) + { + this.ReportErrorAndExit($"Option '{optionName}' requires an absolute URL."); + } + } + private Result CreateClone( ITracer tracer, GVFSEnlistment enlistment, From 63f8dc02f1ebd7f4401fee5b0ba6a36dc9a03ec1 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 2 Sep 2026 11:12:07 -0400 Subject: [PATCH 3/4] docs: Explain endpoint-specific cache routing Context: Administrators need to understand how dedicated GVFS endpoint caches interact with the existing global cache and with gvfs cache-server commands. Justification: Documenting precedence and fallback behavior alongside the configuration keys makes staged cache migrations predictable and preserves the distinction between global and endpoint-specific settings. Implementation: Describe the clone options, local Git config keys, endpoint-to-global fallback order, the sizes-to-origin fallback, and troubleshooting guidance for inspecting or changing endpoint overrides. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/getting-started.md | 14 ++++++++++++++ docs/troubleshooting.md | 20 ++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index aee8b93844..75899bb0ae 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -38,6 +38,20 @@ These options allow a user to customize their initial enlistment. cache servers via the `/gvfs/config` endpoint, then the `clone` command will select a nearby cache server from that list. +* `--prefetch-cache-server-url=`, + `--get-cache-server-url=`, `--post-cache-server-url=`, and + `--sizes-cache-server-url=`: Prefer the specified absolute cache server + URL for `/gvfs/prefetch`, loose-object GET requests, batched-object POST + requests, or `/gvfs/sizes`, respectively. If a dedicated server fails, VFS + for Git retries the request against the server selected by + `--cache-server-url`. Sizes requests retain their additional fallback to the + origin server when the global cache does not support `/gvfs/sizes`. + + These values are saved in the local Git configuration as + `gvfs.prefetch.cache-server`, `gvfs.get.cache-server`, + `gvfs.post.cache-server`, and `gvfs.sizes.cache-server`. They continue to + apply to later mount, hydration, and prefetch operations. + * `--branch=`: Specify the branch to checkout after clone. * `--local-cache-path=`: Use this option to override the path for the diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 44fa175482..8b91ed346a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -237,6 +237,26 @@ Run `gvfs cache-server --list` to see the available cache server URLs. Run `gvfs cache-server --set=` to set your cache server to ``. +Individual GVFS protocol endpoints can prefer dedicated cache servers through +these local Git configuration values: + +| Configuration | Requests | +| --- | --- | +| `gvfs.prefetch.cache-server` | `/gvfs/prefetch` | +| `gvfs.get.cache-server` | Loose-object GET requests under `/gvfs/objects` | +| `gvfs.post.cache-server` | Batched-object POST requests to `/gvfs/objects` | +| `gvfs.sizes.cache-server` | `/gvfs/sizes` | + +Each value must be an absolute URL. A dedicated endpoint server is attempted +before `gvfs.cache-server`; if that request fails, VFS for Git falls back to +the global cache server. Sizes requests also fall back from the global cache +to the origin server when `/gvfs/sizes` is not supported. + +`gvfs cache-server --get` and `--set` operate on the global +`gvfs.cache-server` value. Setting the global server does not clear the four +endpoint-specific values. Inspect or change those values with `git config +--local []`. + ### System-wide Config The `gvfs config` command allows customizing some behavior. From f2480964a103990f9bbcec5a8661248a3b21dc9c Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 2 Sep 2026 11:47:51 -0400 Subject: [PATCH 4/4] test: Cover prefetch failure telemetry redaction Context: The prefetch entry point now reports only URI authority when a request fails, but requestor-level tests did not execute the warning and unsupported-command telemetry paths that consume the terminal request URI. Justification: Exercise the production composition directly so future changes cannot reintroduce credentials, paths, queries, or fragments into prefetch failure diagnostics. These focused cases also raise changed-line coverage above the repository threshold without relying on incidental functional-test execution. Implementation: Add a deterministic prefetch requestor that returns terminal HTTP failures. Verify both general failure warnings and not-supported events emit only the host and port from a credential-bearing request URI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- GVFS/GVFS.UnitTests/Git/GitObjectsTests.cs | 96 ++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/GVFS/GVFS.UnitTests/Git/GitObjectsTests.cs b/GVFS/GVFS.UnitTests/Git/GitObjectsTests.cs index 88be433806..78236a37bc 100644 --- a/GVFS/GVFS.UnitTests/Git/GitObjectsTests.cs +++ b/GVFS/GVFS.UnitTests/Git/GitObjectsTests.cs @@ -1,13 +1,19 @@ using GVFS.Common; using GVFS.Common.Git; +using GVFS.Common.Http; using GVFS.Common.Tracing; using GVFS.Tests.Should; using GVFS.UnitTests.Mock.Common; using GVFS.UnitTests.Mock.FileSystem; using NUnit.Framework; +using System; using System.Collections.Generic; using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; using System.Security; +using System.Threading; namespace GVFS.UnitTests.Git { @@ -124,6 +130,61 @@ public void WriteLooseObject_Success() moved.ShouldBeTrue("File was not moved"); } + [TestCase] + public void PrefetchFailureTelemetryReportsOnlyRequestAuthority() + { + const string RequestUrl = "https://user:secret@cache.example:8443/gvfs/prefetch?token=sensitive"; + MockTracer tracer = new MockTracer(); + MockGVFSEnlistment enlistment = new MockGVFSEnlistment(); + TestPrefetchRequestor requestor = new TestPrefetchRequestor( + tracer, + enlistment, + HttpStatusCode.ServiceUnavailable, + new Uri(RequestUrl)); + GitObjects gitObjects = new GVFSGitObjects( + new GVFSContext(tracer, new MockFileSystemWithCallbacks(), null, enlistment), + requestor); + + gitObjects.TryDownloadPrefetchPacks( + gitProcess: null, + latestTimestamp: 0, + trustPackIndexes: false, + out List _) + .ShouldEqual(false); + + tracer.StartActivityTracer.RelatedWarningEvents.Count.ShouldEqual(1); + tracer.StartActivityTracer.RelatedWarningEvents[0].ShouldContain("\"PrefetchEndpointUrl\":\"cache.example:8443\""); + tracer.StartActivityTracer.RelatedWarningEvents[0].IndexOf("user", StringComparison.Ordinal).ShouldEqual(-1); + tracer.StartActivityTracer.RelatedWarningEvents[0].IndexOf("secret", StringComparison.Ordinal).ShouldEqual(-1); + tracer.StartActivityTracer.RelatedWarningEvents[0].IndexOf("sensitive", StringComparison.Ordinal).ShouldEqual(-1); + } + + [TestCase] + public void UnsupportedPrefetchTelemetryReportsOnlyRequestAuthority() + { + const string RequestUrl = "https://user:secret@cache.example:8443/gvfs/prefetch?token=sensitive"; + MockTracer tracer = new MockTracer(); + MockGVFSEnlistment enlistment = new MockGVFSEnlistment(); + TestPrefetchRequestor requestor = new TestPrefetchRequestor( + tracer, + enlistment, + HttpStatusCode.NotFound, + new Uri(RequestUrl)); + GitObjects gitObjects = new GVFSGitObjects( + new GVFSContext(tracer, new MockFileSystemWithCallbacks(), null, enlistment), + requestor); + + gitObjects.TryDownloadPrefetchPacks( + gitProcess: null, + latestTimestamp: 0, + trustPackIndexes: false, + out List _) + .ShouldEqual(false); + + EventMetadata metadata = tracer.StartActivityTracer.RelatedEventMetadata[0]; + metadata["PrefetchEndpointUrl"].ShouldEqual("cache.example:8443"); + } + private Stream OnOpenFileStream(string path, FileMode mode, FileAccess access) { this.openedPaths.Add(path); @@ -144,5 +205,40 @@ private bool OnFileExists(string path) { return this.pathsToData.TryGetValue(path, out _); } + + private class TestPrefetchRequestor : GitObjectsHttpRequestor + { + private readonly HttpStatusCode statusCode; + private readonly Uri requestUri; + + public TestPrefetchRequestor( + ITracer tracer, + Enlistment enlistment, + HttpStatusCode statusCode, + Uri requestUri) + : base(tracer, enlistment, new CacheServerInfo("https://cache.example/server", "cache"), new RetryConfig(0)) + { + this.statusCode = statusCode; + this.requestUri = requestUri; + } + + public override RetryWrapper.InvocationResult TrySendProtocolRequest( + long requestId, + Func.CallbackResult> onSuccess, + Action.ErrorEventArgs> onFailure, + HttpMethod method, + Func endPointGenerator, + Func requestBodyGenerator, + CancellationToken cancellationToken, + MediaTypeWithQualityHeaderValue acceptType = null, + bool retryOnFailure = true, + Func fallbackEndPointGenerator = null) + { + return new RetryWrapper.InvocationResult( + tryCount: 1, + new GitObjectsHttpException(this.statusCode, "Test failure"), + new GitObjectTaskResult(this.statusCode, this.requestUri)); + } + } } }