diff --git a/GVFS/FastFetch/FastFetchVerb.cs b/GVFS/FastFetch/FastFetchVerb.cs index 737b31ffe3..cf6add3aad 100644 --- a/GVFS/FastFetch/FastFetchVerb.cs +++ b/GVFS/FastFetch/FastFetchVerb.cs @@ -237,7 +237,9 @@ private int ExecuteWithExitCode() string fastfetchLogFile = Enlistment.GetNewLogFileName(enlistment.FastFetchLogRoot, "fastfetch"); tracer.AddLogFileEventListener(fastfetchLogFile, EventLevel.Informational, Keywords.Any); - CacheServerInfo cacheServer = new CacheServerInfo(this.GetRemoteUrl(enlistment), null); + CacheServerInfo cacheServer = string.IsNullOrWhiteSpace(this.CacheServerUrl) + ? CacheServerResolver.GetCacheServerFromConfig(enlistment) + : new CacheServerInfo(this.GetRemoteUrl(enlistment), null); tracer.WriteStartEvent( enlistment.PrimaryEnlistmentRoot, diff --git a/GVFS/GVFS.CommandLine.Tests/GvfsMainCliTests.cs b/GVFS/GVFS.CommandLine.Tests/GvfsMainCliTests.cs index 4eb360808e..5cb6be2407 100644 --- a/GVFS/GVFS.CommandLine.Tests/GvfsMainCliTests.cs +++ b/GVFS/GVFS.CommandLine.Tests/GvfsMainCliTests.cs @@ -215,12 +215,36 @@ public void Clone_FullCommandLine_ParsesCorrectly() { "clone", "https://example.com/repo", @"C:\Users\test\repo", "--cache-server-url", "https://cache.test", + "--prefetch-cache-server-url", "https://prefetch-cache.test", + "--get-cache-server-url", "https://get-cache.test", + "--post-cache-server-url", "https://post-cache.test", + "--sizes-cache-server-url", "https://sizes-cache.test", "-b", "develop", "--single-branch", "--no-mount", "--no-prefetch" }); Assert.That(parseResult.Errors, Is.Empty, "Full clone command should parse without errors"); + Assert.Multiple(() => + { + Assert.That(parseResult.GetValue((Option)FindOptionOnCommand("clone", "--cache-server-url")), Is.EqualTo("https://cache.test")); + Assert.That(parseResult.GetValue((Option)FindOptionOnCommand("clone", "--prefetch-cache-server-url")), Is.EqualTo("https://prefetch-cache.test")); + Assert.That(parseResult.GetValue((Option)FindOptionOnCommand("clone", "--get-cache-server-url")), Is.EqualTo("https://get-cache.test")); + Assert.That(parseResult.GetValue((Option)FindOptionOnCommand("clone", "--post-cache-server-url")), Is.EqualTo("https://post-cache.test")); + Assert.That(parseResult.GetValue((Option)FindOptionOnCommand("clone", "--sizes-cache-server-url")), Is.EqualTo("https://sizes-cache.test")); + }); + } + + [TestCase("--prefetch-cache-server-url")] + [TestCase("--get-cache-server-url")] + [TestCase("--post-cache-server-url")] + [TestCase("--sizes-cache-server-url")] + public void Clone_EndpointCacheServerUrl_RejectsInvalidUrl(string optionName) + { + var parseResult = rootCommand.Parse(new[] { "clone", "https://example.com/repo", optionName, "not-a-url" }); + + Assert.That(parseResult.Errors, Has.Count.EqualTo(1)); + Assert.That(parseResult.Errors[0].Message, Does.Contain("requires an absolute URL")); } [Test] @@ -342,7 +366,19 @@ public void Repair_FullCommandLine_ParsesCorrectly() [Test] public void Clone_HasAllExpectedOptions() { - var expected = new[] { "--cache-server-url", "--branch", "--single-branch", "--no-mount", "--no-prefetch", "--local-cache-path" }; + var expected = new[] + { + "--cache-server-url", + "--prefetch-cache-server-url", + "--get-cache-server-url", + "--post-cache-server-url", + "--sizes-cache-server-url", + "--branch", + "--single-branch", + "--no-mount", + "--no-prefetch", + "--local-cache-path", + }; foreach (var optName in expected) { Assert.That(FindOptionOnCommand("clone", optName), Is.Not.Null, diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 143b59e693..088d771957 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -33,6 +33,10 @@ public static class GitConfig public const string MountId = GVFSPrefix + "mount-id"; public const string EnlistmentId = GVFSPrefix + "enlistment-id"; public const string CacheServer = GVFSPrefix + "cache-server"; + public const string PrefetchCacheServer = GVFSPrefix + "prefetch.cache-server"; + public const string GetCacheServer = GVFSPrefix + "get.cache-server"; + public const string PostCacheServer = GVFSPrefix + "post.cache-server"; + public const string SizesCacheServer = GVFSPrefix + "sizes.cache-server"; public const string DeprecatedCacheEndpointSuffix = ".cache-server-url"; public const string HooksPrefix = GitConfig.GVFSPrefix + "clone.default-"; public const string GVFSTelemetryId = GitConfig.GVFSPrefix + "telemetry-id"; diff --git a/GVFS/GVFS.Common/Git/GitObjects.cs b/GVFS/GVFS.Common/Git/GitObjects.cs index a9b0f2851a..19c1a0e608 100644 --- a/GVFS/GVFS.Common/Git/GitObjects.cs +++ b/GVFS/GVFS.Common/Git/GitObjects.cs @@ -180,6 +180,11 @@ public virtual bool TryDownloadPrefetchPacks(GitProcess gitProcess, long latestT "{0}?lastPackTimestamp={1}", this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl, latestTimestamp)), + fallbackEndPointGenerator: () => new Uri( + string.Format( + "{0}?lastPackTimestamp={1}", + this.GitObjectRequestor.CacheServer.GlobalPrefetchEndpointUrl, + latestTimestamp)), requestBodyGenerator: () => null, cancellationToken: CancellationToken.None, acceptType: new MediaTypeWithQualityHeaderValue(GVFSConstants.MediaTypes.PrefetchPackFilesAndIndexesMediaType)); @@ -188,18 +193,21 @@ public virtual bool TryDownloadPrefetchPacks(GitProcess gitProcess, long latestT if (!result.Succeeded) { + Uri requestUri = result.Result?.RequestUri + ?? new Uri(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl); + string requestAuthority = HttpRequestor.GetAuthorityForTelemetry(requestUri); if (result.Result != null && result.Result.HttpStatusCodeResult == HttpStatusCode.NotFound) { EventMetadata warning = CreateEventMetadata(); warning.Add(TracingConstants.MessageKey.WarningMessage, "The server does not support " + GVFSConstants.Endpoints.GVFSPrefetch); - warning.Add(nameof(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl), this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl); + warning.Add("PrefetchEndpointUrl", requestAuthority); activity.RelatedEvent(EventLevel.Warning, "CommandNotSupported", warning); } else { EventMetadata error = CreateEventMetadata(result.Error); error.Add("latestTimestamp", latestTimestamp); - error.Add(nameof(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl), this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl); + error.Add("PrefetchEndpointUrl", requestAuthority); activity.RelatedWarning(error, "DownloadPrefetchPacks failed.", Keywords.Telemetry); } } diff --git a/GVFS/GVFS.Common/Http/CacheServerInfo.cs b/GVFS/GVFS.Common/Http/CacheServerInfo.cs index 0ec929b0dc..9df0399e42 100644 --- a/GVFS/GVFS.Common/Http/CacheServerInfo.cs +++ b/GVFS/GVFS.Common/Http/CacheServerInfo.cs @@ -11,30 +11,105 @@ public class CacheServerInfo [JsonConstructor] public CacheServerInfo(string url, string name, bool globalDefault = false) + : this(url, name, globalDefault, null, null, null, null) + { + } + + public CacheServerInfo( + string url, + string name, + bool globalDefault, + string prefetchCacheServerUrl, + string getCacheServerUrl, + string postCacheServerUrl, + string sizesCacheServerUrl) { this.Url = url; this.Name = name; this.GlobalDefault = globalDefault; + this.PrefetchCacheServerUrl = prefetchCacheServerUrl; + this.GetCacheServerUrl = getCacheServerUrl; + this.PostCacheServerUrl = postCacheServerUrl; + this.SizesCacheServerUrl = sizesCacheServerUrl; if (this.Url != null) { this.ObjectsEndpointUrl = this.Url + ObjectsEndpointSuffix; - this.PrefetchEndpointUrl = this.Url + PrefetchEndpointSuffix; - this.SizesEndpointUrl = this.Url + SizesEndpointSuffix; } + + this.GlobalPrefetchEndpointUrl = GetEndpointUrl(this.Url, PrefetchEndpointSuffix); + this.GlobalSizesEndpointUrl = GetEndpointUrl(this.Url, SizesEndpointSuffix); + this.PrefetchEndpointUrl = GetEndpointUrl(prefetchCacheServerUrl ?? this.Url, PrefetchEndpointSuffix); + this.ObjectsGetEndpointUrl = GetEndpointUrl(getCacheServerUrl ?? this.Url, ObjectsEndpointSuffix); + this.ObjectsPostEndpointUrl = GetEndpointUrl(postCacheServerUrl ?? this.Url, ObjectsEndpointSuffix); + this.SizesEndpointUrl = GetEndpointUrl(sizesCacheServerUrl ?? this.Url, SizesEndpointSuffix); } public string Url { get; } public string Name { get; } public bool GlobalDefault { get; } + [JsonIgnore] + public string PrefetchCacheServerUrl { get; } + + [JsonIgnore] + public string GetCacheServerUrl { get; } + + [JsonIgnore] + public string PostCacheServerUrl { get; } + + [JsonIgnore] + public string SizesCacheServerUrl { get; } + public string ObjectsEndpointUrl { get; } public string PrefetchEndpointUrl { get; } public string SizesEndpointUrl { get; } + [JsonIgnore] + public string ObjectsGetEndpointUrl { get; } + + [JsonIgnore] + public string ObjectsPostEndpointUrl { get; } + + [JsonIgnore] + public string GlobalPrefetchEndpointUrl { get; } + + [JsonIgnore] + public string GlobalSizesEndpointUrl { get; } + + public CacheServerInfo WithEndpointOverrides( + string prefetchCacheServerUrl, + string getCacheServerUrl, + string postCacheServerUrl, + string sizesCacheServerUrl) + { + return new CacheServerInfo( + this.Url, + this.Name, + this.GlobalDefault, + prefetchCacheServerUrl, + getCacheServerUrl, + postCacheServerUrl, + sizesCacheServerUrl); + } + + public CacheServerInfo WithEndpointOverridesFrom(CacheServerInfo cacheServer) + { + return this.WithEndpointOverrides( + cacheServer.PrefetchCacheServerUrl, + cacheServer.GetCacheServerUrl, + cacheServer.PostCacheServerUrl, + cacheServer.SizesCacheServerUrl); + } + public bool HasValidUrl() { - return Uri.IsWellFormedUriString(this.Url, UriKind.Absolute); + return IsValidUrl(this.Url); + } + + public static bool IsValidUrl(string url) + { + return Uri.IsWellFormedUriString(url, UriKind.Absolute); } public bool IsNone(string repoUrl) @@ -64,5 +139,10 @@ public static class ReservedNames public const string Default = "Default"; public const string UserDefined = "User Defined"; } + + private static string GetEndpointUrl(string cacheServerUrl, string endpointSuffix) + { + return cacheServerUrl == null ? null : cacheServerUrl + endpointSuffix; + } } } diff --git a/GVFS/GVFS.Common/Http/CacheServerResolver.cs b/GVFS/GVFS.Common/Http/CacheServerResolver.cs index bc1df9727b..7a0c6e1a73 100644 --- a/GVFS/GVFS.Common/Http/CacheServerResolver.cs +++ b/GVFS/GVFS.Common/Http/CacheServerResolver.cs @@ -20,10 +20,20 @@ public CacheServerResolver( public static CacheServerInfo GetCacheServerFromConfig(Enlistment enlistment) { + GitProcess git = enlistment.CreateGitProcess(); string url = GetUrlFromConfig(enlistment); + string prefetchCacheServerUrl = GetEndpointUrlFromConfig(git, GVFSConstants.GitConfig.PrefetchCacheServer); + string getCacheServerUrl = GetEndpointUrlFromConfig(git, GVFSConstants.GitConfig.GetCacheServer); + string postCacheServerUrl = GetEndpointUrlFromConfig(git, GVFSConstants.GitConfig.PostCacheServer); + string sizesCacheServerUrl = GetEndpointUrlFromConfig(git, GVFSConstants.GitConfig.SizesCacheServer); return new CacheServerInfo( url, - url == enlistment.RepoUrl ? CacheServerInfo.ReservedNames.None : null); + url == enlistment.RepoUrl ? CacheServerInfo.ReservedNames.None : null, + globalDefault: false, + prefetchCacheServerUrl, + getCacheServerUrl, + postCacheServerUrl, + sizesCacheServerUrl); } public static string GetUrlFromConfig(Enlistment enlistment) @@ -129,6 +139,22 @@ public bool TrySaveUrlToLocalConfig(CacheServerInfo cache, out string error) return result.ExitCodeIsSuccess; } + public bool TrySaveEndpointUrlsToLocalConfig(CacheServerInfo cache, out string error) + { + GitProcess git = this.enlistment.CreateGitProcess(); + + if (!TrySaveEndpointUrl(git, GVFSConstants.GitConfig.PrefetchCacheServer, cache.PrefetchCacheServerUrl, out error) || + !TrySaveEndpointUrl(git, GVFSConstants.GitConfig.GetCacheServer, cache.GetCacheServerUrl, out error) || + !TrySaveEndpointUrl(git, GVFSConstants.GitConfig.PostCacheServer, cache.PostCacheServerUrl, out error) || + !TrySaveEndpointUrl(git, GVFSConstants.GitConfig.SizesCacheServer, cache.SizesCacheServerUrl, out error)) + { + return false; + } + + error = null; + return true; + } + private static string GetValueFromConfig(GitProcess git, string configName, bool localOnly) { GitProcess.ConfigResult result = @@ -144,6 +170,30 @@ private static string GetValueFromConfig(GitProcess git, string configName, bool return value; } + private static string GetEndpointUrlFromConfig(GitProcess git, string configName) + { + string url = GetValueFromConfig(git, configName, localOnly: true); + if (url != null && !CacheServerInfo.IsValidUrl(url)) + { + throw new InvalidRepoException($"Invalid value for {configName}: '{url}' is not an absolute URL."); + } + + return url; + } + + private static bool TrySaveEndpointUrl(GitProcess git, string configName, string url, out string error) + { + error = null; + if (url == null) + { + return true; + } + + GitProcess.Result result = git.SetInLocalConfig(configName, url, replaceAll: true); + error = result.Errors; + return result.ExitCodeIsSuccess; + } + private static string GetDeprecatedCacheConfigSettingName(Enlistment enlistment) { string sectionUrl = diff --git a/GVFS/GVFS.Common/Http/GitEndPointResponseData.cs b/GVFS/GVFS.Common/Http/GitEndPointResponseData.cs index 0d450bf9d1..9eb6164219 100644 --- a/GVFS/GVFS.Common/Http/GitEndPointResponseData.cs +++ b/GVFS/GVFS.Common/Http/GitEndPointResponseData.cs @@ -4,6 +4,8 @@ using System.IO; using System.Net; using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; namespace GVFS.Common.Http { @@ -30,7 +32,7 @@ public GitEndPointResponseData(HttpStatusCode statusCode, Exception error, bool public GitEndPointResponseData(HttpStatusCode statusCode, string contentType, Stream responseStream, HttpResponseMessage message, Action onResponseDisposed) : this(statusCode, null, false, message, onResponseDisposed) { - this.Stream = responseStream; + this.Stream = responseStream == null ? null : new ReadErrorTrackingStream(responseStream); this.ContentType = MapContentType(contentType); } @@ -42,6 +44,11 @@ public GitEndPointResponseData(HttpStatusCode statusCode, string contentType, St public Stream Stream { get; private set; } + public bool StreamReadFailed + { + get { return this.Stream is ReadErrorTrackingStream trackingStream && trackingStream.ReadFailed; } + } + public bool HasErrors { get { return this.StatusCode != HttpStatusCode.OK; } @@ -70,7 +77,7 @@ public string RetryableReadToEnd() { return contentStreamReader.ReadToEnd(); } - catch (Exception ex) + catch (Exception ex) when (!(ex is OperationCanceledException)) { // All exceptions potentially from network should be retried throw new RetryableException("Exception while reading stream. See inner exception for details.", ex); @@ -99,7 +106,7 @@ public List RetryableReadAllLines() line = contentStreamReader.ReadLine(); } - catch (Exception ex) + catch (Exception ex) when (!(ex is OperationCanceledException)) { // All exceptions potentially from network should be retried throw new RetryableException("Exception while reading stream. See inner exception for details.", ex); @@ -159,5 +166,83 @@ private static GitObjectContentType MapContentType(string contentType) return GitObjectContentType.None; } } + + private sealed class ReadErrorTrackingStream : Stream + { + private readonly Stream innerStream; + + public ReadErrorTrackingStream(Stream innerStream) + { + this.innerStream = innerStream; + } + + public bool ReadFailed { get; private set; } + + public override bool CanRead => this.innerStream.CanRead; + + public override bool CanSeek => this.innerStream.CanSeek; + + public override bool CanWrite => this.innerStream.CanWrite; + + public override long Length => this.innerStream.Length; + + public override long Position + { + get { return this.innerStream.Position; } + set { this.innerStream.Position = value; } + } + + public override void Flush() => this.innerStream.Flush(); + + public override int Read(byte[] buffer, int offset, int count) => + this.TrackRead(() => this.innerStream.Read(buffer, offset, count)); + + public override int ReadByte() => this.TrackRead(this.innerStream.ReadByte); + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + this.TrackReadAsync(() => this.innerStream.ReadAsync(buffer, offset, count, cancellationToken)); + + public override long Seek(long offset, SeekOrigin origin) => this.innerStream.Seek(offset, origin); + + public override void SetLength(long value) => this.innerStream.SetLength(value); + + public override void Write(byte[] buffer, int offset, int count) => this.innerStream.Write(buffer, offset, count); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + this.innerStream.Dispose(); + } + + base.Dispose(disposing); + } + + private T TrackRead(Func read) + { + try + { + return read(); + } + catch (Exception ex) when (!(ex is OperationCanceledException)) + { + this.ReadFailed = true; + throw; + } + } + + private async Task TrackReadAsync(Func> read) + { + try + { + return await read().ConfigureAwait(false); + } + catch (Exception ex) when (!(ex is OperationCanceledException)) + { + this.ReadFailed = true; + throw; + } + } + } } } diff --git a/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs b/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs index 2cdffcb8da..278351b8ee 100644 --- a/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs @@ -2,6 +2,7 @@ using GVFS.Common.Tracing; using System; using System.Collections.Generic; +using System.IO; using System.Text.Json.Serialization; using System.Linq; using System.Net; @@ -34,8 +35,12 @@ public virtual List QueryForFileSizes(IEnumerable objectI long requestId = HttpRequestor.GetNewRequestId(); string objectIdsJson = ToJsonList(objectIds); - Uri cacheServerEndpoint = new Uri(this.CacheServer.SizesEndpointUrl); + Uri preferredCacheServerEndpoint = new Uri(this.CacheServer.SizesEndpointUrl); + Uri globalCacheServerEndpoint = new Uri(this.CacheServer.GlobalSizesEndpointUrl); Uri originEndpoint = new Uri(this.enlistment.RepoUrl + GVFSConstants.Endpoints.GVFSSizes); + bool hasEndpointOverride = preferredCacheServerEndpoint != globalCacheServerEndpoint; + bool useGlobalCacheServer = !hasEndpointOverride; + bool useOrigin = this.nextCacheServerAttemptTime >= DateTime.Now; EventMetadata metadata = new EventMetadata(); metadata.Add("RequestId", requestId); @@ -51,38 +56,91 @@ public virtual List QueryForFileSizes(IEnumerable objectI this.Tracer.RelatedEvent(EventLevel.Informational, "QueryFileSizes", metadata, Keywords.Network); - RetryWrapper> retrier = new RetryWrapper>(this.RetryConfig.MaxAttempts, cancellationToken); + RetryWrapper> retrier = new RetryWrapper>( + this.RetryConfig.MaxAttempts + (hasEndpointOverride && !useOrigin ? 2 : 0), + cancellationToken); retrier.OnFailure += RetryWrapper>.StandardErrorHandler(this.Tracer, requestId, "QueryFileSizes"); RetryWrapper>.InvocationResult requestTask = retrier.Invoke( tryCount => { Uri gvfsEndpoint; - if (this.nextCacheServerAttemptTime < DateTime.Now) + if (useOrigin) + { + gvfsEndpoint = originEndpoint; + } + else if (useGlobalCacheServer) { - gvfsEndpoint = cacheServerEndpoint; + gvfsEndpoint = globalCacheServerEndpoint; } else { - gvfsEndpoint = originEndpoint; + gvfsEndpoint = preferredCacheServerEndpoint; } - using (GitEndPointResponseData response = this.SendRequest(requestId, gvfsEndpoint, HttpMethod.Post, objectIdsJson, cancellationToken)) + try { - if (response.StatusCode == HttpStatusCode.NotFound) + using (GitEndPointResponseData response = this.SendProtocolRequest(requestId, gvfsEndpoint, HttpMethod.Post, objectIdsJson, cancellationToken)) { - this.nextCacheServerAttemptTime = DateTime.Now.AddDays(1); - return new RetryWrapper>.CallbackResult(response.Error, true); + if (response.HasErrors && !useGlobalCacheServer && !useOrigin) + { + this.TraceCacheServerFallback( + requestId, + preferredCacheServerEndpoint, + globalCacheServerEndpoint, + "EndpointSpecific", + "Global"); + useGlobalCacheServer = true; + return new RetryWrapper>.CallbackResult( + response.Error, + shouldRetry: true, + result: null, + shouldRecordFailure: false); + } + + if (response.StatusCode == HttpStatusCode.NotFound) + { + if (!useOrigin) + { + this.TraceCacheServerFallback( + requestId, + globalCacheServerEndpoint, + originEndpoint, + "Global", + "Origin"); + } + + this.nextCacheServerAttemptTime = DateTime.Now.AddDays(1); + useOrigin = true; + return new RetryWrapper>.CallbackResult( + response.Error, + shouldRetry: true, + result: null, + shouldRecordFailure: false); + } + + if (response.HasErrors) + { + return new RetryWrapper>.CallbackResult(response.Error, response.ShouldRetry); + } + + string objectSizesString = response.RetryableReadToEnd(); + List objectSizes = GVFSJsonOptions.Deserialize>(objectSizesString); + return new RetryWrapper>.CallbackResult(objectSizes); } - - if (response.HasErrors) - { - return new RetryWrapper>.CallbackResult(response.Error, response.ShouldRetry); - } - - string objectSizesString = response.RetryableReadToEnd(); - List objectSizes = GVFSJsonOptions.Deserialize>(objectSizesString); - return new RetryWrapper>.CallbackResult(objectSizes); + } + catch (Exception e) when ( + (e is HttpRequestException || e is IOException || e is RetryableException) && + !useGlobalCacheServer && + !useOrigin) + { + this.TraceCacheServerFallback(requestId, preferredCacheServerEndpoint, globalCacheServerEndpoint, "EndpointSpecific", "Global"); + useGlobalCacheServer = true; + return new RetryWrapper>.CallbackResult( + e, + shouldRetry: true, + result: null, + shouldRecordFailure: false); } }); @@ -109,7 +167,7 @@ public virtual GitRefs QueryInfoRefs(string branch) RetryWrapper.InvocationResult output = retrier.Invoke( tryCount => { - using (GitEndPointResponseData response = this.SendRequest( + using (GitEndPointResponseData response = this.SendProtocolRequest( requestId, infoRefsEndpoint, HttpMethod.Get, @@ -149,6 +207,7 @@ public virtual RetryWrapper.InvocationResult TryDownloadLoo onSuccess, eArgs => this.HandleDownloadAndSaveObjectError(retryOnFailure, requestId, eArgs), HttpMethod.Get, + new Uri(this.CacheServer.ObjectsGetEndpointUrl + "/" + objectId), new Uri(this.CacheServer.ObjectsEndpointUrl + "/" + objectId), cancellationToken, requestBody: null, @@ -170,10 +229,11 @@ public virtual RetryWrapper.InvocationResult TryDownloadObj onSuccess, onFailure, HttpMethod.Post, - new Uri(this.CacheServer.ObjectsEndpointUrl), - CancellationToken.None, - () => this.ObjectIdsJsonGenerator(requestId, objectIdGenerator), - preferBatchedLooseObjects ? CustomLooseObjectsHeader : null); + () => new Uri(this.CacheServer.ObjectsPostEndpointUrl), + requestBodyGenerator: () => this.ObjectIdsJsonGenerator(requestId, objectIdGenerator), + cancellationToken: CancellationToken.None, + acceptType: preferBatchedLooseObjects ? CustomLooseObjectsHeader : null, + fallbackEndPointGenerator: () => new Uri(this.CacheServer.ObjectsEndpointUrl)); } public virtual RetryWrapper.InvocationResult TryDownloadObjects( @@ -204,12 +264,38 @@ public virtual RetryWrapper.InvocationResult TryDownloadObj onSuccess, onFailure, HttpMethod.Post, + new Uri(this.CacheServer.ObjectsPostEndpointUrl), new Uri(this.CacheServer.ObjectsEndpointUrl), CancellationToken.None, objectIdsJson, preferBatchedLooseObjects ? CustomLooseObjectsHeader : null); } + public virtual RetryWrapper.InvocationResult TrySendProtocolRequest( + long requestId, + Func.CallbackResult> onSuccess, + Action.ErrorEventArgs> onFailure, + HttpMethod method, + Uri endPoint, + Uri fallbackEndPoint, + CancellationToken cancellationToken, + string requestBody = null, + MediaTypeWithQualityHeaderValue acceptType = null, + bool retryOnFailure = true) + { + return this.TrySendProtocolRequest( + requestId, + onSuccess, + onFailure, + method, + () => endPoint, + requestBodyGenerator: () => requestBody, + cancellationToken: cancellationToken, + acceptType: acceptType, + retryOnFailure: retryOnFailure, + fallbackEndPointGenerator: () => fallbackEndPoint); + } + public virtual RetryWrapper.InvocationResult TrySendProtocolRequest( long requestId, Func.CallbackResult> onSuccess, @@ -227,10 +313,11 @@ public virtual RetryWrapper.InvocationResult TrySendProtoco onFailure, method, endPoint, - cancellationToken, - () => requestBody, - acceptType, - retryOnFailure); + fallbackEndPoint: null, + cancellationToken: cancellationToken, + requestBody: requestBody, + acceptType: acceptType, + retryOnFailure: retryOnFailure); } public virtual RetryWrapper.InvocationResult TrySendProtocolRequest( @@ -250,10 +337,10 @@ public virtual RetryWrapper.InvocationResult TrySendProtoco onFailure, method, () => endPoint, - requestBodyGenerator, - cancellationToken, - acceptType, - retryOnFailure); + requestBodyGenerator: requestBodyGenerator, + cancellationToken: cancellationToken, + acceptType: acceptType, + retryOnFailure: retryOnFailure); } public virtual RetryWrapper.InvocationResult TrySendProtocolRequest( @@ -265,10 +352,16 @@ public virtual RetryWrapper.InvocationResult TrySendProtoco Func requestBodyGenerator, CancellationToken cancellationToken, MediaTypeWithQualityHeaderValue acceptType = null, - bool retryOnFailure = true) + bool retryOnFailure = true, + Func fallbackEndPointGenerator = null) { + Uri endPoint = endPointGenerator(); + Uri fallbackEndPoint = fallbackEndPointGenerator?.Invoke(); + bool hasFallbackEndPoint = fallbackEndPoint != null && endPoint != fallbackEndPoint; + bool useFallbackEndPoint = false; + RetryWrapper retrier = new RetryWrapper( - retryOnFailure ? this.RetryConfig.MaxAttempts : 1, + (retryOnFailure ? this.RetryConfig.MaxAttempts : 1) + (hasFallbackEndPoint ? 1 : 0), cancellationToken); if (onFailure != null) { @@ -278,24 +371,182 @@ public virtual RetryWrapper.InvocationResult TrySendProtoco return retrier.Invoke( tryCount => { - using (GitEndPointResponseData response = this.SendRequest( - requestId, - endPointGenerator(), - method, - requestBodyGenerator(), - cancellationToken, - acceptType)) + Uri requestEndPoint = useFallbackEndPoint ? fallbackEndPointGenerator() : endPointGenerator(); + GitEndPointResponseData response; + + try + { + response = this.SendProtocolRequest( + requestId, + requestEndPoint, + method, + requestBodyGenerator(), + cancellationToken, + acceptType); + } + catch (HttpRequestException e) + { + return this.HandleProtocolException( + requestId, + e, + requestEndPoint, + fallbackEndPoint, + hasFallbackEndPoint, + ref useFallbackEndPoint, + retryOnFailure); + } + catch (IOException e) + { + return this.HandleProtocolException( + requestId, + e, + requestEndPoint, + fallbackEndPoint, + hasFallbackEndPoint, + ref useFallbackEndPoint, + retryOnFailure); + } + catch (RetryableException e) + { + return this.HandleProtocolException( + requestId, + e, + requestEndPoint, + fallbackEndPoint, + hasFallbackEndPoint, + ref useFallbackEndPoint, + retryOnFailure); + } + + using (response) { if (response.HasErrors) { - return new RetryWrapper.CallbackResult(response.Error, response.ShouldRetry, new GitObjectTaskResult(response.StatusCode)); + bool shouldFallBack = hasFallbackEndPoint && !useFallbackEndPoint; + if (shouldFallBack) + { + this.TraceCacheServerFallback( + requestId, + requestEndPoint, + fallbackEndPoint, + "EndpointSpecific", + "Global"); + } + + useFallbackEndPoint |= shouldFallBack; + return new RetryWrapper.CallbackResult( + response.Error, + shouldFallBack || response.ShouldRetry, + new GitObjectTaskResult(response.StatusCode, requestEndPoint), + shouldRecordFailure: response.ShouldRetry && !shouldFallBack); } - return onSuccess(tryCount, response); + RetryWrapper.CallbackResult result; + try + { + result = onSuccess(tryCount, response); + } + catch (Exception e) + { + if (response.StreamReadFailed) + { + return this.HandleProtocolException( + requestId, + e, + requestEndPoint, + fallbackEndPoint, + hasFallbackEndPoint, + ref useFallbackEndPoint, + retryOnFailure); + } + + throw; + } + + if (result.HasErrors) + { + bool shouldFallBack = response.StreamReadFailed && hasFallbackEndPoint && !useFallbackEndPoint; + if (shouldFallBack) + { + this.TraceCacheServerFallback( + requestId, + requestEndPoint, + fallbackEndPoint, + "EndpointSpecific", + "Global"); + useFallbackEndPoint = true; + } + + GitObjectTaskResult requestResult = result.Result == null + ? new GitObjectTaskResult(success: false, requestEndPoint) + : result.Result.WithRequestUri(requestEndPoint); + return new RetryWrapper.CallbackResult( + result.Error, + shouldFallBack || result.ShouldRetry, + requestResult, + shouldRecordFailure: result.ShouldRecordFailure && !shouldFallBack); + } + + return result; } }); } + private RetryWrapper.CallbackResult HandleProtocolException( + long requestId, + Exception error, + Uri requestEndPoint, + Uri fallbackEndPoint, + bool hasFallbackEndPoint, + ref bool useFallbackEndPoint, + bool retryOnFailure) + { + bool shouldFallBack = hasFallbackEndPoint && !useFallbackEndPoint; + if (shouldFallBack) + { + this.TraceCacheServerFallback( + requestId, + requestEndPoint, + fallbackEndPoint, + "EndpointSpecific", + "Global"); + useFallbackEndPoint = true; + } + + return new RetryWrapper.CallbackResult( + error, + shouldFallBack || retryOnFailure, + new GitObjectTaskResult(success: false, requestEndPoint), + shouldRecordFailure: retryOnFailure && !shouldFallBack); + } + + private void TraceCacheServerFallback( + long requestId, + Uri source, + Uri target, + string sourceRoute, + string targetRoute) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("RequestId", requestId); + metadata.Add("SourceRoute", sourceRoute); + metadata.Add("SourceAuthority", GetAuthorityForTelemetry(source)); + metadata.Add("TargetRoute", targetRoute); + metadata.Add("TargetAuthority", GetAuthorityForTelemetry(target)); + this.Tracer.RelatedEvent(EventLevel.Informational, "CacheServerFallback", metadata, Keywords.Network | Keywords.Telemetry); + } + + protected virtual GitEndPointResponseData SendProtocolRequest( + long requestId, + Uri requestUri, + HttpMethod httpMethod, + string requestContent, + CancellationToken cancellationToken, + MediaTypeWithQualityHeaderValue acceptType = null) + { + return this.SendRequest(requestId, requestUri, httpMethod, requestContent, cancellationToken, acceptType); + } + private static string ToJsonList(IEnumerable strings) { return "[\"" + string.Join("\",\"", strings) + "\"]"; @@ -356,19 +607,29 @@ public GitObjectSize(string id, long size) public class GitObjectTaskResult { - public GitObjectTaskResult(bool success) + public GitObjectTaskResult(bool success, Uri requestUri = null) { this.Success = success; + this.RequestUri = requestUri; } - public GitObjectTaskResult(HttpStatusCode statusCode) - : this(statusCode == HttpStatusCode.OK) + public GitObjectTaskResult(HttpStatusCode statusCode, Uri requestUri = null) + : this(statusCode == HttpStatusCode.OK, requestUri) { this.HttpStatusCodeResult = statusCode; } public bool Success { get; } - public HttpStatusCode HttpStatusCodeResult { get; } + public HttpStatusCode HttpStatusCodeResult { get; private set; } + public Uri RequestUri { get; } + + public GitObjectTaskResult WithRequestUri(Uri requestUri) + { + return new GitObjectTaskResult(this.Success, requestUri) + { + HttpStatusCodeResult = this.HttpStatusCodeResult, + }; + } } } } \ No newline at end of file diff --git a/GVFS/GVFS.Common/Http/HttpRequestor.cs b/GVFS/GVFS.Common/Http/HttpRequestor.cs index 435e52c2b1..ee1558abf2 100644 --- a/GVFS/GVFS.Common/Http/HttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/HttpRequestor.cs @@ -312,11 +312,11 @@ protected GitEndPointResponseData SendRequest( } private static bool ShouldRetry(HttpStatusCode statusCode) - { + { // Retry timeout, Unauthorized, 429 (Too Many Requests), and 5xx errors int statusInt = (int)statusCode; if (statusCode == HttpStatusCode.RequestTimeout || - statusCode == HttpStatusCode.Unauthorized || + statusCode == HttpStatusCode.Unauthorized || statusInt == 429 || (statusInt >= 500 && statusInt < 600)) { @@ -378,6 +378,11 @@ internal static bool ShouldRejectCredentials(HttpStatusCode statusCode, string r return false; } + internal static string GetAuthorityForTelemetry(Uri uri) + { + return uri.Authority; + } + private static string GetSingleHeaderOrEmpty(HttpHeaders headers, string headerName) { IEnumerable values; diff --git a/GVFS/GVFS.Common/RetryWrapper.cs b/GVFS/GVFS.Common/RetryWrapper.cs index 4d6a0ccd84..4d56ccc1fa 100644 --- a/GVFS/GVFS.Common/RetryWrapper.cs +++ b/GVFS/GVFS.Common/RetryWrapper.cs @@ -88,7 +88,7 @@ public InvocationResult Invoke(Func toInvoke) CallbackResult result = toInvoke(tryCount); if (result.HasErrors) { - if (result.ShouldRetry) + if (result.ShouldRecordFailure) { RetryCircuitBreaker.RecordFailure(); } @@ -224,6 +224,7 @@ public CallbackResult(Exception error, bool shouldRetry) this.HasErrors = true; this.Error = error; this.ShouldRetry = shouldRetry; + this.ShouldRecordFailure = shouldRetry; } public CallbackResult(Exception error, bool shouldRetry, T result) @@ -232,9 +233,16 @@ public CallbackResult(Exception error, bool shouldRetry, T result) this.Result = result; } + public CallbackResult(Exception error, bool shouldRetry, T result, bool shouldRecordFailure) + : this(error, shouldRetry, result) + { + this.ShouldRecordFailure = shouldRecordFailure; + } + public bool HasErrors { get; } public Exception Error { get; } public bool ShouldRetry { get; } + public bool ShouldRecordFailure { get; } public T Result { get; } } } diff --git a/GVFS/GVFS.Mount/InProcessMount.cs b/GVFS/GVFS.Mount/InProcessMount.cs index 629be66138..0a5c930b5f 100644 --- a/GVFS/GVFS.Mount/InProcessMount.cs +++ b/GVFS/GVFS.Mount/InProcessMount.cs @@ -353,7 +353,10 @@ private void MountWithLockAcquired(EventLevel verbosity, Keywords keywords) this.mountProgressMessage = "Resolving cache server"; CacheServerResolver cacheServerResolver = new CacheServerResolver(this.tracer, this.enlistment); - this.cacheServer = cacheServerResolver.ResolveNameFromRemote(this.cacheServer.Url, serverGVFSConfig); + CacheServerInfo cacheServerFromConfig = this.cacheServer; + this.cacheServer = cacheServerResolver + .ResolveNameFromRemote(cacheServerFromConfig.Url, serverGVFSConfig) + .WithEndpointOverridesFrom(cacheServerFromConfig); this.tracer.RelatedEvent( EventLevel.Informational, diff --git a/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs b/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs index 852ecb908a..f338fedadf 100644 --- a/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs +++ b/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs @@ -13,6 +13,10 @@ public class CacheServerResolverTests { private const string CacheServerUrl = "https://cache/server"; private const string CacheServerName = "TestCacheServer"; + private const string PrefetchCacheServerUrl = "https://prefetch-cache/server"; + private const string GetCacheServerUrl = "https://get-cache/server"; + private const string PostCacheServerUrl = "https://post-cache/server"; + private const string SizesCacheServerUrl = "https://sizes-cache/server"; [TestCase] public void CanGetCacheServerFromNewConfig() @@ -43,6 +47,90 @@ public void CanGetCacheServerWithNoConfig() CacheServerResolver.GetUrlFromConfig(enlistment).ShouldEqual(enlistment.RepoUrl); } + [TestCase] + public void EndpointSpecificCacheServersOverrideGlobalCacheServer() + { + MockGVFSEnlistment enlistment = this.CreateEnlistment( + CacheServerUrl, + prefetchCacheServerUrl: PrefetchCacheServerUrl, + getCacheServerUrl: GetCacheServerUrl, + postCacheServerUrl: PostCacheServerUrl, + sizesCacheServerUrl: SizesCacheServerUrl); + + CacheServerInfo cacheServer = CacheServerResolver.GetCacheServerFromConfig(enlistment); + + cacheServer.PrefetchEndpointUrl.ShouldEqual(PrefetchCacheServerUrl + "/gvfs/prefetch"); + cacheServer.ObjectsGetEndpointUrl.ShouldEqual(GetCacheServerUrl + "/gvfs/objects"); + cacheServer.ObjectsPostEndpointUrl.ShouldEqual(PostCacheServerUrl + "/gvfs/objects"); + cacheServer.SizesEndpointUrl.ShouldEqual(SizesCacheServerUrl + "/gvfs/sizes"); + } + + [TestCase] + public void EndpointSpecificCacheServersFallBackToGlobalCacheServer() + { + CacheServerInfo cacheServer = CacheServerResolver.GetCacheServerFromConfig(this.CreateEnlistment(CacheServerUrl)); + + cacheServer.PrefetchEndpointUrl.ShouldEqual(CacheServerUrl + "/gvfs/prefetch"); + cacheServer.ObjectsGetEndpointUrl.ShouldEqual(CacheServerUrl + "/gvfs/objects"); + cacheServer.ObjectsPostEndpointUrl.ShouldEqual(CacheServerUrl + "/gvfs/objects"); + cacheServer.SizesEndpointUrl.ShouldEqual(CacheServerUrl + "/gvfs/sizes"); + } + + [TestCase] + public void EndpointSpecificCacheServersArePreservedWhenGlobalCacheServerIsResolved() + { + CacheServerInfo configuredCacheServer = new CacheServerInfo(CacheServerUrl, CacheServerName) + .WithEndpointOverrides(PrefetchCacheServerUrl, GetCacheServerUrl, PostCacheServerUrl, SizesCacheServerUrl); + CacheServerInfo resolvedCacheServer = new CacheServerInfo("https://resolved-cache/server", "ResolvedCache") + .WithEndpointOverridesFrom(configuredCacheServer); + + resolvedCacheServer.PrefetchCacheServerUrl.ShouldEqual(PrefetchCacheServerUrl); + resolvedCacheServer.GetCacheServerUrl.ShouldEqual(GetCacheServerUrl); + resolvedCacheServer.PostCacheServerUrl.ShouldEqual(PostCacheServerUrl); + resolvedCacheServer.SizesCacheServerUrl.ShouldEqual(SizesCacheServerUrl); + resolvedCacheServer.HasValidUrl().ShouldEqual(true); + } + + [TestCase] + public void InvalidEndpointSpecificCacheServerIsRejected() + { + MockGVFSEnlistment enlistment = this.CreateEnlistment( + CacheServerUrl, + prefetchCacheServerUrl: "not-a-url"); + + InvalidRepoException exception = Assert.Throws( + () => CacheServerResolver.GetCacheServerFromConfig(enlistment)); + + exception.Message.ShouldContain(GVFSConstants.GitConfig.PrefetchCacheServer); + exception.Message.ShouldContain("not an absolute URL"); + } + + [TestCase] + public void CanSaveEndpointSpecificCacheServers() + { + MockGVFSEnlistment enlistment = this.CreateEnlistment(); + MockGitProcess git = (MockGitProcess)enlistment.CreateGitProcess(); + git.SetExpectedCommandResult( + "config --local --replace-all \"gvfs.prefetch.cache-server\" \"https://prefetch-cache/server\"", + () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.SuccessCode)); + git.SetExpectedCommandResult( + "config --local --replace-all \"gvfs.get.cache-server\" \"https://get-cache/server\"", + () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.SuccessCode)); + git.SetExpectedCommandResult( + "config --local --replace-all \"gvfs.post.cache-server\" \"https://post-cache/server\"", + () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.SuccessCode)); + git.SetExpectedCommandResult( + "config --local --replace-all \"gvfs.sizes.cache-server\" \"https://sizes-cache/server\"", + () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.SuccessCode)); + + CacheServerInfo cacheServer = new CacheServerInfo(CacheServerUrl, CacheServerName) + .WithEndpointOverrides(PrefetchCacheServerUrl, GetCacheServerUrl, PostCacheServerUrl, SizesCacheServerUrl); + + new CacheServerResolver(new MockTracer(), enlistment) + .TrySaveEndpointUrlsToLocalConfig(cacheServer, out string error) + .ShouldEqual(true, error); + } + [TestCase] public void CanResolveUrlForKnownName() { @@ -190,7 +278,13 @@ private void ValidateIsNone(Enlistment enlistment, CacheServerInfo cacheServer) cacheServer.Name.ShouldEqual(CacheServerInfo.ReservedNames.None); } - private MockGVFSEnlistment CreateEnlistment(string newConfigValue = null, string oldConfigValue = null) + private MockGVFSEnlistment CreateEnlistment( + string newConfigValue = null, + string oldConfigValue = null, + string prefetchCacheServerUrl = null, + string getCacheServerUrl = null, + string postCacheServerUrl = null, + string sizesCacheServerUrl = null) { MockGitProcess gitProcess = new MockGitProcess(); gitProcess.SetExpectedCommandResult( @@ -199,6 +293,18 @@ private MockGVFSEnlistment CreateEnlistment(string newConfigValue = null, string gitProcess.SetExpectedCommandResult( "config gvfs.mock:..repourl.cache-server-url", () => new GitProcess.Result(oldConfigValue ?? string.Empty, string.Empty, oldConfigValue != null ? GitProcess.Result.SuccessCode : GitProcess.Result.GenericFailureCode)); + gitProcess.SetExpectedCommandResult( + "config --local gvfs.prefetch.cache-server", + () => new GitProcess.Result(prefetchCacheServerUrl ?? string.Empty, string.Empty, prefetchCacheServerUrl != null ? GitProcess.Result.SuccessCode : GitProcess.Result.GenericFailureCode)); + gitProcess.SetExpectedCommandResult( + "config --local gvfs.get.cache-server", + () => new GitProcess.Result(getCacheServerUrl ?? string.Empty, string.Empty, getCacheServerUrl != null ? GitProcess.Result.SuccessCode : GitProcess.Result.GenericFailureCode)); + gitProcess.SetExpectedCommandResult( + "config --local gvfs.post.cache-server", + () => new GitProcess.Result(postCacheServerUrl ?? string.Empty, string.Empty, postCacheServerUrl != null ? GitProcess.Result.SuccessCode : GitProcess.Result.GenericFailureCode)); + gitProcess.SetExpectedCommandResult( + "config --local gvfs.sizes.cache-server", + () => new GitProcess.Result(sizesCacheServerUrl ?? string.Empty, string.Empty, sizesCacheServerUrl != null ? GitProcess.Result.SuccessCode : GitProcess.Result.GenericFailureCode)); return new MockGVFSEnlistment(gitProcess); } diff --git a/GVFS/GVFS.UnitTests/Git/GitObjectsTests.cs b/GVFS/GVFS.UnitTests/Git/GitObjectsTests.cs index 88be433806..78236a37bc 100644 --- a/GVFS/GVFS.UnitTests/Git/GitObjectsTests.cs +++ b/GVFS/GVFS.UnitTests/Git/GitObjectsTests.cs @@ -1,13 +1,19 @@ using GVFS.Common; using GVFS.Common.Git; +using GVFS.Common.Http; using GVFS.Common.Tracing; using GVFS.Tests.Should; using GVFS.UnitTests.Mock.Common; using GVFS.UnitTests.Mock.FileSystem; using NUnit.Framework; +using System; using System.Collections.Generic; using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; using System.Security; +using System.Threading; namespace GVFS.UnitTests.Git { @@ -124,6 +130,61 @@ public void WriteLooseObject_Success() moved.ShouldBeTrue("File was not moved"); } + [TestCase] + public void PrefetchFailureTelemetryReportsOnlyRequestAuthority() + { + const string RequestUrl = "https://user:secret@cache.example:8443/gvfs/prefetch?token=sensitive"; + MockTracer tracer = new MockTracer(); + MockGVFSEnlistment enlistment = new MockGVFSEnlistment(); + TestPrefetchRequestor requestor = new TestPrefetchRequestor( + tracer, + enlistment, + HttpStatusCode.ServiceUnavailable, + new Uri(RequestUrl)); + GitObjects gitObjects = new GVFSGitObjects( + new GVFSContext(tracer, new MockFileSystemWithCallbacks(), null, enlistment), + requestor); + + gitObjects.TryDownloadPrefetchPacks( + gitProcess: null, + latestTimestamp: 0, + trustPackIndexes: false, + out List _) + .ShouldEqual(false); + + tracer.StartActivityTracer.RelatedWarningEvents.Count.ShouldEqual(1); + tracer.StartActivityTracer.RelatedWarningEvents[0].ShouldContain("\"PrefetchEndpointUrl\":\"cache.example:8443\""); + tracer.StartActivityTracer.RelatedWarningEvents[0].IndexOf("user", StringComparison.Ordinal).ShouldEqual(-1); + tracer.StartActivityTracer.RelatedWarningEvents[0].IndexOf("secret", StringComparison.Ordinal).ShouldEqual(-1); + tracer.StartActivityTracer.RelatedWarningEvents[0].IndexOf("sensitive", StringComparison.Ordinal).ShouldEqual(-1); + } + + [TestCase] + public void UnsupportedPrefetchTelemetryReportsOnlyRequestAuthority() + { + const string RequestUrl = "https://user:secret@cache.example:8443/gvfs/prefetch?token=sensitive"; + MockTracer tracer = new MockTracer(); + MockGVFSEnlistment enlistment = new MockGVFSEnlistment(); + TestPrefetchRequestor requestor = new TestPrefetchRequestor( + tracer, + enlistment, + HttpStatusCode.NotFound, + new Uri(RequestUrl)); + GitObjects gitObjects = new GVFSGitObjects( + new GVFSContext(tracer, new MockFileSystemWithCallbacks(), null, enlistment), + requestor); + + gitObjects.TryDownloadPrefetchPacks( + gitProcess: null, + latestTimestamp: 0, + trustPackIndexes: false, + out List _) + .ShouldEqual(false); + + EventMetadata metadata = tracer.StartActivityTracer.RelatedEventMetadata[0]; + metadata["PrefetchEndpointUrl"].ShouldEqual("cache.example:8443"); + } + private Stream OnOpenFileStream(string path, FileMode mode, FileAccess access) { this.openedPaths.Add(path); @@ -144,5 +205,40 @@ private bool OnFileExists(string path) { return this.pathsToData.TryGetValue(path, out _); } + + private class TestPrefetchRequestor : GitObjectsHttpRequestor + { + private readonly HttpStatusCode statusCode; + private readonly Uri requestUri; + + public TestPrefetchRequestor( + ITracer tracer, + Enlistment enlistment, + HttpStatusCode statusCode, + Uri requestUri) + : base(tracer, enlistment, new CacheServerInfo("https://cache.example/server", "cache"), new RetryConfig(0)) + { + this.statusCode = statusCode; + this.requestUri = requestUri; + } + + public override RetryWrapper.InvocationResult TrySendProtocolRequest( + long requestId, + Func.CallbackResult> onSuccess, + Action.ErrorEventArgs> onFailure, + HttpMethod method, + Func endPointGenerator, + Func requestBodyGenerator, + CancellationToken cancellationToken, + MediaTypeWithQualityHeaderValue acceptType = null, + bool retryOnFailure = true, + Func fallbackEndPointGenerator = null) + { + return new RetryWrapper.InvocationResult( + tryCount: 1, + new GitObjectsHttpException(this.statusCode, "Test failure"), + new GitObjectTaskResult(this.statusCode, this.requestUri)); + } + } } } diff --git a/GVFS/GVFS.UnitTests/Http/GitObjectsHttpRequestorTests.cs b/GVFS/GVFS.UnitTests/Http/GitObjectsHttpRequestorTests.cs new file mode 100644 index 0000000000..ee89a68ae2 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Http/GitObjectsHttpRequestorTests.cs @@ -0,0 +1,485 @@ +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Http; +using GVFS.Common.Tracing; +using GVFS.Tests.Should; +using GVFS.UnitTests.Mock.Common; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; + +namespace GVFS.UnitTests.Http +{ + [TestFixture] + public class GitObjectsHttpRequestorTests + { + private const string GlobalCacheServerUrl = "https://global-cache/server"; + private const string EndpointCacheServerUrl = "https://endpoint-cache/server"; + + [SetUp] + public void SetUp() + { + RetryCircuitBreaker.Reset(); + } + + [TearDown] + public void TearDown() + { + RetryCircuitBreaker.Reset(); + } + + [TestCase] + public void LooseObjectFallsBackToGlobalCacheServerWhenEndpointRequestFails() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + requestor.EnqueueResponse(HttpStatusCode.OK); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadLooseObject( + "0123456789abcdef", + retryOnFailure: false, + CancellationToken.None, + requestSource: "test", + onSuccess: SuccessfulRequest); + + result.Succeeded.ShouldEqual(true); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects/0123456789abcdef"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects/0123456789abcdef"); + requestor.TestTracer.RelatedEventNames.ShouldContain(name => name == "CacheServerFallback"); + requestor.TestTracer.RelatedEventKeywords.ShouldContain( + keywords => (keywords & Keywords.Telemetry) == Keywords.Telemetry); + } + + [TestCase] + public void BatchedObjectRequestFallsBackToGlobalCacheServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable, shouldRetry: true); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: SuccessfulRequest, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(false); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void PrefetchRequestFallsBackToGlobalCacheServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(HttpStatusCode.BadRequest); + requestor.EnqueueResponse(HttpStatusCode.OK); + + RetryWrapper.InvocationResult result = + requestor.TrySendProtocolRequest( + requestId: 1, + onSuccess: SuccessfulRequest, + onFailure: null, + method: HttpMethod.Get, + endPointGenerator: () => new Uri(EndpointCacheServerUrl + "/gvfs/prefetch?lastPackTimestamp=0"), + fallbackEndPointGenerator: () => new Uri(GlobalCacheServerUrl + "/gvfs/prefetch?lastPackTimestamp=0"), + requestBodyGenerator: () => null, + cancellationToken: CancellationToken.None); + + result.Succeeded.ShouldEqual(true); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/prefetch?lastPackTimestamp=0"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/prefetch?lastPackTimestamp=0"); + } + + [TestCase] + public void TransportExceptionFallsBackToGlobalCacheServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueException(new HttpRequestException("Test failure")); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: SuccessfulRequest, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(false); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void ResponseBodyReadFailureFallsBackToGlobalCacheServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(new ThrowingReadStream()); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: (tryCount, response) => + { + response.Stream.ReadByte(); + return SuccessfulRequest(tryCount, response); + }, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(false); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void ResponseBodyReadFailureReportedByHandlerFallsBackToGlobalCacheServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(new ThrowingReadStream()); + requestor.EnqueueResponse(HttpStatusCode.OK); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: (tryCount, response) => + { + try + { + response.Stream.ReadByte(); + return SuccessfulRequest(tryCount, response); + } + catch (IOException e) + { + return new RetryWrapper.CallbackResult( + e, + shouldRetry: true); + } + }, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(true); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void ResponseBodyCancellationIsNotRetriedOrReportedAsFallback() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 1); + requestor.EnqueueResponse(new CancelingReadStream()); + + Assert.Throws( + () => requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: (tryCount, response) => + { + response.RetryableReadToEnd(); + return SuccessfulRequest(tryCount, response); + }, + onFailure: null, + preferBatchedLooseObjects: false)); + + requestor.RequestUris.Count.ShouldEqual(1); + requestor.TestTracer.RelatedEventNames.ShouldNotContain(name => name == "CacheServerFallback"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void FallbackTelemetryExcludesCredentialsAndRequestPath() + { + const string CredentialedGlobalUrl = "https://global-user:global-secret@global-cache:8443/server"; + const string CredentialedEndpointUrl = "https://endpoint-user:endpoint-secret@endpoint-cache:9443/server"; + TestGitObjectsHttpRequestor requestor = this.CreateRequestor( + maxRetries: 0, + globalCacheServerUrl: CredentialedGlobalUrl, + endpointCacheServerUrl: CredentialedEndpointUrl); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + requestor.EnqueueResponse(HttpStatusCode.OK); + + requestor.TryDownloadLooseObject( + "0123456789abcdef", + retryOnFailure: false, + CancellationToken.None, + requestSource: "test", + onSuccess: SuccessfulRequest); + + int fallbackEventIndex = requestor.TestTracer.RelatedEventNames.IndexOf("CacheServerFallback"); + EventMetadata metadata = requestor.TestTracer.RelatedEventMetadata[fallbackEventIndex]; + metadata["SourceAuthority"].ShouldEqual("endpoint-cache:9443"); + metadata["TargetAuthority"].ShouldEqual("global-cache:8443"); + } + + [TestCase] + public void NoEndpointOverrideUsesNormalGlobalCacheRetries() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 1, endpointOverrides: false); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable, shouldRetry: true); + requestor.EnqueueResponse(HttpStatusCode.OK); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: SuccessfulRequest, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(true); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + requestor.TestTracer.RelatedEventNames.ShouldNotContain(name => name == "CacheServerFallback"); + } + + [TestCase] + public void TerminalFallbackFailureReportsGlobalCacheServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: SuccessfulRequest, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(false); + result.Attempts.ShouldEqual(2); + result.Result.HttpStatusCodeResult.ShouldEqual(HttpStatusCode.NotFound); + result.Result.RequestUri.AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/objects"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void SuccessHandlerFailureRetriesTheEndpointSpecificServer() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 1); + requestor.EnqueueResponse(HttpStatusCode.OK); + requestor.EnqueueResponse(HttpStatusCode.OK); + int successHandlerCalls = 0; + + RetryWrapper.InvocationResult result = + requestor.TryDownloadObjects( + new[] { "0123456789abcdef" }, + onSuccess: (tryCount, response) => + { + if (++successHandlerCalls == 1) + { + throw new RetryableException("Local write failed"); + } + + return SuccessfulRequest(tryCount, response); + }, + onFailure: null, + preferBatchedLooseObjects: false); + + result.Succeeded.ShouldEqual(true); + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/objects"); + requestor.TestTracer.RelatedEventNames.ShouldNotContain(name => name == "CacheServerFallback"); + } + + [TestCase] + public void SizesRequestFallsBackThroughGlobalCacheServerToOrigin() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable); + requestor.EnqueueResponse(HttpStatusCode.NotFound); + requestor.EnqueueResponse(HttpStatusCode.OK, "[]"); + + requestor.QueryForFileSizes(new[] { "0123456789abcdef" }, CancellationToken.None); + + requestor.RequestUris.Count.ShouldEqual(3); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/sizes"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/sizes"); + requestor.RequestUris[2].AbsoluteUri.ShouldEqual("mock://repourl/gvfs/sizes"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void SizesHttpFallbackDoesNotChargeCircuitBreaker() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable, shouldRetry: true); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable); + + requestor.QueryForFileSizes(new[] { "0123456789abcdef" }, CancellationToken.None); + + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/sizes"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/sizes"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + [TestCase] + public void SizesTransportFallbackDoesNotChargeCircuitBreaker() + { + TestGitObjectsHttpRequestor requestor = this.CreateRequestor(maxRetries: 0); + requestor.EnqueueException(new HttpRequestException("Test failure")); + requestor.EnqueueResponse(HttpStatusCode.ServiceUnavailable); + + requestor.QueryForFileSizes(new[] { "0123456789abcdef" }, CancellationToken.None); + + requestor.RequestUris.Count.ShouldEqual(2); + requestor.RequestUris[0].AbsoluteUri.ShouldEqual(EndpointCacheServerUrl + "/gvfs/sizes"); + requestor.RequestUris[1].AbsoluteUri.ShouldEqual(GlobalCacheServerUrl + "/gvfs/sizes"); + RetryCircuitBreaker.ConsecutiveFailures.ShouldEqual(0); + } + + private static RetryWrapper.CallbackResult SuccessfulRequest( + int tryCount, + GitEndPointResponseData response) + { + return new RetryWrapper.CallbackResult( + new GitObjectsHttpRequestor.GitObjectTaskResult(true)); + } + + private TestGitObjectsHttpRequestor CreateRequestor( + int maxRetries, + bool endpointOverrides = true, + string globalCacheServerUrl = GlobalCacheServerUrl, + string endpointCacheServerUrl = EndpointCacheServerUrl) + { + CacheServerInfo cacheServer = new CacheServerInfo(globalCacheServerUrl, "global"); + if (endpointOverrides) + { + cacheServer = cacheServer.WithEndpointOverrides( + endpointCacheServerUrl, + endpointCacheServerUrl, + endpointCacheServerUrl, + endpointCacheServerUrl); + } + + return new TestGitObjectsHttpRequestor( + new MockGVFSEnlistment(), + cacheServer, + new RetryConfig(maxRetries)); + } + + private class TestGitObjectsHttpRequestor : GitObjectsHttpRequestor + { + private readonly Queue responses = new Queue(); + + public TestGitObjectsHttpRequestor( + Enlistment enlistment, + CacheServerInfo cacheServer, + RetryConfig retryConfig) + : this(new MockTracer(), enlistment, cacheServer, retryConfig) + { + } + + private TestGitObjectsHttpRequestor( + MockTracer tracer, + Enlistment enlistment, + CacheServerInfo cacheServer, + RetryConfig retryConfig) + : base(tracer, enlistment, cacheServer, retryConfig) + { + this.TestTracer = tracer; + this.RequestUris = new List(); + } + + public MockTracer TestTracer { get; } + public List RequestUris { get; } + + public void EnqueueResponse(HttpStatusCode statusCode, string body = "", bool shouldRetry = false) + { + this.responses.Enqueue(Tuple.Create(statusCode, body, shouldRetry)); + } + + public void EnqueueException(Exception exception) + { + this.responses.Enqueue(exception); + } + + public void EnqueueResponse(Stream stream) + { + this.responses.Enqueue(stream); + } + + protected override GitEndPointResponseData SendProtocolRequest( + long requestId, + Uri requestUri, + HttpMethod httpMethod, + string requestContent, + CancellationToken cancellationToken, + MediaTypeWithQualityHeaderValue acceptType = null) + { + this.RequestUris.Add(requestUri); + object nextResponse = this.responses.Dequeue(); + if (nextResponse is Exception exception) + { + throw exception; + } + + if (nextResponse is Stream stream) + { + return new GitEndPointResponseData( + HttpStatusCode.OK, + "application/json", + stream, + message: null, + onResponseDisposed: null); + } + + Tuple response = (Tuple)nextResponse; + + if (response.Item1 == HttpStatusCode.OK) + { + return new GitEndPointResponseData( + response.Item1, + "application/json", + new MemoryStream(Encoding.UTF8.GetBytes(response.Item2)), + message: null, + onResponseDisposed: null); + } + + return new GitEndPointResponseData( + response.Item1, + new GitObjectsHttpException(response.Item1, "Test failure"), + shouldRetry: response.Item3, + message: null, + onResponseDisposed: null); + } + } + + private class ThrowingReadStream : MemoryStream + { + public override int ReadByte() + { + throw new IOException("Response body read failed"); + } + } + + private class CancelingReadStream : MemoryStream + { + public override int Read(byte[] buffer, int offset, int count) + { + throw new OperationCanceledException("Response body read canceled"); + } + } + } +} diff --git a/GVFS/GVFS.UnitTests/Http/HttpRequestorTests.cs b/GVFS/GVFS.UnitTests/Http/HttpRequestorTests.cs index 33692dd03b..e81b81170d 100644 --- a/GVFS/GVFS.UnitTests/Http/HttpRequestorTests.cs +++ b/GVFS/GVFS.UnitTests/Http/HttpRequestorTests.cs @@ -1,7 +1,8 @@ -using System.Net; using GVFS.Common.Http; using GVFS.Tests.Should; using NUnit.Framework; +using System; +using System.Net; namespace GVFS.UnitTests.Http { @@ -75,5 +76,13 @@ public void CommonNonAuthStatusesDoNotRejectCredentials() HttpRequestor.ShouldRejectCredentials(HttpStatusCode.RequestTimeout, responseBody: null) .ShouldEqual(false, "A 408 must NOT reject credentials"); } + + [TestCase] + public void AuthorityForTelemetryExcludesCredentialsAndRequestPath() + { + Uri uri = new Uri("https://alice:secret@cache.example.com:8443/private/path?token=sensitive#fragment"); + + HttpRequestor.GetAuthorityForTelemetry(uri).ShouldEqual("cache.example.com:8443"); + } } } diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs index d933584e94..47ad66e295 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs @@ -17,6 +17,8 @@ public MockTracer() this.RelatedWarningEvents = new List(); this.RelatedErrorEvents = new List(); this.RelatedEventNames = new List(); + this.RelatedEventKeywords = new List(); + this.RelatedEventMetadata = new List(); } public MockTracer StartActivityTracer { get; private set; } @@ -29,6 +31,8 @@ public MockTracer() // Names of events reported via RelatedEvent (which, unlike RelatedInfo/Warning/Error, // do not otherwise get recorded). Lets tests assert a specific diagnostic event fired. public List RelatedEventNames { get; } + public List RelatedEventKeywords { get; } + public List RelatedEventMetadata { get; } public void WaitForRelatedEvent() { @@ -38,6 +42,8 @@ public void WaitForRelatedEvent() public void RelatedEvent(EventLevel error, string eventName, EventMetadata metadata) { this.RelatedEventNames.Add(eventName); + this.RelatedEventKeywords.Add(Keywords.None); + this.RelatedEventMetadata.Add(metadata); if (eventName == this.WaitRelatedEventName) { this.waitEvent.Set(); @@ -47,6 +53,8 @@ public void RelatedEvent(EventLevel error, string eventName, EventMetadata metad public void RelatedEvent(EventLevel error, string eventName, EventMetadata metadata, Keywords keyword) { this.RelatedEventNames.Add(eventName); + this.RelatedEventKeywords.Add(keyword); + this.RelatedEventMetadata.Add(metadata); if (eventName == this.WaitRelatedEventName) { this.waitEvent.Set(); diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 1277355ab6..2e9bb52762 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -24,6 +24,14 @@ public class CloneVerb : GVFSVerb public string CacheServerUrl { get; set; } + public string PrefetchCacheServerUrl { get; set; } + + public string GetCacheServerUrl { get; set; } + + public string PostCacheServerUrl { get; set; } + + public string SizesCacheServerUrl { get; set; } + public string Branch { get; set; } public bool SingleBranch { get; set; } @@ -56,6 +64,22 @@ public static System.CommandLine.Command CreateCommand() System.CommandLine.Option cacheServerOption = new System.CommandLine.Option("--cache-server-url") { Description = "The url or friendly name of the cache server" }; cmd.Add(cacheServerOption); + System.CommandLine.Option prefetchCacheServerOption = new System.CommandLine.Option("--prefetch-cache-server-url") { Description = "The cache server URL for the prefetch endpoint" }; + AddEndpointCacheServerUrlValidator(prefetchCacheServerOption); + cmd.Add(prefetchCacheServerOption); + + System.CommandLine.Option getCacheServerOption = new System.CommandLine.Option("--get-cache-server-url") { Description = "The cache server URL for the objects GET endpoint" }; + AddEndpointCacheServerUrlValidator(getCacheServerOption); + cmd.Add(getCacheServerOption); + + System.CommandLine.Option postCacheServerOption = new System.CommandLine.Option("--post-cache-server-url") { Description = "The cache server URL for the objects POST endpoint" }; + AddEndpointCacheServerUrlValidator(postCacheServerOption); + cmd.Add(postCacheServerOption); + + System.CommandLine.Option sizesCacheServerOption = new System.CommandLine.Option("--sizes-cache-server-url") { Description = "The cache server URL for the sizes endpoint" }; + AddEndpointCacheServerUrlValidator(sizesCacheServerOption); + cmd.Add(sizesCacheServerOption); + System.CommandLine.Option branchOption = new System.CommandLine.Option("--branch", new[] { "-b" }) { Description = "Branch to checkout after clone" }; cmd.Add(branchOption); @@ -86,6 +110,10 @@ public static System.CommandLine.Command CreateCommand() } verb.CacheServerUrl = result.GetValue(cacheServerOption); + verb.PrefetchCacheServerUrl = result.GetValue(prefetchCacheServerOption); + verb.GetCacheServerUrl = result.GetValue(getCacheServerOption); + verb.PostCacheServerUrl = result.GetValue(postCacheServerOption); + verb.SizesCacheServerUrl = result.GetValue(sizesCacheServerOption); verb.Branch = result.GetValue(branchOption); verb.SingleBranch = result.GetValue(singleBranchOption); verb.NoMount = result.GetValue(noMountOption); @@ -107,6 +135,19 @@ public static System.CommandLine.Command CreateCommand() return cmd; } + private static void AddEndpointCacheServerUrlValidator(System.CommandLine.Option option) + { + option.Validators.Add( + result => + { + string url = result.GetValueOrDefault(); + if (url != null && !CacheServerInfo.IsValidUrl(url)) + { + result.AddError($"Option '{option.Name}' requires an absolute URL."); + } + }); + } + protected override string VerbName { get { return CloneVerbName; } @@ -144,6 +185,14 @@ public override void Execute() this.CheckKernelDriverSupported(normalizedEnlistmentRootPath); this.CheckNotInsideExistingRepo(normalizedEnlistmentRootPath); this.BlockEmptyCacheServerUrl(this.CacheServerUrl); + this.BlockEmptyCacheServerUrl(this.PrefetchCacheServerUrl); + this.BlockEmptyCacheServerUrl(this.GetCacheServerUrl); + this.BlockEmptyCacheServerUrl(this.PostCacheServerUrl); + this.BlockEmptyCacheServerUrl(this.SizesCacheServerUrl); + this.BlockInvalidEndpointCacheServerUrl("--prefetch-cache-server-url", this.PrefetchCacheServerUrl); + this.BlockInvalidEndpointCacheServerUrl("--get-cache-server-url", this.GetCacheServerUrl); + this.BlockInvalidEndpointCacheServerUrl("--post-cache-server-url", this.PostCacheServerUrl); + this.BlockInvalidEndpointCacheServerUrl("--sizes-cache-server-url", this.SizesCacheServerUrl); try { @@ -231,6 +280,11 @@ public override void Execute() } cacheServer = this.ResolveCacheServer(tracer, cacheServer, cacheServerResolver, serverGVFSConfig); + cacheServer = cacheServer.WithEndpointOverrides( + this.PrefetchCacheServerUrl, + this.GetCacheServerUrl, + this.PostCacheServerUrl, + this.SizesCacheServerUrl); this.ValidateClientVersions(tracer, enlistment, serverGVFSConfig, showWarnings: true); @@ -596,6 +650,14 @@ private bool TryDetermineLocalCacheAndInitializePaths( return true; } + private void BlockInvalidEndpointCacheServerUrl(string optionName, string url) + { + if (url != null && !CacheServerInfo.IsValidUrl(url)) + { + this.ReportErrorAndExit($"Option '{optionName}' requires an absolute URL."); + } + } + private Result CreateClone( ITracer tracer, GVFSEnlistment enlistment, @@ -643,6 +705,11 @@ private Result CreateClone( return new Result("Unable to configure cache server: " + errorMessage); } + if (!cacheServerResolver.TrySaveEndpointUrlsToLocalConfig(objectRequestor.CacheServer, out errorMessage)) + { + return new Result("Unable to configure endpoint-specific cache servers: " + errorMessage); + } + GitProcess git = new GitProcess(enlistment); string originBranchName = "origin/" + branch; GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName); diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 51b693578d..84a4c678bf 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -475,6 +475,7 @@ protected CacheServerInfo ResolveCacheServer( resolvedCacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServer.Url, serverGVFSConfig); } + resolvedCacheServer = resolvedCacheServer.WithEndpointOverridesFrom(cacheServer); this.Output.WriteLine("Using cache server: " + resolvedCacheServer); return resolvedCacheServer; } @@ -526,7 +527,7 @@ protected bool TryDownloadCommit( if (!gitObjects.TryDownloadCommit(commitId)) { - error = "Could not download commit " + commitId + " from: " + Uri.EscapeDataString(objectRequestor.CacheServer.ObjectsEndpointUrl); + error = "Could not download commit " + commitId + " from: " + Uri.EscapeDataString(objectRequestor.CacheServer.ObjectsPostEndpointUrl); return false; } diff --git a/GVFS/GVFS/CommandLine/PrefetchVerb.cs b/GVFS/GVFS/CommandLine/PrefetchVerb.cs index 6fa0d91f42..f82f4c0898 100644 --- a/GVFS/GVFS/CommandLine/PrefetchVerb.cs +++ b/GVFS/GVFS/CommandLine/PrefetchVerb.cs @@ -340,7 +340,9 @@ private void InitializeServerConnection( CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); - resolvedCacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerFromConfig.Url, serverGVFSConfig); + resolvedCacheServer = cacheServerResolver + .ResolveNameFromRemote(cacheServerFromConfig.Url, serverGVFSConfig) + .WithEndpointOverridesFrom(cacheServerFromConfig); if (!this.SkipVersionCheck) { diff --git a/docs/getting-started.md b/docs/getting-started.md index aee8b93844..75899bb0ae 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -38,6 +38,20 @@ These options allow a user to customize their initial enlistment. cache servers via the `/gvfs/config` endpoint, then the `clone` command will select a nearby cache server from that list. +* `--prefetch-cache-server-url=`, + `--get-cache-server-url=`, `--post-cache-server-url=`, and + `--sizes-cache-server-url=`: Prefer the specified absolute cache server + URL for `/gvfs/prefetch`, loose-object GET requests, batched-object POST + requests, or `/gvfs/sizes`, respectively. If a dedicated server fails, VFS + for Git retries the request against the server selected by + `--cache-server-url`. Sizes requests retain their additional fallback to the + origin server when the global cache does not support `/gvfs/sizes`. + + These values are saved in the local Git configuration as + `gvfs.prefetch.cache-server`, `gvfs.get.cache-server`, + `gvfs.post.cache-server`, and `gvfs.sizes.cache-server`. They continue to + apply to later mount, hydration, and prefetch operations. + * `--branch=`: Specify the branch to checkout after clone. * `--local-cache-path=`: Use this option to override the path for the diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 44fa175482..8b91ed346a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -237,6 +237,26 @@ Run `gvfs cache-server --list` to see the available cache server URLs. Run `gvfs cache-server --set=` to set your cache server to ``. +Individual GVFS protocol endpoints can prefer dedicated cache servers through +these local Git configuration values: + +| Configuration | Requests | +| --- | --- | +| `gvfs.prefetch.cache-server` | `/gvfs/prefetch` | +| `gvfs.get.cache-server` | Loose-object GET requests under `/gvfs/objects` | +| `gvfs.post.cache-server` | Batched-object POST requests to `/gvfs/objects` | +| `gvfs.sizes.cache-server` | `/gvfs/sizes` | + +Each value must be an absolute URL. A dedicated endpoint server is attempted +before `gvfs.cache-server`; if that request fails, VFS for Git falls back to +the global cache server. Sizes requests also fall back from the global cache +to the origin server when `/gvfs/sizes` is not supported. + +`gvfs cache-server --get` and `--set` operate on the global +`gvfs.cache-server` value. Setting the global server does not clear the four +endpoint-specific values. Inspect or change those values with `git config +--local []`. + ### System-wide Config The `gvfs config` command allows customizing some behavior.