Transient Fault Handling

USE_CASES.md

@@ -8,6 +8,7 @@ This documentation provides examples for specific use cases. Please [open an iss
 * [Email - Send a Single Email to a Single Recipient](#singleemailsinglerecipient)
 * [Email - Send Multiple Emails to Multiple Recipients](#multipleemailsmultiplerecipients)
 * [Email - Transactional Templates](#transactional_templates)
+* [Transient Fault Handling](#transient_faults)
 
 <a name="attachments"></a>
 # Attachments
@@ -582,3 +583,61 @@ namespace Example
     }
 }
 ```
+
+<a name="transient_faults"></a>
+# Transient Fault Handling
+
+The SendGridClient provides functionality for handling transient errors that might occur when sending an HttpRequest. This includes client side timeouts while sending the mail, or certain errors returned within the 500 range. Errors within the 500 range are limited to 500 Internal Server Error, 502 Bad Gateway, 503 Service unavailable and 504 Gateway timeout.
+
+By default, retry behaviour is off, you must explicitly enable it by setting the retry count to a value greater than zero. To set the retry count, you must use the SendGridClient construct that takes a **SendGridClientOptions** object, allowing you to configure the **ReliabilitySettings**
+
+### RetryCount
+
+The amount of times to retry the operation before reporting an exception to the caller. This is in addition to the initial attempt so setting a value of 1 would result in 2 attempts, the initial attempt and the retry. Defaults to zero, retry behaviour is not enabled. The maximum amount of retries permitted is 5. 
+
+### MinimumBackOff
+
+The minimum amount of time to wait between retries. 
+
+### MaximumBackOff
+
+The maximum possible amount of time to wait between retries. The maximum value allowed is 30 seconds
+
+### DeltaBackOff
+
+The value that will be used to calculate a random delta in the exponential delay between retries.  A random element of time is factored into the delta calculation as this helps avoid many clients retrying at regular intervals.
+
+
+## Examples
+
+In this example we are setting RetryCount to 2, with a mimimum wait time of 1 seconds, a maximum of 10 seconds and a delta of 3 seconds
+
+```csharp
+
+var options = new SendGridClientOptions
+{
+    ApiKey = "Your-Api-Key",
+    ReliabilitySettings = new ReliabilitySettings(2, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(3))
+};
+
+var client = new SendGridClient(options);
+
+```
+
+The SendGridClientOptions object defines all the settings that can be set for the client, e.g.
+
+```csharp
+
+var options = new SendGridClientOptions
+{
+    ApiKey = "Your-Api-Key",
+    ReliabilitySettings = new ReliabilitySettings(2, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(3)),
+    Host = "Your-Host",
+    UrlPath = "Url-Path",
+    Version = "3",
+    RequestHeaders = new Dictionary<string, string>() {{"header-key", "header-value"}}
+};
+
+var client = new SendGridClient(options);
+
+```

src/SendGrid/Reliability/ReliabilitySettings.cs

@@ -0,0 +1,88 @@
+namespace SendGrid.Helpers.Reliability
+{
+    using System;
+
+    /// <summary>
+    /// Defines the reliability settings to use on HTTP requests
+    /// </summary>
+    public class ReliabilitySettings
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ReliabilitySettings"/> class with default settings.
+        /// </summary>
+        public ReliabilitySettings()
+            : this(0, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ReliabilitySettings"/> class.
+        /// </summary>
+        /// <param name="maximumNumberOfRetries">The maximum number of retries to execute against when sending an HTTP Request before throwing an exception</param>
+        /// <param name="minimumBackoff">The minimum amount of time to wait between between HTTP retries</param>
+        /// <param name="maximumBackOff">the maximum amount of time to wait between between HTTP retries</param>
+        /// <param name="deltaBackOff">the value that will be used to calculate a random delta in the exponential delay between retries</param>
+        public ReliabilitySettings(int maximumNumberOfRetries, TimeSpan minimumBackoff, TimeSpan maximumBackOff, TimeSpan deltaBackOff)
+        {
+            if (maximumNumberOfRetries < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(maximumNumberOfRetries), "maximumNumberOfRetries must be greater than 0");
+            }
+
+            if (maximumNumberOfRetries > 5)
+            {
+                throw new ArgumentOutOfRangeException(nameof(maximumNumberOfRetries), "The maximum number of retries allowed is 5");
+            }
+
+            if (minimumBackoff.Ticks < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(minimumBackoff), "minimumBackoff must be greater than 0");
+            }
+
+            if (maximumBackOff.Ticks < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(maximumBackOff), "maximumBackOff must be greater than 0");
+            }
+
+            if (maximumBackOff.TotalSeconds > 30)
+            {
+                throw new ArgumentOutOfRangeException(nameof(maximumBackOff), "maximumBackOff must be less than 30 seconds");
+            }
+
+            if (deltaBackOff.Ticks < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(deltaBackOff), "deltaBackOff must be greater than 0");
+            }
+
+            if (minimumBackoff.TotalMilliseconds > maximumBackOff.TotalMilliseconds)
+            {
+                throw new ArgumentOutOfRangeException(nameof(minimumBackoff), "minimumBackoff must be less than maximumBackOff");
+            }
+
+            this.MaximumNumberOfRetries = maximumNumberOfRetries;
+            this.MinimumBackOff = minimumBackoff;
+            this.DeltaBackOff = deltaBackOff;
+            this.MaximumBackOff = maximumBackOff;
+        }
+
+        /// <summary>
+        ///     Gets the maximum number of retries to execute against when sending an HTTP Request before throwing an exception. Defaults to 0 (no retries, you must explicitly enable)
+        /// </summary>
+        public int MaximumNumberOfRetries { get; }
+
+        /// <summary>
+        /// Gets the minimum amount of time to wait between between HTTP retries. Defaults to 1 second
+        /// </summary>
+        public TimeSpan MinimumBackOff { get; }
+
+        /// <summary>
+        /// Gets the maximum amount of time to wait between between HTTP retries. Defaults to 10 seconds
+        /// </summary>
+        public TimeSpan MaximumBackOff { get; }
+
+        /// <summary>
+        /// Gets the value that will be used to calculate a random delta in the exponential delay between retries. Defaults to 1 second
+        /// </summary>
+        public TimeSpan DeltaBackOff { get; }
+    }
+}

src/SendGrid/Reliability/RetryDelegatingHandler.cs

@@ -0,0 +1,120 @@
+namespace SendGrid.Helpers.Reliability
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Net;
+    using System.Net.Http;
+    using System.Threading;
+    using System.Threading.Tasks;
+
+    /// <summary>
+    /// A delegating handler that provides retry functionality while executing a request
+    /// </summary>
+    public class RetryDelegatingHandler : DelegatingHandler
+    {
+        private static readonly List<HttpStatusCode> RetriableServerErrorStatusCodes =
+            new List<HttpStatusCode>()
+            {
+                HttpStatusCode.InternalServerError,
+                HttpStatusCode.BadGateway,
+                HttpStatusCode.ServiceUnavailable,
+                HttpStatusCode.GatewayTimeout
+            };
+
+        private readonly ReliabilitySettings settings;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="RetryDelegatingHandler"/> class.
+        /// </summary>
+        /// <param name="settings">A ReliabilitySettings instance</param>
+        public RetryDelegatingHandler(ReliabilitySettings settings)
+            : this(new HttpClientHandler(), settings)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="RetryDelegatingHandler"/> class.
+        /// </summary>
+        /// <param name="innerHandler">A HttpMessageHandler instance to set as the innner handler</param>
+        /// <param name="settings">A ReliabilitySettings instance</param>
+        public RetryDelegatingHandler(HttpMessageHandler innerHandler, ReliabilitySettings settings)
+            : base(innerHandler)
+        {
+            this.settings = settings;
+        }
+
+        protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            if (this.settings.MaximumNumberOfRetries == 0)
+            {
+                return await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
+            }
+
+            HttpResponseMessage responseMessage = null;
+
+            var numberOfAttempts = 0;
+            var sent = false;
+
+            while (!sent)
+            {
+                var waitFor = this.GetNextWaitInterval(numberOfAttempts);
+
+                try
+                {
+                    responseMessage = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
+
+                    ThrowHttpRequestExceptionIfResponseCodeCanBeRetried(responseMessage);
+
+                    sent = true;
+                }
+                catch (TaskCanceledException)
+                {
+                    numberOfAttempts++;
+
+                    if (numberOfAttempts > this.settings.MaximumNumberOfRetries)
+                    {
+                        throw new TimeoutException();
+                    }
+
+                    // ReSharper disable once MethodSupportsCancellation, cancel will be indicated on the token
+                    await Task.Delay(waitFor).ConfigureAwait(false);
+                }
+                catch (HttpRequestException)
+                {
+                    numberOfAttempts++;
+
+                    if (numberOfAttempts > this.settings.MaximumNumberOfRetries)
+                    {
+                        throw;
+                    }
+
+                    await Task.Delay(waitFor).ConfigureAwait(false);
+                }
+            }
+
+            return responseMessage;
+        }
+
+        private static void ThrowHttpRequestExceptionIfResponseCodeCanBeRetried(HttpResponseMessage responseMessage)
+        {
+            if (RetriableServerErrorStatusCodes.Contains(responseMessage.StatusCode))
+            {
+                throw new HttpRequestException(string.Format("Http status code '{0}' indicates server error", responseMessage.StatusCode));
+            }
+        }
+
+        private TimeSpan GetNextWaitInterval(int numberOfAttempts)
+        {
+            var random = new Random();
+
+            var delta = (int)((Math.Pow(2.0, numberOfAttempts) - 1.0) *
+                               random.Next(
+                                   (int)(this.settings.DeltaBackOff.TotalMilliseconds * 0.8),
+                                   (int)(this.settings.DeltaBackOff.TotalMilliseconds * 1.2)));
+
+            var interval = (int)Math.Min(this.settings.MinimumBackOff.TotalMilliseconds + delta, this.settings.MaximumBackOff.TotalMilliseconds);
+
+            return TimeSpan.FromMilliseconds(interval);
+        }
+    }
+}