Add an eviction policy to the GBPriceCache.

MarketData/MarketDataLib/Cache/GBPriceCache.cs

@@ -46,6 +46,7 @@ namespace MarketData.Cache
     private Thread cacheMonitorThread = null;
     private volatile bool threadRun = true;
     private Object thisLock = new Object();
+    private static volatile bool isShutdown = false;
 
     private CacheSnapshot snapshot;
     private DateGenerator dateGenerator = new DateGenerator();
@@ -61,10 +62,16 @@ namespace MarketData.Cache
       cacheMonitorThread.Start();
     }
 
+    /// <summary>
+    /// GetInstance 
+    /// </summary>
+    /// <returns></returns>
+    /// <exception cref="ObjectDisposedException"></exception>
     public static GBPriceCache GetInstance()
     {
       lock (typeof(GBPriceCache))
       {
+        if (isShutdown) throw new ObjectDisposedException(nameof(GBPriceCache), "Cache has been shut down.");
         if (null == priceCacheInstance)
         {
           priceCacheInstance = new GBPriceCache();
@@ -81,11 +88,17 @@ namespace MarketData.Cache
       }
     }
 
+    /// <summary>
+    /// Shuts down the cache and stops the monitor thread. This is a full application-level
+    /// shutdown. Callers should always access the cache via GetInstance() and not hold
+    /// long-lived references to the instance.
+    /// </summary>
     public void Dispose()
     {
       lock (thisLock)
       {
         if (null == priceCacheInstance || !threadRun) return;
+        isShutdown = true;
         threadRun = false;
         if (null != cacheMonitorThread)
         {
@@ -97,33 +110,6 @@ namespace MarketData.Cache
       }
     }
 
-    public void ClearCacheOnOrBefore(DateTime onOrBeforeDate, bool collect = false)
-    {
-      lock (thisLock)
-      {
-        Dictionary<String, PricesByDate> newPriceCache = new Dictionary<String, PricesByDate>();
-        foreach (KeyValuePair<String, PricesByDate> entry in snapshot.PriceCache)
-        {
-          String symbol = entry.Key;
-          PricesByDate filteredPrices = new PricesByDate();
-          PricesByDate existingPrices = entry.Value;
-          foreach (KeyValuePair<DateTime, Price> kvp in existingPrices)
-          {
-            if (kvp.Key >= onOrBeforeDate)
-            {
-              filteredPrices.Add(kvp.Key, kvp.Value);
-            }
-          }
-          if (filteredPrices.Count > 0)
-          {
-            newPriceCache.Add(symbol, filteredPrices);
-          }
-        }
-        UpdateSnapshot(newPriceCache, snapshot.RealTimePriceCache, snapshot.NullCache);
-        if (collect) GC.Collect();
-      }
-    }
-
     public Price GetPriceOrLatestAvailable(String symbol, DateTime date)
     {
       Price price = GetPrice(symbol, date);
@@ -294,6 +280,38 @@ namespace MarketData.Cache
       return pricesByDate.ContainsKey(date);
     }
 
+    public void ClearCacheOnOrBefore(DateTime onOrBeforeDate, bool collect = false)
+    {
+      lock (thisLock)
+      {
+        UpdateSnapshot(BuildEvictedPriceCache(onOrBeforeDate), snapshot.RealTimePriceCache, new Dictionary<String, bool>());                     
+        if (collect) GC.Collect();
+      }
+    }
+
+    private Dictionary<String, PricesByDate> BuildEvictedPriceCache(DateTime onOrBeforeDate)
+    {
+      Dictionary<String, PricesByDate> newPriceCache = new Dictionary<String, PricesByDate>();
+      foreach (KeyValuePair<String, PricesByDate> entry in snapshot.PriceCache)
+      {
+        String symbol = entry.Key;
+        PricesByDate filteredPrices = new PricesByDate();
+        PricesByDate existingPrices = entry.Value;
+        foreach (KeyValuePair<DateTime, Price> kvp in existingPrices)
+        {
+          if (kvp.Key >= onOrBeforeDate)
+          {
+            filteredPrices.Add(kvp.Key, kvp.Value);
+          }
+        }
+        if (filteredPrices.Count > 0)
+        {
+          newPriceCache.Add(symbol, filteredPrices);
+        }
+      }
+      return newPriceCache;
+    }      
+
     private void ThreadProc()
     {
       int quantums = 0;
@@ -302,18 +320,25 @@ namespace MarketData.Cache
       while (threadRun)
       {
         Thread.Sleep(quantumInterval);
-        if(!threadRun)break;
+        if (!threadRun) break;
         quantums += quantumInterval;
         if (quantums > cacheRefreshAfter)
         {
           quantums = 0;
           lock (thisLock)
           {
-            UpdateSnapshot(snapshot.PriceCache, new Dictionary<String, Price>(), snapshot.NullCache);
+            DateTime maxDate = snapshot.PriceCache.Values.SelectMany(p => p.Keys).DefaultIfEmpty(DateTime.MinValue).Max();
+            if (maxDate != DateTime.MinValue)
+            {
+              DateTime evictBefore = dateGenerator.GenerateHistoricalDates(maxDate, 252).Min();
+              MDTrace.WriteLine(LogLevel.DEBUG, $"GBPriceCache, clearing cache on or before {evictBefore.ToShortDateString()}");
+              UpdateSnapshot(BuildEvictedPriceCache(evictBefore), new Dictionary<String, Price>(), new Dictionary<String, bool>());
+              GC.Collect();
+            }
           }
         }
       }
-    }
+    }     
 
     private void UpdateSnapshot(Dictionary<String, PricesByDate> newPriceCache,Dictionary<String, Price> newRealtimePriceCache, Dictionary<String, bool> newNullCache)
     {

MarketData/MarketDataLib/Integration/HttpNetRequest.cs

@@ -1,3 +1,4 @@
+#pragma warning disable SYSLIB0014
 using System.Net;
 using System.Text;
 using System.IO.Compression;
@@ -1843,18 +1844,27 @@ namespace MarketData.Integration
       return webRequest;
     }
 
+    /// <summary>
+    /// IsMovedException - MovedPermanently, Found, RedirectKeepVerb, PermanentRedirect will all return a "Location" in the response
+    /// That we will forward the request to.
+    /// </summary>
+    /// <param name="webException"></param>
+    /// <returns></returns>
     private static bool IsMovedException(WebException webException)
     {
-      if(webException.Status.Equals(WebExceptionStatus.ProtocolError))
+      if (webException.Status.Equals(WebExceptionStatus.ProtocolError))
       {
         HttpWebResponse response = webException.Response as HttpWebResponse;
-        if(response.StatusCode.Equals(HttpStatusCode.MovedPermanently))
+        if (response.StatusCode.Equals(HttpStatusCode.MovedPermanently) ||
+            response.StatusCode.Equals(HttpStatusCode.Found) ||
+            response.StatusCode.Equals(HttpStatusCode.RedirectKeepVerb) ||
+            response.StatusCode.Equals(HttpStatusCode.PermanentRedirect))
         {
-          return true;
+            return true;
         }
-      } 
+      }
       return false;
-    }
+    }    
 
     /// <summary>
     /// Get the proxy for the specified service or null if it is not congfigured to use the proxy
@@ -1921,3 +1931,4 @@ namespace MarketData.Integration
     }
   }  
 }
+#pragma warning restore SYSLIB0014