Update GBPriceCache.

MarketDataLib/Cache/GBPriceCache.cs

@@ -5,7 +5,6 @@ using System.Threading;
 using MarketData.MarketDataModel;
 using MarketData.Utils;
 using MarketData.Helper;
-using MarketData.Numerical;
 using MarketData.DataAccess;
 
 namespace MarketData.Cache
@@ -43,6 +42,7 @@ namespace MarketData.Cache
 
   public class GBPriceCache : IDisposable
   {
+    private static readonly int EVICTION_DAYCOUNT=252;  // upon eviction trigger remove all data older than maxdate - evictionPolicyThreshholdDays
     private Thread cacheMonitorThread = null;
     private volatile bool threadRun = true;
     private Object thisLock = new Object();
@@ -61,6 +61,11 @@ namespace MarketData.Cache
       cacheMonitorThread.Start();
     }
 
+    /// <summary>
+    /// GetInstance 
+    /// </summary>
+    /// <returns></returns>
+    /// <exception cref="ObjectDisposedException"></exception>
     public static GBPriceCache GetInstance()
     {
       lock (typeof(GBPriceCache))
@@ -81,6 +86,11 @@ namespace MarketData.Cache
       }
     }
 
+    /// <summary>
+    /// Shuts down the cache and stops the monitor thread. This is a full application-level
+    /// shutdown. Callers should always access the cache via GetInstance() and not hold
+    /// long-lived references to the instance.
+    /// </summary>
     public void Dispose()
     {
       lock (thisLock)
@@ -97,40 +107,15 @@ namespace MarketData.Cache
       }
     }
 
-    public void ClearCacheOnOrBefore(DateTime onOrBeforeDate, bool collect = false)
-    {
-      lock (thisLock)
-      {
-        Dictionary<String, PricesByDate> newPriceCache = new Dictionary<String, PricesByDate>();
-        foreach (KeyValuePair<String, PricesByDate> entry in snapshot.PriceCache)
-        {
-          String symbol = entry.Key;
-          PricesByDate filteredPrices = new PricesByDate();
-          PricesByDate existingPrices = entry.Value;
-          foreach (KeyValuePair<DateTime, Price> kvp in existingPrices)
-          {
-            if (kvp.Key >= onOrBeforeDate)
-            {
-              filteredPrices.Add(kvp.Key, kvp.Value);
-            }
-          }
-          if (filteredPrices.Count > 0)
-          {
-            newPriceCache.Add(symbol, filteredPrices);
-          }
-        }
-        UpdateSnapshot(newPriceCache, snapshot.RealTimePriceCache, snapshot.NullCache);
-        if (collect) GC.Collect();
-      }
-    }
-
     public Price GetPriceOrLatestAvailable(String symbol, DateTime date)
     {
       Price price = GetPrice(symbol, date);
       if (null != price) return price;
+
       DateTime latestPricingDate = PricingDataAccess.GetLatestDateOnOrBefore(symbol, date);
       price = GetPrice(symbol, latestPricingDate);
       if (null != price) return price;
+
       fetchSemaphore.Wait();
       try
       {
@@ -140,7 +125,8 @@ namespace MarketData.Cache
       {
         fetchSemaphore.Release();
       }
-      if (null !=price) AddPrice(price);
+
+      if (null != price) AddPrice(price);
       return price;
     }
 
@@ -152,27 +138,33 @@ namespace MarketData.Cache
       }
 
       Price price = MarketDataHelper.GetLatestPrice(symbol);
+
       if (null != price)
       {
         Dictionary<String, Price> newRealtime = new Dictionary<String, Price>(snapshot.RealTimePriceCache);
         newRealtime.Add(symbol, price);
         UpdateSnapshot(snapshot.PriceCache, newRealtime, snapshot.NullCache);
       }
+
       return price;
     }
 
     public Price GetPrice(String symbol, DateTime date)
     {
       date = date.Date;
+
       if (!ContainsPrice(symbol, date))
       {
         String key = symbol + Utility.DateTimeToStringMMHDDHYYYY(date);
+
         if (snapshot.NullCache.ContainsKey(key))
         {
           return null;
         }
+
         fetchSemaphore.Wait();
         Price price;
+
         try
         {
           price = PricingDataAccess.GetPrice(symbol, date);
@@ -181,7 +173,8 @@ namespace MarketData.Cache
         {
           fetchSemaphore.Release();
         }
-        if (null ==price)
+
+        if (null == price)
         {
           Dictionary<String, bool> newNullCache = new Dictionary<String, bool>(snapshot.NullCache);
           newNullCache.Add(key, true);
@@ -191,23 +184,29 @@ namespace MarketData.Cache
 
         AddPrice(price);
       }
+
       if (!snapshot.PriceCache.ContainsKey(symbol)) return null;
+
       PricesByDate pricesByDate = snapshot.PriceCache[symbol];
       if (!pricesByDate.ContainsKey(date)) return null;
+
       return pricesByDate[date];
     }
 
     public Prices GetPrices(String symbol, DateTime earlierDate, DateTime laterDate)
     {
       DateGenerator localDateGenerator = new DateGenerator();
+
       if (laterDate < earlierDate)
       {
         DateTime tempDate = earlierDate;
         earlierDate = laterDate;
         laterDate = tempDate;
       }
+
       List<DateTime> datesList = localDateGenerator.GenerateHistoricalDates(earlierDate, laterDate);
       datesList = datesList.Where(x => x >= earlierDate).ToList();
+
       return GetPrices(symbol, laterDate, datesList.Count);
     }
 
@@ -215,17 +214,20 @@ namespace MarketData.Cache
     {
       List<DateTime> historicalDates = dateGenerator.GenerateHistoricalDates(startDate, dayCount + 60);
       List<DateTime> missingDates = new List<DateTime>();
+
       foreach (DateTime historicalDate in historicalDates)
       {
         if (!ContainsPrice(symbol, historicalDate))
         {
           String key = symbol + Utility.DateTimeToStringMMHDDHYYYY(historicalDate);
+
           if (!snapshot.NullCache.ContainsKey(key))
           {
             missingDates.Add(historicalDate);
           }
         }
       }
+
       if (missingDates.Count > 0)
       {
         DateTime minDate = missingDates.Min();
@@ -233,6 +235,7 @@ namespace MarketData.Cache
 
         fetchSemaphore.Wait();
         Prices loadedPrices;
+
         try
         {
           loadedPrices = PricingDataAccess.GetPrices(symbol, maxDate, minDate);
@@ -249,11 +252,14 @@ namespace MarketData.Cache
       }
 
       Prices prices = new Prices();
+
       foreach (DateTime historicalDate in historicalDates)
       {
         if (!snapshot.PriceCache.ContainsKey(symbol)) continue;
+
         PricesByDate pricesByDate = snapshot.PriceCache[symbol];
         if (!pricesByDate.ContainsKey(historicalDate)) continue;
+
         prices.Add(pricesByDate[historicalDate]);
       }
 
@@ -268,17 +274,21 @@ namespace MarketData.Cache
       lock (thisLock)
       {
         PricesByDate pricesByDate;
+
         if (!snapshot.PriceCache.ContainsKey(price.Symbol))
         {
           pricesByDate = new PricesByDate();
           pricesByDate.Add(price.Date, price);
+
           Dictionary<String, PricesByDate> newCache = new Dictionary<String, PricesByDate>(snapshot.PriceCache);
           newCache.Add(price.Symbol, pricesByDate);
+
           UpdateSnapshot(newCache, snapshot.RealTimePriceCache, snapshot.NullCache);
         }
         else
         {
           pricesByDate = snapshot.PriceCache[price.Symbol];
+
           if (!pricesByDate.ContainsKey(price.Date))
           {
             pricesByDate.Add(price.Date, price);
@@ -290,10 +300,47 @@ namespace MarketData.Cache
     public bool ContainsPrice(String symbol, DateTime date)
     {
       if (!snapshot.PriceCache.ContainsKey(symbol)) return false;
+
       PricesByDate pricesByDate = snapshot.PriceCache[symbol];
       return pricesByDate.ContainsKey(date);
     }
 
+    public void ClearCacheOnOrBefore(DateTime onOrBeforeDate, bool collect = false)
+    {
+      lock (thisLock)
+      {
+        UpdateSnapshot(BuildEvictedPriceCache(onOrBeforeDate), snapshot.RealTimePriceCache, new Dictionary<String, bool>());
+        if (collect) GC.Collect();
+      }
+    }
+
+    private Dictionary<String, PricesByDate> BuildEvictedPriceCache(DateTime onOrBeforeDate)
+    {
+      Dictionary<String, PricesByDate> newPriceCache = new Dictionary<String, PricesByDate>();
+
+      foreach (KeyValuePair<String, PricesByDate> entry in snapshot.PriceCache)
+      {
+        String symbol = entry.Key;
+        PricesByDate filteredPrices = new PricesByDate();
+        PricesByDate existingPrices = entry.Value;
+
+        foreach (KeyValuePair<DateTime, Price> kvp in existingPrices)
+        {
+          if (kvp.Key >= onOrBeforeDate)
+          {
+            filteredPrices.Add(kvp.Key, kvp.Value);
+          }
+        }
+
+        if (filteredPrices.Count > 0)
+        {
+          newPriceCache.Add(symbol, filteredPrices);
+        }
+      }
+
+      return newPriceCache;
+    }
+
     private void ThreadProc()
     {
       int quantums = 0;
@@ -302,22 +349,37 @@ namespace MarketData.Cache
       while (threadRun)
       {
         Thread.Sleep(quantumInterval);
-        if(!threadRun)break;
+        if (!threadRun) break;
+
         quantums += quantumInterval;
+
         if (quantums > cacheRefreshAfter)
         {
           quantums = 0;
+
           lock (thisLock)
           {
-            UpdateSnapshot(snapshot.PriceCache, new Dictionary<String, Price>(), snapshot.NullCache);
+            DateTime maxDate = snapshot.PriceCache.Values.SelectMany(p => p.Keys).DefaultIfEmpty(DateTime.MinValue).Max();
+
+            if (maxDate != DateTime.MinValue)
+            {
+              DateTime evictBefore = dateGenerator.GenerateHistoricalDates(maxDate, EVICTION_DAYCOUNT).Min();
+              int beforeCount = snapshot.PriceCache.Values.Sum(p => p.Count);
+              Dictionary<String, PricesByDate> newCache = BuildEvictedPriceCache(evictBefore);
+              int afterCount = newCache.Values.Sum(p => p.Count);
+              int removed = beforeCount - afterCount;
+              MDTrace.WriteLine(LogLevel.DEBUG, $"GBPriceCache eviction: removed {removed} prices (before={beforeCount}, after={afterCount}) on or before {evictBefore.ToShortDateString()}");
+              UpdateSnapshot(newCache, new Dictionary<String, Price>(), new Dictionary<String, bool>());
+              GC.Collect();
+            }
           }
         }
       }
     }
 
-    private void UpdateSnapshot(Dictionary<String, PricesByDate> newPriceCache,Dictionary<String, Price> newRealtimePriceCache, Dictionary<String, bool> newNullCache)
+    private void UpdateSnapshot(Dictionary<String, PricesByDate> newPriceCache, Dictionary<String, Price> newRealtimePriceCache, Dictionary<String, bool> newNullCache)
     {
       snapshot = new CacheSnapshot(newPriceCache, newRealtimePriceCache, newNullCache);
     }
-  }
 }
+}