diff --git a/MarketDataLib/Cache/GBPriceCache.cs b/MarketDataLib/Cache/GBPriceCache.cs index 1e8a641..cfe748e 100644 --- a/MarketDataLib/Cache/GBPriceCache.cs +++ b/MarketDataLib/Cache/GBPriceCache.cs @@ -5,7 +5,6 @@ using System.Threading; using MarketData.MarketDataModel; using MarketData.Utils; using MarketData.Helper; -using MarketData.Numerical; using MarketData.DataAccess; namespace MarketData.Cache @@ -43,6 +42,7 @@ namespace MarketData.Cache public class GBPriceCache : IDisposable { + private static readonly int EVICTION_DAYCOUNT=252; // upon eviction trigger remove all data older than maxdate - evictionPolicyThreshholdDays private Thread cacheMonitorThread = null; private volatile bool threadRun = true; private Object thisLock = new Object(); @@ -61,6 +61,11 @@ namespace MarketData.Cache cacheMonitorThread.Start(); } + /// + /// GetInstance + /// + /// + /// public static GBPriceCache GetInstance() { lock (typeof(GBPriceCache)) @@ -81,6 +86,11 @@ namespace MarketData.Cache } } + /// + /// Shuts down the cache and stops the monitor thread. This is a full application-level + /// shutdown. Callers should always access the cache via GetInstance() and not hold + /// long-lived references to the instance. + /// public void Dispose() { lock (thisLock) @@ -97,40 +107,15 @@ namespace MarketData.Cache } } - public void ClearCacheOnOrBefore(DateTime onOrBeforeDate, bool collect = false) - { - lock (thisLock) - { - Dictionary newPriceCache = new Dictionary(); - foreach (KeyValuePair entry in snapshot.PriceCache) - { - String symbol = entry.Key; - PricesByDate filteredPrices = new PricesByDate(); - PricesByDate existingPrices = entry.Value; - foreach (KeyValuePair kvp in existingPrices) - { - if (kvp.Key >= onOrBeforeDate) - { - filteredPrices.Add(kvp.Key, kvp.Value); - } - } - if (filteredPrices.Count > 0) - { - newPriceCache.Add(symbol, filteredPrices); - } - } - UpdateSnapshot(newPriceCache, snapshot.RealTimePriceCache, snapshot.NullCache); - if (collect) GC.Collect(); - } - } - public Price GetPriceOrLatestAvailable(String symbol, DateTime date) { Price price = GetPrice(symbol, date); if (null != price) return price; + DateTime latestPricingDate = PricingDataAccess.GetLatestDateOnOrBefore(symbol, date); price = GetPrice(symbol, latestPricingDate); if (null != price) return price; + fetchSemaphore.Wait(); try { @@ -140,7 +125,8 @@ namespace MarketData.Cache { fetchSemaphore.Release(); } - if (null !=price) AddPrice(price); + + if (null != price) AddPrice(price); return price; } @@ -152,27 +138,33 @@ namespace MarketData.Cache } Price price = MarketDataHelper.GetLatestPrice(symbol); + if (null != price) { Dictionary newRealtime = new Dictionary(snapshot.RealTimePriceCache); newRealtime.Add(symbol, price); UpdateSnapshot(snapshot.PriceCache, newRealtime, snapshot.NullCache); } + return price; } public Price GetPrice(String symbol, DateTime date) { date = date.Date; + if (!ContainsPrice(symbol, date)) { String key = symbol + Utility.DateTimeToStringMMHDDHYYYY(date); + if (snapshot.NullCache.ContainsKey(key)) { return null; } + fetchSemaphore.Wait(); Price price; + try { price = PricingDataAccess.GetPrice(symbol, date); @@ -181,7 +173,8 @@ namespace MarketData.Cache { fetchSemaphore.Release(); } - if (null ==price) + + if (null == price) { Dictionary newNullCache = new Dictionary(snapshot.NullCache); newNullCache.Add(key, true); @@ -191,23 +184,29 @@ namespace MarketData.Cache AddPrice(price); } + if (!snapshot.PriceCache.ContainsKey(symbol)) return null; + PricesByDate pricesByDate = snapshot.PriceCache[symbol]; if (!pricesByDate.ContainsKey(date)) return null; + return pricesByDate[date]; } public Prices GetPrices(String symbol, DateTime earlierDate, DateTime laterDate) { DateGenerator localDateGenerator = new DateGenerator(); + if (laterDate < earlierDate) { DateTime tempDate = earlierDate; earlierDate = laterDate; laterDate = tempDate; } + List datesList = localDateGenerator.GenerateHistoricalDates(earlierDate, laterDate); datesList = datesList.Where(x => x >= earlierDate).ToList(); + return GetPrices(symbol, laterDate, datesList.Count); } @@ -215,17 +214,20 @@ namespace MarketData.Cache { List historicalDates = dateGenerator.GenerateHistoricalDates(startDate, dayCount + 60); List missingDates = new List(); + foreach (DateTime historicalDate in historicalDates) { if (!ContainsPrice(symbol, historicalDate)) { String key = symbol + Utility.DateTimeToStringMMHDDHYYYY(historicalDate); + if (!snapshot.NullCache.ContainsKey(key)) { missingDates.Add(historicalDate); } } } + if (missingDates.Count > 0) { DateTime minDate = missingDates.Min(); @@ -233,6 +235,7 @@ namespace MarketData.Cache fetchSemaphore.Wait(); Prices loadedPrices; + try { loadedPrices = PricingDataAccess.GetPrices(symbol, maxDate, minDate); @@ -249,11 +252,14 @@ namespace MarketData.Cache } Prices prices = new Prices(); + foreach (DateTime historicalDate in historicalDates) { if (!snapshot.PriceCache.ContainsKey(symbol)) continue; + PricesByDate pricesByDate = snapshot.PriceCache[symbol]; if (!pricesByDate.ContainsKey(historicalDate)) continue; + prices.Add(pricesByDate[historicalDate]); } @@ -268,17 +274,21 @@ namespace MarketData.Cache lock (thisLock) { PricesByDate pricesByDate; + if (!snapshot.PriceCache.ContainsKey(price.Symbol)) { pricesByDate = new PricesByDate(); pricesByDate.Add(price.Date, price); + Dictionary newCache = new Dictionary(snapshot.PriceCache); newCache.Add(price.Symbol, pricesByDate); + UpdateSnapshot(newCache, snapshot.RealTimePriceCache, snapshot.NullCache); } else { pricesByDate = snapshot.PriceCache[price.Symbol]; + if (!pricesByDate.ContainsKey(price.Date)) { pricesByDate.Add(price.Date, price); @@ -290,10 +300,47 @@ namespace MarketData.Cache public bool ContainsPrice(String symbol, DateTime date) { if (!snapshot.PriceCache.ContainsKey(symbol)) return false; + PricesByDate pricesByDate = snapshot.PriceCache[symbol]; return pricesByDate.ContainsKey(date); } + public void ClearCacheOnOrBefore(DateTime onOrBeforeDate, bool collect = false) + { + lock (thisLock) + { + UpdateSnapshot(BuildEvictedPriceCache(onOrBeforeDate), snapshot.RealTimePriceCache, new Dictionary()); + if (collect) GC.Collect(); + } + } + + private Dictionary BuildEvictedPriceCache(DateTime onOrBeforeDate) + { + Dictionary newPriceCache = new Dictionary(); + + foreach (KeyValuePair entry in snapshot.PriceCache) + { + String symbol = entry.Key; + PricesByDate filteredPrices = new PricesByDate(); + PricesByDate existingPrices = entry.Value; + + foreach (KeyValuePair kvp in existingPrices) + { + if (kvp.Key >= onOrBeforeDate) + { + filteredPrices.Add(kvp.Key, kvp.Value); + } + } + + if (filteredPrices.Count > 0) + { + newPriceCache.Add(symbol, filteredPrices); + } + } + + return newPriceCache; + } + private void ThreadProc() { int quantums = 0; @@ -302,22 +349,37 @@ namespace MarketData.Cache while (threadRun) { Thread.Sleep(quantumInterval); - if(!threadRun)break; + if (!threadRun) break; + quantums += quantumInterval; + if (quantums > cacheRefreshAfter) { quantums = 0; + lock (thisLock) { - UpdateSnapshot(snapshot.PriceCache, new Dictionary(), snapshot.NullCache); + DateTime maxDate = snapshot.PriceCache.Values.SelectMany(p => p.Keys).DefaultIfEmpty(DateTime.MinValue).Max(); + + if (maxDate != DateTime.MinValue) + { + DateTime evictBefore = dateGenerator.GenerateHistoricalDates(maxDate, EVICTION_DAYCOUNT).Min(); + int beforeCount = snapshot.PriceCache.Values.Sum(p => p.Count); + Dictionary newCache = BuildEvictedPriceCache(evictBefore); + int afterCount = newCache.Values.Sum(p => p.Count); + int removed = beforeCount - afterCount; + MDTrace.WriteLine(LogLevel.DEBUG, $"GBPriceCache eviction: removed {removed} prices (before={beforeCount}, after={afterCount}) on or before {evictBefore.ToShortDateString()}"); + UpdateSnapshot(newCache, new Dictionary(), new Dictionary()); + GC.Collect(); + } } } } } - private void UpdateSnapshot(Dictionary newPriceCache,Dictionary newRealtimePriceCache, Dictionary newNullCache) + private void UpdateSnapshot(Dictionary newPriceCache, Dictionary newRealtimePriceCache, Dictionary newNullCache) { snapshot = new CacheSnapshot(newPriceCache, newRealtimePriceCache, newNullCache); } - } } +} \ No newline at end of file