From 2f9c01e5a0036a6980d045f6bb5695670675e581 Mon Sep 17 00:00:00 2001 From: Sean Date: Thu, 26 Feb 2026 16:24:35 -0500 Subject: [PATCH] Fix GLPriceCache --- MarketDataLib/Cache/GLPriceCache.cs | 478 ++++++++++++++++++ .../GainLoss/ActiveGainLossGenerator.cs | 8 +- .../Generator/GainLoss/GainLossGenerator.cs | 8 +- .../GainLoss/GainLossGeneratorCum.cs | 12 +- .../Generator/GainLoss/GainLossHelper.cs | 8 +- MarketDataLib/MarketDataLib.csproj | 1 + 6 files changed, 497 insertions(+), 18 deletions(-) create mode 100644 MarketDataLib/Cache/GLPriceCache.cs diff --git a/MarketDataLib/Cache/GLPriceCache.cs b/MarketDataLib/Cache/GLPriceCache.cs new file mode 100644 index 0000000..9e8269f --- /dev/null +++ b/MarketDataLib/Cache/GLPriceCache.cs @@ -0,0 +1,478 @@ +using MarketData.MarketDataModel; +using MarketData.Utils; +using MarketData.DataAccess; +using System.Collections.Concurrent; +using System.Threading; +using System.Collections.Generic; +using System; +using System.Threading.Tasks; +using System.Linq; + +namespace MarketData.Cache +{ + public class GLPriceCache + { + private Dictionary priceCache = new Dictionary(); + private static GLPriceCache instance = null; + private DateTime latestDate = Utility.Epoch; + private Thread cacheMonitorThread = null; + private volatile bool threadRun = true; + private int cacheCycle = 300000; + private object thisLock = new object(); + private object fetchLock = new object(); + + private GLPriceCache() + { + cacheMonitorThread = new Thread(new ThreadStart(ThreadProc)); + cacheMonitorThread.Start(); + } + + public void Dispose() + { + Thread threadToJoin = null; + + lock (thisLock) + { + if (instance == null || !threadRun) return; + threadRun = false; + threadToJoin = cacheMonitorThread; + cacheMonitorThread = null; + instance = null; + } + + if (threadToJoin != null) + { + MDTrace.WriteLine(LogLevel.DEBUG, $"[GLPriceCache:Dispose] Thread state is '{Utility.ThreadStateToString(threadToJoin)}'. Joining..."); + threadToJoin.Join(5000); + } + + MDTrace.WriteLine(LogLevel.DEBUG, "[GLPriceCache:Dispose] End"); + } + + public static GLPriceCache GetInstance() + { + lock (typeof(GLPriceCache)) + { + if (instance == null) + { + instance = new GLPriceCache(); + } + return instance; + } + } + + public void Add(PortfolioTrades portfolioTrades) + { + List symbols = portfolioTrades.Symbols; + DateTime today = DateTime.Today; + + Dictionary minTradeDates = symbols.ToDictionary( + sym => sym, sym => portfolioTrades.GetMinTradeDate(sym)); + + // Symbols that need an intraday refresh: + // - open positions (no close date), or + // - closed today (close price may still be settling) + HashSet mutableSymbols = new HashSet(symbols.Where(sym => portfolioTrades.HasOpenPositions(sym))); + //|| portfolioTrades.GetMaxTradeDate(sym).Date == today)); + + Dictionary minCacheDates; + lock (thisLock) + { + minCacheDates = symbols.ToDictionary( + sym => sym, + sym => priceCache.ContainsKey(sym) ? priceCache[sym].MinDate : DateTime.MaxValue); + } + + ConcurrentDictionary fetchedPrices = new ConcurrentDictionary(); + ConcurrentDictionary latestPrices = new ConcurrentDictionary(); + + Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol => + { + DateTime minTradeDate = minTradeDates[symbol]; + DateTime minCacheDate = minCacheDates[symbol]; + + try + { + // Historical fetch — only when cache is missing or incomplete + Prices prices = null; + if (minCacheDate == DateTime.MaxValue) + { + prices = PricingDA.GetPrices(symbol, minTradeDate); + } + else if (minTradeDate < minCacheDate) + { + prices = PricingDA.GetPrices(symbol, minCacheDate, minTradeDate); + } + + if (prices != null && prices.Count > 0) + { + fetchedPrices[symbol] = prices; + } + + // Intraday refresh — open positions and positions closed today only + if (mutableSymbols.Contains(symbol)) + { + Price latestPrice = PricingDA.GetPrice(symbol); + if (latestPrice != null) + latestPrices[symbol] = latestPrice; + } + } + catch (Exception ex) + { + MDTrace.WriteLine(LogLevel.DEBUG, $"Error fetching prices for {symbol}: {ex.Message}"); + } + }); + + lock (thisLock) + { + // Historical prices — idempotent, will not overwrite existing entries + foreach (var kvp in fetchedPrices) + { + foreach (var price in kvp.Value) + { + Add(price); + } + } + + // Latest prices — unconditional overwrite to capture any intraday updates + foreach (var kvp in latestPrices) + { + if (!priceCache.TryGetValue(kvp.Key, out var pricesByDate)) + { + pricesByDate = new PricesByDate(); + priceCache[kvp.Key] = pricesByDate; + } + + if (pricesByDate.ContainsKey(kvp.Value.Date)) + pricesByDate.Remove(kvp.Value.Date); + pricesByDate.Add(kvp.Value.Date, kvp.Value); + } + } + + MDTrace.WriteLine(LogLevel.DEBUG, + $"[GLPriceCache:Add] Symbols: {symbols.Count}, Mutable: {mutableSymbols.Count}, " + + $"Historical fetches: {fetchedPrices.Count}, Intraday updates: {latestPrices.Count}"); + } + + //public void Add(PortfolioTrades portfolioTrades) + //{ + // List symbols = portfolioTrades.Symbols; + // Dictionary minTradeDates = symbols.ToDictionary(sym => sym, sym => portfolioTrades.GetMinTradeDate(sym)); + + // Dictionary minCacheDates; + // lock (thisLock) + // { + // minCacheDates = symbols.ToDictionary(sym => sym, sym => priceCache.ContainsKey(sym) ? priceCache[sym].MinDate : DateTime.MaxValue); + // } + + // ConcurrentDictionary fetchedPrices = new ConcurrentDictionary(); + + // Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol => + // { + // DateTime minTradeDate = minTradeDates[symbol]; + // DateTime minCacheDate = minCacheDates[symbol]; + // Prices prices = null; + + // try + // { + // if (minCacheDate == DateTime.MaxValue) + // { + // prices = PricingDA.GetPrices(symbol, minTradeDate); + // } + // else if (minTradeDate < minCacheDate) + // { + // prices = PricingDA.GetPrices(symbol, minCacheDate, minTradeDate); + // } + + // if (prices != null && prices.Count > 0) + // { + // fetchedPrices[symbol] = prices; + // } + // } + // catch (Exception ex) + // { + // MDTrace.WriteLine(LogLevel.DEBUG, $"Error fetching prices for {symbol}: {ex.Message}"); + // } + // }); + + // lock (thisLock) + // { + // foreach (var kvp in fetchedPrices) + // { + // foreach (var price in kvp.Value) + // { + // Add(price); + // } + // } + // } + //} + + public DateTime GetLatestDate() + { + lock (thisLock) + { + if (Utility.IsEpoch(latestDate)) + { + RefreshLatestDate(); + } + return latestDate; + } + } + + public void RefreshLatestDate() + { + lock (thisLock) + { + latestDate = PricingDA.GetLatestDate(); + } + } + + public void Refresh() + { + List symbols; + Dictionary currentMaxDates; + + lock (thisLock) + { + symbols = priceCache.Keys.ToList(); + currentMaxDates = priceCache.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.MaxDate); + } + if (symbols.Count == 0) return; + ConcurrentDictionary fullReloads = new ConcurrentDictionary(); + ConcurrentDictionary singleUpdates = new ConcurrentDictionary(); + DateTime latestDateFromDb; + lock (fetchLock) + { + Dictionary maxDbDates = PricingDA.GetLatestDates(symbols); + latestDateFromDb = PricingDA.GetLatestDate(); + + Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol => + { + if (!currentMaxDates.TryGetValue(symbol, out var cachedMax)) return; + + if (maxDbDates.TryGetValue(symbol, out var dbMax) && dbMax.Date != cachedMax.Date) + { + Prices prices = PricingDA.GetPrices(symbol, cachedMax); + if (prices != null) fullReloads[symbol] = prices.GetPricesByDate(); + } + else + { + Price price = PricingDA.GetPrice(symbol, cachedMax); + if (price != null) singleUpdates[symbol] = price; + } + }); + } + + lock (thisLock) + { + latestDate = latestDateFromDb; + + foreach (var kvp in fullReloads) + { + if (priceCache.TryGetValue(kvp.Key, out PricesByDate existing) && existing.MaxDate == currentMaxDates[kvp.Key]) + { + priceCache[kvp.Key] = kvp.Value; + } + } + + foreach (var kvp in singleUpdates) + { + if (priceCache.TryGetValue(kvp.Key, out PricesByDate pricesByDate) && pricesByDate.MaxDate == currentMaxDates[kvp.Key]) + { + // Remove the old price (if any) and add the new price properly + if (pricesByDate.ContainsKey(kvp.Value.Date)) + pricesByDate.Remove(kvp.Value.Date); + pricesByDate.Add(kvp.Value.Date, kvp.Value); + } + } + } + + MDTrace.WriteLine(LogLevel.DEBUG, $"Full reloads: {fullReloads.Count}, Single updates: {singleUpdates.Count}"); + } + + + public void Add(Prices prices) + { + foreach (Price price in prices) + { + Add(price); + } + } + + public void Add(List symbols, DateTime pricingDate) + { + if (symbols == null || symbols.Count == 0) return; + + ConcurrentDictionary fetchedPrices = new ConcurrentDictionary(); + + Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol => + { + lock (thisLock) + { + if (ContainsPrice(symbol, pricingDate)) return; + } + + try + { + Price price = PricingDA.GetPrice(symbol, pricingDate); + if (price != null) fetchedPrices[symbol] = price; + } + catch (Exception ex) + { + MDTrace.WriteLine(LogLevel.DEBUG, $"Error fetching price for {symbol} on {pricingDate:yyyy-MM-dd}: {ex.Message}"); + } + }); + + lock (thisLock) + { + foreach (var kvp in fetchedPrices) + { + Add(kvp.Value); + } + } + } + + public void Add(Price price) + { + if (price == null) return; + + lock (thisLock) + { + if (!priceCache.TryGetValue(price.Symbol, out var pricesByDate)) + { + pricesByDate = new PricesByDate(); + priceCache[price.Symbol] = pricesByDate; + } + if (!pricesByDate.ContainsKey(price.Date)) + { + pricesByDate.Add(price.Date, price); // must use Add() to update MinDate/MaxDate + } + } + } + + public DateTime GetMinCacheDate(string symbol) + { + lock (thisLock) + { + if (!priceCache.TryGetValue(symbol, out var symbolPrices) || symbolPrices.Count == 0) + { + return Utility.Epoch; + } + return symbolPrices.MinDate; + } + } + + //public void RemoveDate(DateTime date) + //{ + // lock (thisLock) + // { + // foreach (var kvp in priceCache) + // { + // kvp.Value.Remove(date); + // } + // } + //} + + public Prices GetPrices(string symbol, DateTime endDate, int dayCount) + { + lock (thisLock) + { + if (!priceCache.TryGetValue(symbol, out var pricesByDate)) return new Prices(); + + DateGenerator dateGenerator = new DateGenerator(); + List historicalDates = dateGenerator.GenerateHistoricalDates(endDate, dayCount); + + Prices result = new Prices(); + foreach (DateTime date in historicalDates) + { + if (pricesByDate.ContainsKey(date)) + { + result.Add(pricesByDate[date]); + } + } + + return result; + } + } + + public Price GetPrice(string symbol, DateTime date) + { + lock (thisLock) + { + if (!priceCache.TryGetValue(symbol, out var pricesByDate)) return null; + return pricesByDate.TryGetValue(date, out var price) ? price : null; + } + } + + public bool ContainsPrice(string symbol, DateTime date) + { + lock (thisLock) + { + if (!priceCache.TryGetValue(symbol, out var pricesByDate)) return false; + return pricesByDate.ContainsKey(date); + } + } + + public bool ContainsPrice(List symbols, DateTime date) + { + if (symbols == null || symbols.Count == 0) return false; + + lock (thisLock) + { + foreach (string symbol in symbols) + { + if (!priceCache.TryGetValue(symbol, out var pricesByDate) || !pricesByDate.ContainsKey(date)) + { + return false; + } + } + return true; + } + } + + public bool ContainsSymbol(string symbol) + { + lock (thisLock) + { + return priceCache.ContainsKey(symbol); + } + } + + private long Count() + { + lock (thisLock) + { + long count = 0; + foreach (var pricesByDate in priceCache.Values) + { + count += pricesByDate.Count; + } + return count; + } + } + + private void ThreadProc() + { + int quantums = 0; + int quantumInterval = 1000; + long lastCount = 0; + + while (threadRun) + { + Thread.Sleep(quantumInterval); + quantums += quantumInterval; + if (quantums > cacheCycle) + { + quantums = 0; + lock (thisLock) + { + lastCount = Count(); + MDTrace.WriteLine(LogLevel.DEBUG, $"[GLPriceCache:ThreadProc] Symbols: {priceCache.Keys.Count}. Items in cache: {Utility.FormatNumber(lastCount,0,true)}."); + } + } + } + + MDTrace.WriteLine(LogLevel.DEBUG, $"[GLPriceCache:ThreadProc] Thread ended. Items in cache:{Utility.FormatNumber(lastCount,0,true)}"); + } + } +} diff --git a/MarketDataLib/Generator/GainLoss/ActiveGainLossGenerator.cs b/MarketDataLib/Generator/GainLoss/ActiveGainLossGenerator.cs index 8320f51..7c6a4f4 100644 --- a/MarketDataLib/Generator/GainLoss/ActiveGainLossGenerator.cs +++ b/MarketDataLib/Generator/GainLoss/ActiveGainLossGenerator.cs @@ -18,7 +18,7 @@ namespace MarketData.Generator.GainLoss } //public void RefreshPriceCache() //{ - // LocalPriceCache.GetInstance().Refresh(); + // GLPriceCache.GetInstance().Refresh(); //} // ***************************************************************************************************************************************************************** // ************************************************ G E N E R A T E A C T I V E G A I N L O S S / G A I N L O S S P E R C E N T ***************************** @@ -26,7 +26,7 @@ namespace MarketData.Generator.GainLoss public GainLossCollection GenerateGainLoss(PortfolioTrades portfolioTrades,DateTime? maxDateRef=null) { if (null == portfolioTrades || 0 == portfolioTrades.Count) return null; - LocalPriceCache.GetInstance().Add(portfolioTrades); + GLPriceCache.GetInstance().Add(portfolioTrades); DateTime minTradeDate = portfolioTrades.GetMinTradeDate(); DateTime maxDate = PricingDA.GetLatestDate(); if(null!=maxDateRef)maxDate=maxDateRef.Value; @@ -46,11 +46,11 @@ namespace MarketData.Generator.GainLoss gainLoss.Add(holdingDate, new GainLossItem(holdingDate, 0,0,false)); continue; } - if(!LocalPriceCache.GetInstance().ContainsPrice(openTrades.Symbols,holdingDate)) + if(!GLPriceCache.GetInstance().ContainsPrice(openTrades.Symbols,holdingDate)) { if(holdingDate.Date.Equals(maxDate)) { - LocalPriceCache.GetInstance().Add(openTrades.Symbols,holdingDate); + GLPriceCache.GetInstance().Add(openTrades.Symbols,holdingDate); }else continue; } foreach (PortfolioTrade portfolioTrade in openTrades) diff --git a/MarketDataLib/Generator/GainLoss/GainLossGenerator.cs b/MarketDataLib/Generator/GainLoss/GainLossGenerator.cs index 89dd369..c2e565f 100644 --- a/MarketDataLib/Generator/GainLoss/GainLossGenerator.cs +++ b/MarketDataLib/Generator/GainLoss/GainLossGenerator.cs @@ -22,10 +22,10 @@ namespace MarketData.Generator.GainLoss public TotalGainLossCollection GenerateTotalGainLoss(PortfolioTrades portfolioTrades,DateTime? maxDateRef=null) { if (null == portfolioTrades || 0 == portfolioTrades.Count) return null; - LocalPriceCache.GetInstance().Add(portfolioTrades); + GLPriceCache.GetInstance().Add(portfolioTrades); DateTime minTradeDate = portfolioTrades.GetMinTradeDate(); // DateTime maxDate = PricingDA.GetLatestDate(); - DateTime maxDate=LocalPriceCache.GetInstance().GetLatestDate(); + DateTime maxDate=GLPriceCache.GetInstance().GetLatestDate(); if(null!=maxDateRef)maxDate=maxDateRef.Value; Dictionary gainLossCollection = new Dictionary(); DateGenerator dateGenerator = new DateGenerator(); @@ -75,10 +75,10 @@ namespace MarketData.Generator.GainLoss public TotalGainLossCollection GenerateTotalGainLossWithDividends(PortfolioTrades portfolioTrades,DividendPayments dividendPayments,DateTime? maxDateRef=null) { if (null == portfolioTrades || 0 == portfolioTrades.Count) return null; - LocalPriceCache.GetInstance().Add(portfolioTrades); + GLPriceCache.GetInstance().Add(portfolioTrades); DateTime minTradeDate = portfolioTrades.GetMinTradeDate(); // DateTime maxDate = PricingDA.GetLatestDate(); - DateTime maxDate=LocalPriceCache.GetInstance().GetLatestDate(); + DateTime maxDate=GLPriceCache.GetInstance().GetLatestDate(); if(null!=maxDateRef)maxDate=maxDateRef.Value; Dictionary gainLossCollection = new Dictionary(); DateGenerator dateGenerator = new DateGenerator(); diff --git a/MarketDataLib/Generator/GainLoss/GainLossGeneratorCum.cs b/MarketDataLib/Generator/GainLoss/GainLossGeneratorCum.cs index 0d3c043..9db4eae 100644 --- a/MarketDataLib/Generator/GainLoss/GainLossGeneratorCum.cs +++ b/MarketDataLib/Generator/GainLoss/GainLossGeneratorCum.cs @@ -19,12 +19,12 @@ namespace MarketData.Generator.GainLoss DateGenerator dateGenerator=new DateGenerator(); ModelPerformanceSeries performanceSeries=new ModelPerformanceSeries(); List gainLossList=new List(); - LocalPriceCache.GetInstance().Add(portfolioTrades); + GLPriceCache.GetInstance().Add(portfolioTrades); try { if(!ValidatePortfolioTrades(portfolioTrades))return null; DateTime minDate=portfolioTrades.GetMinTradeDate(); - DateTime maxDate = LocalPriceCache.GetInstance().GetLatestDate(); + DateTime maxDate = GLPriceCache.GetInstance().GetLatestDate(); if(null!=maxDateRef) maxDate=maxDateRef.Value; double prevGainLoss=double.NaN; List historicalDates=dateGenerator.GenerateHistoricalDates(minDate,maxDate); @@ -46,7 +46,7 @@ namespace MarketData.Generator.GainLoss foreach(PortfolioTrade openPosition in openPositions) { exposure+=openPosition.Shares*openPosition.Price; - Price price=LocalPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate); + Price price=GLPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate); if(null==price) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",openPosition.Symbol,currentDate.ToShortDateString())); @@ -98,12 +98,12 @@ namespace MarketData.Generator.GainLoss DateGenerator dateGenerator=new DateGenerator(); ModelPerformanceSeries performanceSeries=new ModelPerformanceSeries(); List gainLossList=new List(); - LocalPriceCache.GetInstance().Add(portfolioTrades); + GLPriceCache.GetInstance().Add(portfolioTrades); try { if(!ValidatePortfolioTrades(portfolioTrades)) return null; DateTime minDate=portfolioTrades.Min(x => x.TradeDate); - DateTime maxDate = LocalPriceCache.GetInstance().GetLatestDate(); + DateTime maxDate = GLPriceCache.GetInstance().GetLatestDate(); double prevGainLoss=double.NaN; List historicalDates=dateGenerator.GenerateHistoricalDates(minDate,maxDate); @@ -123,7 +123,7 @@ namespace MarketData.Generator.GainLoss foreach(PortfolioTrade openPosition in openPositions) { exposure+=openPosition.Shares*openPosition.Price; - Price price=LocalPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate); + Price price=GLPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate); if(null==price) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",openPosition.Symbol,currentDate.ToShortDateString())); diff --git a/MarketDataLib/Generator/GainLoss/GainLossHelper.cs b/MarketDataLib/Generator/GainLoss/GainLossHelper.cs index 088959a..f1d7dd1 100644 --- a/MarketDataLib/Generator/GainLoss/GainLossHelper.cs +++ b/MarketDataLib/Generator/GainLoss/GainLossHelper.cs @@ -20,7 +20,7 @@ namespace MarketData.Generator.GainLoss if(holdingDateholdingDate)) { - Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate); + Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate); if(null==price) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate))); @@ -35,7 +35,7 @@ namespace MarketData.Generator.GainLoss if(holdingDateholdingDate)) { - Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate); + Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate); if(null==price) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate))); @@ -69,7 +69,7 @@ namespace MarketData.Generator.GainLoss { return (portfolioTrade.SellPrice*portfolioTrade.Shares)-(portfolioTrade.Price*portfolioTrade.Shares); } - Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate); + Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate); if(null==price) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate))); @@ -83,7 +83,7 @@ namespace MarketData.Generator.GainLoss if(!portfolioTrade.SellDate.Equals(Utility.Epoch)&&holdingDate>portfolioTrade.SellDate) return null; // check to see if we bought and sold on the same date. if(portfolioTrade.SellDate.Equals(portfolioTrade.TradeDate)) return (portfolioTrade.SellPrice*portfolioTrade.Shares); - Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate); + Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate); if(null==price) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate))); diff --git a/MarketDataLib/MarketDataLib.csproj b/MarketDataLib/MarketDataLib.csproj index 6e7d612..0931650 100644 --- a/MarketDataLib/MarketDataLib.csproj +++ b/MarketDataLib/MarketDataLib.csproj @@ -74,6 +74,7 @@ +