Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix GLPriceCache

Browse Source
This commit is contained in:
Sean
2026-02-26 16:24:35 -05:00
parent d0eb2f38f6
commit 2f9c01e5a0
6 changed files with 497 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

478
MarketDataLib/Cache/GLPriceCache.cs Normal file
View File

@@ -0,0 +1,478 @@
using MarketData.MarketDataModel;
using MarketData.Utils;
using MarketData.DataAccess;
using System.Collections.Concurrent;
using System.Threading;
using System.Collections.Generic;
using System;
using System.Threading.Tasks;
using System.Linq;
namespace MarketData.Cache
{
public class GLPriceCache
{
private Dictionary<string, PricesByDate> priceCache = new Dictionary<string, PricesByDate>();
private static GLPriceCache instance = null;
private DateTime latestDate = Utility.Epoch;
private Thread cacheMonitorThread = null;
private volatile bool threadRun = true;
private int cacheCycle = 300000;
private object thisLock = new object();
private object fetchLock = new object();
private GLPriceCache()
{
cacheMonitorThread = new Thread(new ThreadStart(ThreadProc));
cacheMonitorThread.Start();
}
public void Dispose()
{
Thread threadToJoin = null;
lock (thisLock)
{
if (instance == null || !threadRun) return;
threadRun = false;
threadToJoin = cacheMonitorThread;
cacheMonitorThread = null;
instance = null;
}
if (threadToJoin != null)
{
MDTrace.WriteLine(LogLevel.DEBUG, $"[GLPriceCache:Dispose] Thread state is '{Utility.ThreadStateToString(threadToJoin)}'. Joining...");
threadToJoin.Join(5000);
}
MDTrace.WriteLine(LogLevel.DEBUG, "[GLPriceCache:Dispose] End");
}
public static GLPriceCache GetInstance()
{
lock (typeof(GLPriceCache))
{
if (instance == null)
{
instance = new GLPriceCache();
}
return instance;
}
}
public void Add(PortfolioTrades portfolioTrades)
{
List<string> symbols = portfolioTrades.Symbols;
DateTime today = DateTime.Today;
Dictionary<string, DateTime> minTradeDates = symbols.ToDictionary(
sym => sym, sym => portfolioTrades.GetMinTradeDate(sym));
// Symbols that need an intraday refresh:
// - open positions (no close date), or
// - closed today (close price may still be settling)
HashSet<string> mutableSymbols = new HashSet<string>(symbols.Where(sym => portfolioTrades.HasOpenPositions(sym)));
//|| portfolioTrades.GetMaxTradeDate(sym).Date == today));
Dictionary<string, DateTime> minCacheDates;
lock (thisLock)
{
minCacheDates = symbols.ToDictionary(
sym => sym,
sym => priceCache.ContainsKey(sym) ? priceCache[sym].MinDate : DateTime.MaxValue);
}
ConcurrentDictionary<string, Prices> fetchedPrices = new ConcurrentDictionary<string, Prices>();
ConcurrentDictionary<string, Price> latestPrices = new ConcurrentDictionary<string, Price>();
Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol =>
{
DateTime minTradeDate = minTradeDates[symbol];
DateTime minCacheDate = minCacheDates[symbol];
try
{
// Historical fetch — only when cache is missing or incomplete
Prices prices = null;
if (minCacheDate == DateTime.MaxValue)
{
prices = PricingDA.GetPrices(symbol, minTradeDate);
}
else if (minTradeDate < minCacheDate)
{
prices = PricingDA.GetPrices(symbol, minCacheDate, minTradeDate);
}
if (prices != null && prices.Count > 0)
{
fetchedPrices[symbol] = prices;
}
// Intraday refresh — open positions and positions closed today only
if (mutableSymbols.Contains(symbol))
{
Price latestPrice = PricingDA.GetPrice(symbol);
if (latestPrice != null)
latestPrices[symbol] = latestPrice;
}
}
catch (Exception ex)
{
MDTrace.WriteLine(LogLevel.DEBUG, $"Error fetching prices for {symbol}: {ex.Message}");
}
});
lock (thisLock)
{
// Historical prices — idempotent, will not overwrite existing entries
foreach (var kvp in fetchedPrices)
{
foreach (var price in kvp.Value)
{
Add(price);
}
}
// Latest prices — unconditional overwrite to capture any intraday updates
foreach (var kvp in latestPrices)
{
if (!priceCache.TryGetValue(kvp.Key, out var pricesByDate))
{
pricesByDate = new PricesByDate();
priceCache[kvp.Key] = pricesByDate;
}
if (pricesByDate.ContainsKey(kvp.Value.Date))
pricesByDate.Remove(kvp.Value.Date);
pricesByDate.Add(kvp.Value.Date, kvp.Value);
}
}
MDTrace.WriteLine(LogLevel.DEBUG,
$"[GLPriceCache:Add] Symbols: {symbols.Count}, Mutable: {mutableSymbols.Count}, " +
$"Historical fetches: {fetchedPrices.Count}, Intraday updates: {latestPrices.Count}");
}
//public void Add(PortfolioTrades portfolioTrades)
//{
// List<string> symbols = portfolioTrades.Symbols;
// Dictionary<string, DateTime> minTradeDates = symbols.ToDictionary(sym => sym, sym => portfolioTrades.GetMinTradeDate(sym));
// Dictionary<string, DateTime> minCacheDates;
// lock (thisLock)
// {
// minCacheDates = symbols.ToDictionary(sym => sym, sym => priceCache.ContainsKey(sym) ? priceCache[sym].MinDate : DateTime.MaxValue);
// }
// ConcurrentDictionary<string, Prices> fetchedPrices = new ConcurrentDictionary<string, Prices>();
// Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol =>
// {
// DateTime minTradeDate = minTradeDates[symbol];
// DateTime minCacheDate = minCacheDates[symbol];
// Prices prices = null;
// try
// {
// if (minCacheDate == DateTime.MaxValue)
// {
// prices = PricingDA.GetPrices(symbol, minTradeDate);
// }
// else if (minTradeDate < minCacheDate)
// {
// prices = PricingDA.GetPrices(symbol, minCacheDate, minTradeDate);
// }
// if (prices != null && prices.Count > 0)
// {
// fetchedPrices[symbol] = prices;
// }
// }
// catch (Exception ex)
// {
// MDTrace.WriteLine(LogLevel.DEBUG, $"Error fetching prices for {symbol}: {ex.Message}");
// }
// });
// lock (thisLock)
// {
// foreach (var kvp in fetchedPrices)
// {
// foreach (var price in kvp.Value)
// {
// Add(price);
// }
// }
// }
//}
public DateTime GetLatestDate()
{
lock (thisLock)
{
if (Utility.IsEpoch(latestDate))
{
RefreshLatestDate();
}
return latestDate;
}
}
public void RefreshLatestDate()
{
lock (thisLock)
{
latestDate = PricingDA.GetLatestDate();
}
}
public void Refresh()
{
List<string> symbols;
Dictionary<string, DateTime> currentMaxDates;
lock (thisLock)
{
symbols = priceCache.Keys.ToList();
currentMaxDates = priceCache.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.MaxDate);
}
if (symbols.Count == 0) return;
ConcurrentDictionary<string, PricesByDate> fullReloads = new ConcurrentDictionary<string, PricesByDate>();
ConcurrentDictionary<string, Price> singleUpdates = new ConcurrentDictionary<string, Price>();
DateTime latestDateFromDb;
lock (fetchLock)
{
Dictionary<string, DateTime> maxDbDates = PricingDA.GetLatestDates(symbols);
latestDateFromDb = PricingDA.GetLatestDate();
Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol =>
{
if (!currentMaxDates.TryGetValue(symbol, out var cachedMax)) return;
if (maxDbDates.TryGetValue(symbol, out var dbMax) && dbMax.Date != cachedMax.Date)
{
Prices prices = PricingDA.GetPrices(symbol, cachedMax);
if (prices != null) fullReloads[symbol] = prices.GetPricesByDate();
}
else
{
Price price = PricingDA.GetPrice(symbol, cachedMax);
if (price != null) singleUpdates[symbol] = price;
}
});
}
lock (thisLock)
{
latestDate = latestDateFromDb;
foreach (var kvp in fullReloads)
{
if (priceCache.TryGetValue(kvp.Key, out PricesByDate existing) && existing.MaxDate == currentMaxDates[kvp.Key])
{
priceCache[kvp.Key] = kvp.Value;
}
}
foreach (var kvp in singleUpdates)
{
if (priceCache.TryGetValue(kvp.Key, out PricesByDate pricesByDate) && pricesByDate.MaxDate == currentMaxDates[kvp.Key])
{
// Remove the old price (if any) and add the new price properly
if (pricesByDate.ContainsKey(kvp.Value.Date))
pricesByDate.Remove(kvp.Value.Date);
pricesByDate.Add(kvp.Value.Date, kvp.Value);
}
}
}
MDTrace.WriteLine(LogLevel.DEBUG, $"Full reloads: {fullReloads.Count}, Single updates: {singleUpdates.Count}");
}
public void Add(Prices prices)
{
foreach (Price price in prices)
{
Add(price);
}
}
public void Add(List<string> symbols, DateTime pricingDate)
{
if (symbols == null || symbols.Count == 0) return;
ConcurrentDictionary<string, Price> fetchedPrices = new ConcurrentDictionary<string, Price>();
Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol =>
{
lock (thisLock)
{
if (ContainsPrice(symbol, pricingDate)) return;
}
try
{
Price price = PricingDA.GetPrice(symbol, pricingDate);
if (price != null) fetchedPrices[symbol] = price;
}
catch (Exception ex)
{
MDTrace.WriteLine(LogLevel.DEBUG, $"Error fetching price for {symbol} on {pricingDate:yyyy-MM-dd}: {ex.Message}");
}
});
lock (thisLock)
{
foreach (var kvp in fetchedPrices)
{
Add(kvp.Value);
}
}
}
public void Add(Price price)
{
if (price == null) return;
lock (thisLock)
{
if (!priceCache.TryGetValue(price.Symbol, out var pricesByDate))
{
pricesByDate = new PricesByDate();
priceCache[price.Symbol] = pricesByDate;
}
if (!pricesByDate.ContainsKey(price.Date))
{
pricesByDate.Add(price.Date, price); // must use Add() to update MinDate/MaxDate
}
}
}
public DateTime GetMinCacheDate(string symbol)
{
lock (thisLock)
{
if (!priceCache.TryGetValue(symbol, out var symbolPrices) || symbolPrices.Count == 0)
{
return Utility.Epoch;
}
return symbolPrices.MinDate;
}
}
//public void RemoveDate(DateTime date)
//{
// lock (thisLock)
// {
// foreach (var kvp in priceCache)
// {
// kvp.Value.Remove(date);
// }
// }
//}
public Prices GetPrices(string symbol, DateTime endDate, int dayCount)
{
lock (thisLock)
{
if (!priceCache.TryGetValue(symbol, out var pricesByDate)) return new Prices();
DateGenerator dateGenerator = new DateGenerator();
List<DateTime> historicalDates = dateGenerator.GenerateHistoricalDates(endDate, dayCount);
Prices result = new Prices();
foreach (DateTime date in historicalDates)
{
if (pricesByDate.ContainsKey(date))
{
result.Add(pricesByDate[date]);
}
}
return result;
}
}
public Price GetPrice(string symbol, DateTime date)
{
lock (thisLock)
{
if (!priceCache.TryGetValue(symbol, out var pricesByDate)) return null;
return pricesByDate.TryGetValue(date, out var price) ? price : null;
}
}
public bool ContainsPrice(string symbol, DateTime date)
{
lock (thisLock)
{
if (!priceCache.TryGetValue(symbol, out var pricesByDate)) return false;
return pricesByDate.ContainsKey(date);
}
}
public bool ContainsPrice(List<string> symbols, DateTime date)
{
if (symbols == null || symbols.Count == 0) return false;
lock (thisLock)
{
foreach (string symbol in symbols)
{
if (!priceCache.TryGetValue(symbol, out var pricesByDate) || !pricesByDate.ContainsKey(date))
{
return false;
}
}
return true;
}
}
public bool ContainsSymbol(string symbol)
{
lock (thisLock)
{
return priceCache.ContainsKey(symbol);
}
}
private long Count()
{
lock (thisLock)
{
long count = 0;
foreach (var pricesByDate in priceCache.Values)
{
count += pricesByDate.Count;
}
return count;
}
}
private void ThreadProc()
{
int quantums = 0;
int quantumInterval = 1000;
long lastCount = 0;
while (threadRun)
{
Thread.Sleep(quantumInterval);
quantums += quantumInterval;
if (quantums > cacheCycle)
{
quantums = 0;
lock (thisLock)
{
lastCount = Count();
MDTrace.WriteLine(LogLevel.DEBUG, $"[GLPriceCache:ThreadProc] Symbols: {priceCache.Keys.Count}. Items in cache: {Utility.FormatNumber(lastCount,0,true)}.");
}
}
}
MDTrace.WriteLine(LogLevel.DEBUG, $"[GLPriceCache:ThreadProc] Thread ended. Items in cache:{Utility.FormatNumber(lastCount,0,true)}");
}
}
}

8
MarketDataLib/Generator/GainLoss/ActiveGainLossGenerator.cs
View File

@@ -18,7 +18,7 @@ namespace MarketData.Generator.GainLoss
}
//public void RefreshPriceCache()
//{
// LocalPriceCache.GetInstance().Refresh();
// GLPriceCache.GetInstance().Refresh();
//}
// *****************************************************************************************************************************************************************
// ************************************************ G E N E R A T E A C T I V E G A I N L O S S / G A I N L O S S P E R C E N T *****************************
@@ -26,7 +26,7 @@ namespace MarketData.Generator.GainLoss
public GainLossCollection GenerateGainLoss(PortfolioTrades portfolioTrades,DateTime? maxDateRef=null)
{
if (null == portfolioTrades || 0 == portfolioTrades.Count) return null;
LocalPriceCache.GetInstance().Add(portfolioTrades);
GLPriceCache.GetInstance().Add(portfolioTrades);
DateTime minTradeDate = portfolioTrades.GetMinTradeDate();
DateTime maxDate = PricingDA.GetLatestDate();
if(null!=maxDateRef)maxDate=maxDateRef.Value;
@@ -46,11 +46,11 @@ namespace MarketData.Generator.GainLoss
gainLoss.Add(holdingDate, new GainLossItem(holdingDate, 0,0,false));
continue;
}
if(!LocalPriceCache.GetInstance().ContainsPrice(openTrades.Symbols,holdingDate))
if(!GLPriceCache.GetInstance().ContainsPrice(openTrades.Symbols,holdingDate))
{
if(holdingDate.Date.Equals(maxDate))
{
LocalPriceCache.GetInstance().Add(openTrades.Symbols,holdingDate);
GLPriceCache.GetInstance().Add(openTrades.Symbols,holdingDate);
}else continue;
}
foreach (PortfolioTrade portfolioTrade in openTrades)

8
MarketDataLib/Generator/GainLoss/GainLossGenerator.cs
View File

@@ -22,10 +22,10 @@ namespace MarketData.Generator.GainLoss
public TotalGainLossCollection GenerateTotalGainLoss(PortfolioTrades portfolioTrades,DateTime? maxDateRef=null)
{
if (null == portfolioTrades || 0 == portfolioTrades.Count) return null;
LocalPriceCache.GetInstance().Add(portfolioTrades);
GLPriceCache.GetInstance().Add(portfolioTrades);
DateTime minTradeDate = portfolioTrades.GetMinTradeDate();
// DateTime maxDate = PricingDA.GetLatestDate();
DateTime maxDate=LocalPriceCache.GetInstance().GetLatestDate();
DateTime maxDate=GLPriceCache.GetInstance().GetLatestDate();
if(null!=maxDateRef)maxDate=maxDateRef.Value;
Dictionary<DateTime,TotalGainLossItem> gainLossCollection = new Dictionary<DateTime, TotalGainLossItem>();
DateGenerator dateGenerator = new DateGenerator();
@@ -75,10 +75,10 @@ namespace MarketData.Generator.GainLoss
public TotalGainLossCollection GenerateTotalGainLossWithDividends(PortfolioTrades portfolioTrades,DividendPayments dividendPayments,DateTime? maxDateRef=null)
{
if (null == portfolioTrades || 0 == portfolioTrades.Count) return null;
LocalPriceCache.GetInstance().Add(portfolioTrades);
GLPriceCache.GetInstance().Add(portfolioTrades);
DateTime minTradeDate = portfolioTrades.GetMinTradeDate();
// DateTime maxDate = PricingDA.GetLatestDate();
DateTime maxDate=LocalPriceCache.GetInstance().GetLatestDate();
DateTime maxDate=GLPriceCache.GetInstance().GetLatestDate();
if(null!=maxDateRef)maxDate=maxDateRef.Value;
Dictionary<DateTime,TotalGainLossItem> gainLossCollection = new Dictionary<DateTime, TotalGainLossItem>();
DateGenerator dateGenerator = new DateGenerator();

12
MarketDataLib/Generator/GainLoss/GainLossGeneratorCum.cs
View File

@@ -19,12 +19,12 @@ namespace MarketData.Generator.GainLoss
DateGenerator dateGenerator=new DateGenerator();
ModelPerformanceSeries performanceSeries=new ModelPerformanceSeries();
List<TotalGainLossItem> gainLossList=new List<TotalGainLossItem>();
LocalPriceCache.GetInstance().Add(portfolioTrades);
GLPriceCache.GetInstance().Add(portfolioTrades);
try
{
if(!ValidatePortfolioTrades(portfolioTrades))return null;
DateTime minDate=portfolioTrades.GetMinTradeDate();
DateTime maxDate = LocalPriceCache.GetInstance().GetLatestDate();
DateTime maxDate = GLPriceCache.GetInstance().GetLatestDate();
if(null!=maxDateRef) maxDate=maxDateRef.Value;
double prevGainLoss=double.NaN;
List<DateTime> historicalDates=dateGenerator.GenerateHistoricalDates(minDate,maxDate);
@@ -46,7 +46,7 @@ namespace MarketData.Generator.GainLoss
foreach(PortfolioTrade openPosition in openPositions)
{
exposure+=openPosition.Shares*openPosition.Price;
Price price=LocalPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate);
Price price=GLPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",openPosition.Symbol,currentDate.ToShortDateString()));
@@ -98,12 +98,12 @@ namespace MarketData.Generator.GainLoss
DateGenerator dateGenerator=new DateGenerator();
ModelPerformanceSeries performanceSeries=new ModelPerformanceSeries();
List<TotalGainLossItem> gainLossList=new List<TotalGainLossItem>();
LocalPriceCache.GetInstance().Add(portfolioTrades);
GLPriceCache.GetInstance().Add(portfolioTrades);
try
{
if(!ValidatePortfolioTrades(portfolioTrades)) return null;
DateTime minDate=portfolioTrades.Min(x => x.TradeDate);
DateTime maxDate = LocalPriceCache.GetInstance().GetLatestDate();
DateTime maxDate = GLPriceCache.GetInstance().GetLatestDate();
double prevGainLoss=double.NaN;
List<DateTime> historicalDates=dateGenerator.GenerateHistoricalDates(minDate,maxDate);
@@ -123,7 +123,7 @@ namespace MarketData.Generator.GainLoss
foreach(PortfolioTrade openPosition in openPositions)
{
exposure+=openPosition.Shares*openPosition.Price;
Price price=LocalPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate);
Price price=GLPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",openPosition.Symbol,currentDate.ToShortDateString()));

8
MarketDataLib/Generator/GainLoss/GainLossHelper.cs
View File

@@ -20,7 +20,7 @@ namespace MarketData.Generator.GainLoss
if(holdingDate<portfolioTrade.TradeDate) return null;
if(portfolioTrade.IsOpen||(portfolioTrade.IsClosed&&portfolioTrade.SellDate>holdingDate))
{
Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate)));
@@ -35,7 +35,7 @@ namespace MarketData.Generator.GainLoss
if(holdingDate<portfolioTrade.TradeDate) return null;
if(portfolioTrade.IsOpen||(portfolioTrade.IsClosed&&portfolioTrade.SellDate>holdingDate))
{
Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate)));
@@ -69,7 +69,7 @@ namespace MarketData.Generator.GainLoss
{
return (portfolioTrade.SellPrice*portfolioTrade.Shares)-(portfolioTrade.Price*portfolioTrade.Shares);
}
Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate)));
@@ -83,7 +83,7 @@ namespace MarketData.Generator.GainLoss
if(!portfolioTrade.SellDate.Equals(Utility.Epoch)&&holdingDate>portfolioTrade.SellDate) return null;
// check to see if we bought and sold on the same date.
if(portfolioTrade.SellDate.Equals(portfolioTrade.TradeDate)) return (portfolioTrade.SellPrice*portfolioTrade.Shares);
Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate)));

1
MarketDataLib/MarketDataLib.csproj
View File

@@ -74,6 +74,7 @@
<ItemGroup>
<Compile Include="Cache\DividendHistoryCache.cs" />
<Compile Include="Cache\GBPriceCache.cs" />
<Compile Include="Cache\GLPriceCache.cs" />
<Compile Include="Cache\LocalPriceCache.cs" />
<Compile Include="CNNProcessing\BitmapExtensions.cs" />
<Compile Include="CNNProcessing\CNNClient.cs" />