diff --git a/MarketData/MarketDataLib/Cache/GLPriceCache.cs b/MarketData/MarketDataLib/Cache/GLPriceCache.cs
new file mode 100644
index 0000000..19ab997
--- /dev/null
+++ b/MarketData/MarketDataLib/Cache/GLPriceCache.cs
@@ -0,0 +1,409 @@
+using MarketData.MarketDataModel;
+using MarketData.Utils;
+using MarketData.DataAccess;
+using System.Collections.Concurrent;
+using System.Threading;
+using System.Collections.Generic;
+using System;
+using System.Threading.Tasks;
+using System.Linq;
+
+namespace MarketData.Cache
+{
+ ///
+ /// This cache is used in the GainLoss Generators
+ ///
+ public class GLPriceCache
+ {
+ private Dictionary priceCache = new Dictionary();
+ private static GLPriceCache instance = null;
+ private DateTime latestDate = Utility.Epoch;
+ private Thread cacheMonitorThread = null;
+ private volatile bool threadRun = true;
+ private int cacheCycle = 300000;
+ private object thisLock = new object();
+ private object fetchLock = new object();
+
+ private GLPriceCache()
+ {
+ cacheMonitorThread = new Thread(new ThreadStart(ThreadProc));
+ cacheMonitorThread.Start();
+ }
+
+ public void Clear()
+ {
+ lock (thisLock)
+ {
+ priceCache = new Dictionary();
+ RefreshLatestDate();
+ }
+ }
+
+ public void Dispose()
+ {
+ Thread threadToJoin = null;
+
+ lock (thisLock)
+ {
+ if (instance == null || !threadRun) return;
+ threadRun = false;
+ threadToJoin = cacheMonitorThread;
+ cacheMonitorThread = null;
+ instance = null;
+ }
+
+ if (threadToJoin != null)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG, $"[GLPriceCache:Dispose] Thread state is '{Utility.ThreadStateToString(threadToJoin)}'. Joining...");
+ threadToJoin.Join(5000);
+ }
+
+ MDTrace.WriteLine(LogLevel.DEBUG, "[GLPriceCache:Dispose] End");
+ }
+
+ public static GLPriceCache GetInstance()
+ {
+ lock (typeof(LocalPriceCache))
+ {
+ if (instance == null)
+ {
+ instance = new GLPriceCache();
+ }
+ return instance;
+ }
+ }
+
+ public void RefreshLatestDate()
+ {
+ lock (thisLock)
+ {
+ latestDate = PricingDA.GetLatestDate();
+ }
+ }
+
+ public DateTime GetLatestDate()
+ {
+ lock (thisLock)
+ {
+ if (Utility.IsEpoch(latestDate))
+ {
+ RefreshLatestDate();
+ }
+ return latestDate;
+ }
+ }
+
+ public void Refresh()
+ {
+ List symbols;
+ Dictionary currentMaxDates;
+
+ lock (thisLock)
+ {
+ symbols = priceCache.Keys.ToList();
+ currentMaxDates = priceCache.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.MaxDate);
+ }
+
+ if (symbols.Count == 0) return;
+
+ ConcurrentDictionary fullReloads = new ConcurrentDictionary();
+ ConcurrentDictionary singleUpdates = new ConcurrentDictionary();
+ DateTime latestDateFromDb;
+
+ lock (fetchLock)
+ {
+ Dictionary maxDbDates = PricingDA.GetLatestDates(symbols);
+ latestDateFromDb = PricingDA.GetLatestDate();
+
+ Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol =>
+ {
+ DateTime cachedMax;
+ if (!currentMaxDates.TryGetValue(symbol, out cachedMax)) return;
+
+ DateTime dbMax;
+ if (maxDbDates.TryGetValue(symbol, out dbMax) && dbMax.Date != cachedMax.Date)
+ {
+ Prices prices = PricingDA.GetPrices(symbol, cachedMax);
+ if (prices != null) fullReloads[symbol] = prices.GetPricesByDate();
+ }
+ else
+ {
+ Price price = PricingDA.GetPrice(symbol, cachedMax);
+ if (price != null) singleUpdates[symbol] = price;
+ }
+ });
+ }
+
+ lock (thisLock)
+ {
+ latestDate = latestDateFromDb;
+
+ foreach (KeyValuePair kvp in fullReloads)
+ {
+ if (priceCache.TryGetValue(kvp.Key, out PricesByDate existing) && existing.MaxDate == currentMaxDates[kvp.Key])
+ {
+ priceCache[kvp.Key] = kvp.Value;
+ }
+ }
+
+ foreach (KeyValuePair kvp in singleUpdates)
+ {
+ if (priceCache.TryGetValue(kvp.Key, out PricesByDate pricesByDate) && pricesByDate.MaxDate == currentMaxDates[kvp.Key])
+ {
+ if (pricesByDate.ContainsKey(kvp.Value.Date))
+ pricesByDate.Remove(kvp.Value.Date);
+ pricesByDate.Add(kvp.Value.Date, kvp.Value);
+ }
+ }
+ }
+
+ MDTrace.WriteLine(LogLevel.DEBUG, $"Full reloads: {fullReloads.Count}, Single updates: {singleUpdates.Count}");
+ }
+
+ public void Add(PortfolioTrades portfolioTrades)
+ {
+ List symbols = portfolioTrades.Symbols;
+ Dictionary minTradeDates = symbols.ToDictionary(sym => sym, sym => portfolioTrades.GetMinTradeDate(sym));
+
+ Dictionary minCacheDates;
+ lock (thisLock)
+ {
+ minCacheDates = symbols.ToDictionary(sym => sym, sym => priceCache.ContainsKey(sym) ? priceCache[sym].MinDate : DateTime.MaxValue);
+ }
+
+ ConcurrentDictionary fetchedPrices = new ConcurrentDictionary();
+
+ Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol =>
+ {
+ DateTime minTradeDate = minTradeDates[symbol];
+ DateTime minCacheDate = minCacheDates[symbol];
+ Prices prices = null;
+
+ try
+ {
+ if (minCacheDate == DateTime.MaxValue)
+ {
+ prices = PricingDA.GetPrices(symbol, minTradeDate);
+ }
+ else if (minTradeDate < minCacheDate)
+ {
+ prices = PricingDA.GetPrices(symbol, minCacheDate, minTradeDate);
+ }
+
+ if (prices != null && prices.Count > 0)
+ {
+ fetchedPrices[symbol] = prices;
+ }
+ }
+ catch (Exception ex)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG, $"Error fetching prices for {symbol}: {ex.Message}");
+ }
+ });
+
+ lock (thisLock)
+ {
+ foreach (KeyValuePair kvp in fetchedPrices)
+ {
+ foreach (Price price in kvp.Value)
+ {
+ Add(price);
+ }
+ }
+ }
+ }
+
+ public void Add(Prices prices)
+ {
+ foreach (Price price in prices)
+ {
+ Add(price);
+ }
+ }
+
+ public void Add(List symbols, DateTime pricingDate)
+ {
+ if (symbols == null || symbols.Count == 0) return;
+
+ ConcurrentDictionary fetchedPrices = new ConcurrentDictionary();
+
+ Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol =>
+ {
+ lock (thisLock)
+ {
+ if (ContainsPrice(symbol, pricingDate)) return;
+ }
+
+ try
+ {
+ Price price = PricingDA.GetPrice(symbol, pricingDate);
+ if (price != null) fetchedPrices[symbol] = price;
+ }
+ catch (Exception ex)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG, $"GLPriceCache: Error fetching price for {symbol} on {pricingDate:yyyy-MM-dd}: {ex.Message}");
+ }
+ });
+
+ lock (thisLock)
+ {
+ foreach (KeyValuePair kvp in fetchedPrices)
+ {
+ Add(kvp.Value);
+ }
+ }
+ }
+
+ public void Add(Price price)
+ {
+ if (price == null) return;
+
+ lock (thisLock)
+ {
+ PricesByDate pricesByDate;
+ if (!priceCache.TryGetValue(price.Symbol, out pricesByDate))
+ {
+ pricesByDate = new PricesByDate();
+ priceCache[price.Symbol] = pricesByDate;
+ }
+ if (!pricesByDate.ContainsKey(price.Date))
+ {
+ pricesByDate.Add(price.Date, price);
+ }
+ }
+ }
+
+ public DateTime GetMinCacheDate(string symbol)
+ {
+ lock (thisLock)
+ {
+ PricesByDate symbolPrices;
+ if (!priceCache.TryGetValue(symbol, out symbolPrices) || symbolPrices.Count == 0)
+ {
+ return Utility.Epoch;
+ }
+ return symbolPrices.MinDate;
+ }
+ }
+
+ public void RemoveDate(DateTime date)
+ {
+ lock (thisLock)
+ {
+ foreach (KeyValuePair kvp in priceCache)
+ {
+ kvp.Value.Remove(date);
+ }
+ }
+ }
+
+ public Prices GetPrices(string symbol, DateTime endDate, int dayCount)
+ {
+ lock (thisLock)
+ {
+ PricesByDate pricesByDate;
+ if (!priceCache.TryGetValue(symbol, out pricesByDate)) return new Prices();
+
+ DateGenerator dateGenerator = new DateGenerator();
+ List historicalDates = dateGenerator.GenerateHistoricalDates(endDate, dayCount);
+
+ Prices result = new Prices();
+ foreach (DateTime date in historicalDates)
+ {
+ if (pricesByDate.ContainsKey(date))
+ {
+ result.Add(pricesByDate[date]);
+ }
+ }
+
+ return result;
+ }
+ }
+
+ public Price GetPrice(string symbol, DateTime date)
+ {
+ lock (thisLock)
+ {
+ PricesByDate pricesByDate;
+ if (!priceCache.TryGetValue(symbol, out pricesByDate)) return null;
+
+ Price price;
+ return pricesByDate.TryGetValue(date, out price) ? price : null;
+ }
+ }
+
+ public bool ContainsPrice(string symbol, DateTime date)
+ {
+ lock (thisLock)
+ {
+ PricesByDate pricesByDate;
+ if (!priceCache.TryGetValue(symbol, out pricesByDate)) return false;
+ return pricesByDate.ContainsKey(date);
+ }
+ }
+
+ public bool ContainsPrice(List symbols, DateTime date)
+ {
+ if (symbols == null || symbols.Count == 0) return false;
+
+ lock (thisLock)
+ {
+ foreach (string symbol in symbols)
+ {
+ PricesByDate pricesByDate;
+ if (!priceCache.TryGetValue(symbol, out pricesByDate) || !pricesByDate.ContainsKey(date))
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+
+ public bool ContainsSymbol(string symbol)
+ {
+ lock (thisLock)
+ {
+ return priceCache.ContainsKey(symbol);
+ }
+ }
+
+ private long Count()
+ {
+ lock (thisLock)
+ {
+ long count = 0;
+ foreach (PricesByDate pricesByDate in priceCache.Values)
+ {
+ count += pricesByDate.Count;
+ }
+ return count;
+ }
+ }
+
+ private void ThreadProc()
+ {
+ int quantums = 0;
+ int quantumInterval = 1000;
+ long lastCount = 0;
+
+ while (threadRun)
+ {
+ Thread.Sleep(quantumInterval);
+ if(!threadRun)break;
+ quantums += quantumInterval;
+ if (quantums > cacheCycle)
+ {
+ quantums = 0;
+ lock (thisLock)
+ {
+ lastCount = Count();
+ MDTrace.WriteLine(LogLevel.DEBUG, $"[GLPriceCache:ThreadProc] Symbols: {priceCache.Keys.Count}. Items in cache: {Utility.FormatNumber(lastCount,0,true)}.");
+ }
+ }
+ }
+
+ MDTrace.WriteLine(LogLevel.DEBUG, $"[GLPriceCache:ThreadProc] Thread ended. Items in cache:{Utility.FormatNumber(lastCount,0,true)}");
+ }
+ }
+}
diff --git a/MarketData/MarketDataLib/Generator/GainLoss/ActiveGainLossGenerator.cs b/MarketData/MarketDataLib/Generator/GainLoss/ActiveGainLossGenerator.cs
index 49fc148..5add690 100755
--- a/MarketData/MarketDataLib/Generator/GainLoss/ActiveGainLossGenerator.cs
+++ b/MarketData/MarketDataLib/Generator/GainLoss/ActiveGainLossGenerator.cs
@@ -22,9 +22,9 @@ namespace MarketData.Generator.GainLoss
public GainLossCollection GenerateGainLoss(PortfolioTrades portfolioTrades,DateTime? maxDateRef=null)
{
if (null == portfolioTrades || 0 == portfolioTrades.Count) return null;
- LocalPriceCache.GetInstance().Add(portfolioTrades);
+ GLPriceCache.GetInstance().Add(portfolioTrades);
DateTime minTradeDate = portfolioTrades.GetMinTradeDate();
- DateTime maxDate=LocalPriceCache.GetInstance().GetLatestDate();
+ DateTime maxDate=GLPriceCache.GetInstance().GetLatestDate();
if(null!=maxDateRef)maxDate=maxDateRef.Value;
Dictionary gainLoss = new Dictionary();
DateGenerator dateGenerator = new DateGenerator();
@@ -42,11 +42,11 @@ namespace MarketData.Generator.GainLoss
gainLoss.Add(holdingDate, new GainLossItem(holdingDate, 0,0,false));
continue;
}
- if(!LocalPriceCache.GetInstance().ContainsPrice(openTrades.Symbols,holdingDate))
+ if(!GLPriceCache.GetInstance().ContainsPrice(openTrades.Symbols,holdingDate))
{
if(holdingDate.Date.Equals(maxDate))
{
- LocalPriceCache.GetInstance().Add(openTrades.Symbols,holdingDate);
+ GLPriceCache.GetInstance().Add(openTrades.Symbols,holdingDate);
}else continue;
}
foreach (PortfolioTrade portfolioTrade in openTrades)
diff --git a/MarketData/MarketDataLib/Generator/GainLoss/GainLossGenerator.cs b/MarketData/MarketDataLib/Generator/GainLoss/GainLossGenerator.cs
index 695a93c..6e8e453 100755
--- a/MarketData/MarketDataLib/Generator/GainLoss/GainLossGenerator.cs
+++ b/MarketData/MarketDataLib/Generator/GainLoss/GainLossGenerator.cs
@@ -22,9 +22,9 @@ namespace MarketData.Generator.GainLoss
public TotalGainLossCollection GenerateTotalGainLoss(PortfolioTrades portfolioTrades,DateTime? maxDateRef=null)
{
if (null == portfolioTrades || 0 == portfolioTrades.Count) return null;
- LocalPriceCache.GetInstance().Add(portfolioTrades);
+ GLPriceCache.GetInstance().Add(portfolioTrades);
DateTime minTradeDate = portfolioTrades.GetMinTradeDate();
- DateTime maxDate=LocalPriceCache.GetInstance().GetLatestDate();
+ DateTime maxDate=GLPriceCache.GetInstance().GetLatestDate();
if(null!=maxDateRef)maxDate=maxDateRef.Value;
Dictionary gainLossCollection = new Dictionary();
DateGenerator dateGenerator = new DateGenerator();
@@ -74,9 +74,9 @@ namespace MarketData.Generator.GainLoss
public TotalGainLossCollection GenerateTotalGainLossWithDividends(PortfolioTrades portfolioTrades,DividendPayments dividendPayments,DateTime? maxDateRef=null)
{
if (null == portfolioTrades || 0 == portfolioTrades.Count) return null;
- LocalPriceCache.GetInstance().Add(portfolioTrades);
+ GLPriceCache.GetInstance().Add(portfolioTrades);
DateTime minTradeDate = portfolioTrades.GetMinTradeDate();
- DateTime maxDate=LocalPriceCache.GetInstance().GetLatestDate();
+ DateTime maxDate=GLPriceCache.GetInstance().GetLatestDate();
if(null!=maxDateRef)maxDate=maxDateRef.Value;
Dictionary gainLossCollection = new Dictionary();
DateGenerator dateGenerator = new DateGenerator();
diff --git a/MarketData/MarketDataLib/Generator/GainLoss/GainLossGeneratorCum.cs b/MarketData/MarketDataLib/Generator/GainLoss/GainLossGeneratorCum.cs
index 27cad84..2df123e 100755
--- a/MarketData/MarketDataLib/Generator/GainLoss/GainLossGeneratorCum.cs
+++ b/MarketData/MarketDataLib/Generator/GainLoss/GainLossGeneratorCum.cs
@@ -19,12 +19,12 @@ namespace MarketData.Generator.GainLoss
DateGenerator dateGenerator=new DateGenerator();
ModelPerformanceSeries performanceSeries=new ModelPerformanceSeries();
List gainLossList=new List();
- LocalPriceCache.GetInstance().Add(portfolioTrades);
+ GLPriceCache.GetInstance().Add(portfolioTrades);
try
{
if(!ValidatePortfolioTrades(portfolioTrades))return null;
DateTime minDate=portfolioTrades.GetMinTradeDate();
- DateTime maxDate = LocalPriceCache.GetInstance().GetLatestDate();
+ DateTime maxDate = GLPriceCache.GetInstance().GetLatestDate();
if(null!=maxDateRef) maxDate=maxDateRef.Value;
double prevGainLoss=double.NaN;
List historicalDates=dateGenerator.GenerateHistoricalDates(minDate,maxDate);
@@ -46,7 +46,7 @@ namespace MarketData.Generator.GainLoss
foreach(PortfolioTrade openPosition in openPositions)
{
exposure+=openPosition.Shares*openPosition.Price;
- Price price=LocalPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate);
+ Price price=GLPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",openPosition.Symbol,currentDate.ToShortDateString()));
@@ -98,12 +98,12 @@ namespace MarketData.Generator.GainLoss
DateGenerator dateGenerator=new DateGenerator();
ModelPerformanceSeries performanceSeries=new ModelPerformanceSeries();
List gainLossList=new List();
- LocalPriceCache.GetInstance().Add(portfolioTrades);
+ GLPriceCache.GetInstance().Add(portfolioTrades);
try
{
if(!ValidatePortfolioTrades(portfolioTrades)) return null;
DateTime minDate=portfolioTrades.Min(x => x.TradeDate);
- DateTime maxDate = LocalPriceCache.GetInstance().GetLatestDate();
+ DateTime maxDate = GLPriceCache.GetInstance().GetLatestDate();
double prevGainLoss=double.NaN;
List historicalDates=dateGenerator.GenerateHistoricalDates(minDate,maxDate);
@@ -123,7 +123,7 @@ namespace MarketData.Generator.GainLoss
foreach(PortfolioTrade openPosition in openPositions)
{
exposure+=openPosition.Shares*openPosition.Price;
- Price price=LocalPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate);
+ Price price=GLPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",openPosition.Symbol,currentDate.ToShortDateString()));
diff --git a/MarketData/MarketDataLib/Generator/GainLoss/GainLossHelper.cs b/MarketData/MarketDataLib/Generator/GainLoss/GainLossHelper.cs
index baf7a6f..3513ea1 100755
--- a/MarketData/MarketDataLib/Generator/GainLoss/GainLossHelper.cs
+++ b/MarketData/MarketDataLib/Generator/GainLoss/GainLossHelper.cs
@@ -20,7 +20,7 @@ namespace MarketData.Generator.GainLoss
if(holdingDateholdingDate))
{
- Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
+ Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate)));
@@ -35,7 +35,7 @@ namespace MarketData.Generator.GainLoss
if(holdingDateholdingDate))
{
- Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
+ Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate)));
@@ -69,7 +69,7 @@ namespace MarketData.Generator.GainLoss
{
return (portfolioTrade.SellPrice*portfolioTrade.Shares)-(portfolioTrade.Price*portfolioTrade.Shares);
}
- Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
+ Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate)));
@@ -83,7 +83,7 @@ namespace MarketData.Generator.GainLoss
if(!portfolioTrade.SellDate.Equals(Utility.Epoch)&&holdingDate>portfolioTrade.SellDate) return null;
// check to see if we bought and sold on the same date.
if(portfolioTrade.SellDate.Equals(portfolioTrade.TradeDate)) return (portfolioTrade.SellPrice*portfolioTrade.Shares);
- Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
+ Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate)));