Create GLPriceCache to be used by GainLoss Generator.

MarketData/MarketDataLib/Cache/GLPriceCache.cs

@@ -0,0 +1,409 @@
+using MarketData.MarketDataModel;
+using MarketData.Utils;
+using MarketData.DataAccess;
+using System.Collections.Concurrent;
+using System.Threading;
+using System.Collections.Generic;
+using System;
+using System.Threading.Tasks;
+using System.Linq;
+
+namespace MarketData.Cache
+{
+  /// <summary>
+  /// This cache is used in the GainLoss Generators
+  /// </summary>
+  public class GLPriceCache
+  {
+    private Dictionary<string, PricesByDate> priceCache = new Dictionary<string, PricesByDate>();
+    private static GLPriceCache instance = null;
+    private DateTime latestDate = Utility.Epoch;
+    private Thread cacheMonitorThread = null;
+    private volatile bool threadRun = true;
+    private int cacheCycle = 300000;
+    private object thisLock = new object();
+    private object fetchLock = new object();
+
+    private GLPriceCache()
+    {
+      cacheMonitorThread = new Thread(new ThreadStart(ThreadProc));
+      cacheMonitorThread.Start();
+    }
+
+    public void Clear()
+    {
+      lock (thisLock)
+      {
+        priceCache = new Dictionary<string, PricesByDate>();
+        RefreshLatestDate();
+      }
+    }
+
+    public void Dispose()
+    {
+      Thread threadToJoin = null;
+
+      lock (thisLock)
+      {
+        if (instance == null || !threadRun) return;
+        threadRun = false;
+        threadToJoin = cacheMonitorThread;
+        cacheMonitorThread = null;
+        instance = null;
+      }
+
+      if (threadToJoin != null)
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG, $"[GLPriceCache:Dispose] Thread state is '{Utility.ThreadStateToString(threadToJoin)}'. Joining...");
+        threadToJoin.Join(5000);
+      }
+
+      MDTrace.WriteLine(LogLevel.DEBUG, "[GLPriceCache:Dispose] End");
+    }
+
+    public static GLPriceCache GetInstance()
+    {
+      lock (typeof(LocalPriceCache))
+      {
+        if (instance == null)
+        {
+          instance = new GLPriceCache();
+        }
+        return instance;
+      }
+    }
+
+    public void RefreshLatestDate()
+    {
+      lock (thisLock)
+      {
+        latestDate = PricingDA.GetLatestDate();
+      }
+    }
+
+    public DateTime GetLatestDate()
+    {
+      lock (thisLock)
+      {
+        if (Utility.IsEpoch(latestDate))
+        {
+          RefreshLatestDate();
+        }
+        return latestDate;
+      }
+    }
+
+    public void Refresh()
+    {
+      List<string> symbols;
+      Dictionary<string, DateTime> currentMaxDates;
+
+      lock (thisLock)
+      {
+        symbols = priceCache.Keys.ToList();
+        currentMaxDates = priceCache.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.MaxDate);
+      }
+
+      if (symbols.Count == 0) return;
+
+      ConcurrentDictionary<string, PricesByDate> fullReloads = new ConcurrentDictionary<string, PricesByDate>();
+      ConcurrentDictionary<string, Price> singleUpdates = new ConcurrentDictionary<string, Price>();
+      DateTime latestDateFromDb;
+
+      lock (fetchLock)
+      {
+        Dictionary<string, DateTime> maxDbDates = PricingDA.GetLatestDates(symbols);
+        latestDateFromDb = PricingDA.GetLatestDate();
+
+        Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol =>
+        {
+          DateTime cachedMax;
+          if (!currentMaxDates.TryGetValue(symbol, out cachedMax)) return;
+
+          DateTime dbMax;
+          if (maxDbDates.TryGetValue(symbol, out dbMax) && dbMax.Date != cachedMax.Date)
+          {
+            Prices prices = PricingDA.GetPrices(symbol, cachedMax);
+            if (prices != null) fullReloads[symbol] = prices.GetPricesByDate();
+          }
+          else
+          {
+            Price price = PricingDA.GetPrice(symbol, cachedMax);
+            if (price != null) singleUpdates[symbol] = price;
+          }
+        });
+      }
+
+      lock (thisLock)
+      {
+        latestDate = latestDateFromDb;
+
+        foreach (KeyValuePair<string, PricesByDate> kvp in fullReloads)
+        {
+          if (priceCache.TryGetValue(kvp.Key, out PricesByDate existing) && existing.MaxDate == currentMaxDates[kvp.Key])
+          {
+            priceCache[kvp.Key] = kvp.Value;
+          }
+        }
+
+        foreach (KeyValuePair<string, Price> kvp in singleUpdates)
+        {
+          if (priceCache.TryGetValue(kvp.Key, out PricesByDate pricesByDate) && pricesByDate.MaxDate == currentMaxDates[kvp.Key])
+          {
+            if (pricesByDate.ContainsKey(kvp.Value.Date))
+              pricesByDate.Remove(kvp.Value.Date);
+            pricesByDate.Add(kvp.Value.Date, kvp.Value);
+          }
+        }
+      }
+
+      MDTrace.WriteLine(LogLevel.DEBUG, $"Full reloads: {fullReloads.Count}, Single updates: {singleUpdates.Count}");
+    }
+
+    public void Add(PortfolioTrades portfolioTrades)
+    {
+      List<string> symbols = portfolioTrades.Symbols;
+      Dictionary<string, DateTime> minTradeDates = symbols.ToDictionary(sym => sym, sym => portfolioTrades.GetMinTradeDate(sym));
+
+      Dictionary<string, DateTime> minCacheDates;
+      lock (thisLock)
+      {
+        minCacheDates = symbols.ToDictionary(sym => sym, sym => priceCache.ContainsKey(sym) ? priceCache[sym].MinDate : DateTime.MaxValue);
+      }
+
+      ConcurrentDictionary<string, Prices> fetchedPrices = new ConcurrentDictionary<string, Prices>();
+
+      Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol =>
+      {
+        DateTime minTradeDate = minTradeDates[symbol];
+        DateTime minCacheDate = minCacheDates[symbol];
+        Prices prices = null;
+
+        try
+        {
+          if (minCacheDate == DateTime.MaxValue)
+          {
+            prices = PricingDA.GetPrices(symbol, minTradeDate);
+          }
+          else if (minTradeDate < minCacheDate)
+          {
+            prices = PricingDA.GetPrices(symbol, minCacheDate, minTradeDate);
+          }
+
+          if (prices != null && prices.Count > 0)
+          {
+            fetchedPrices[symbol] = prices;
+          }
+        }
+        catch (Exception ex)
+        {
+          MDTrace.WriteLine(LogLevel.DEBUG, $"Error fetching prices for {symbol}: {ex.Message}");
+        }
+      });
+
+      lock (thisLock)
+      {
+        foreach (KeyValuePair<string, Prices> kvp in fetchedPrices)
+        {
+          foreach (Price price in kvp.Value)
+          {
+            Add(price);
+          }
+        }
+      }
+    }
+
+    public void Add(Prices prices)
+    {
+      foreach (Price price in prices)
+      {
+        Add(price);
+      }
+    }
+
+    public void Add(List<string> symbols, DateTime pricingDate)
+    {
+      if (symbols == null || symbols.Count == 0) return;
+
+      ConcurrentDictionary<string, Price> fetchedPrices = new ConcurrentDictionary<string, Price>();
+
+      Parallel.ForEach(symbols, new ParallelOptions { MaxDegreeOfParallelism = 8 }, symbol =>
+      {
+        lock (thisLock)
+        {
+          if (ContainsPrice(symbol, pricingDate)) return;
+        }
+
+        try
+        {
+          Price price = PricingDA.GetPrice(symbol, pricingDate);
+          if (price != null) fetchedPrices[symbol] = price;
+        }
+        catch (Exception ex)
+        {
+          MDTrace.WriteLine(LogLevel.DEBUG, $"GLPriceCache: Error fetching price for {symbol} on {pricingDate:yyyy-MM-dd}: {ex.Message}");
+        }
+      });
+
+      lock (thisLock)
+      {
+        foreach (KeyValuePair<string, Price> kvp in fetchedPrices)
+        {
+          Add(kvp.Value);
+        }
+      }
+    }
+
+    public void Add(Price price)
+    {
+      if (price == null) return;
+
+      lock (thisLock)
+      {
+        PricesByDate pricesByDate;
+        if (!priceCache.TryGetValue(price.Symbol, out pricesByDate))
+        {
+          pricesByDate = new PricesByDate();
+          priceCache[price.Symbol] = pricesByDate;
+        }
+        if (!pricesByDate.ContainsKey(price.Date))
+        {
+          pricesByDate.Add(price.Date, price);
+        }
+      }
+    }
+
+    public DateTime GetMinCacheDate(string symbol)
+    {
+      lock (thisLock)
+      {
+        PricesByDate symbolPrices;
+        if (!priceCache.TryGetValue(symbol, out symbolPrices) || symbolPrices.Count == 0)
+        {
+          return Utility.Epoch;
+        }
+        return symbolPrices.MinDate;
+      }
+    }
+
+    public void RemoveDate(DateTime date)
+    {
+      lock (thisLock)
+      {
+        foreach (KeyValuePair<string, PricesByDate> kvp in priceCache)
+        {
+          kvp.Value.Remove(date);
+        }
+      }
+    }
+
+    public Prices GetPrices(string symbol, DateTime endDate, int dayCount)
+    {
+      lock (thisLock)
+      {
+        PricesByDate pricesByDate;
+        if (!priceCache.TryGetValue(symbol, out pricesByDate)) return new Prices();
+
+        DateGenerator dateGenerator = new DateGenerator();
+        List<DateTime> historicalDates = dateGenerator.GenerateHistoricalDates(endDate, dayCount);
+
+        Prices result = new Prices();
+        foreach (DateTime date in historicalDates)
+        {
+          if (pricesByDate.ContainsKey(date))
+          {
+            result.Add(pricesByDate[date]);
+          }
+        }
+
+        return result;
+      }
+    }
+
+    public Price GetPrice(string symbol, DateTime date)
+    {
+      lock (thisLock)
+      {
+        PricesByDate pricesByDate;
+        if (!priceCache.TryGetValue(symbol, out pricesByDate)) return null;
+
+        Price price;
+        return pricesByDate.TryGetValue(date, out price) ? price : null;
+      }
+    }
+
+    public bool ContainsPrice(string symbol, DateTime date)
+    {
+      lock (thisLock)
+      {
+        PricesByDate pricesByDate;
+        if (!priceCache.TryGetValue(symbol, out pricesByDate)) return false;
+        return pricesByDate.ContainsKey(date);
+      }
+    }
+
+    public bool ContainsPrice(List<string> symbols, DateTime date)
+    {
+      if (symbols == null || symbols.Count == 0) return false;
+
+      lock (thisLock)
+      {
+        foreach (string symbol in symbols)
+        {
+          PricesByDate pricesByDate;
+          if (!priceCache.TryGetValue(symbol, out pricesByDate) || !pricesByDate.ContainsKey(date))
+          {
+            return false;
+          }
+        }
+        return true;
+      }
+    }
+
+    public bool ContainsSymbol(string symbol)
+    {
+      lock (thisLock)
+      {
+        return priceCache.ContainsKey(symbol);
+      }
+    }
+
+    private long Count()
+    {
+      lock (thisLock)
+      {
+        long count = 0;
+        foreach (PricesByDate pricesByDate in priceCache.Values)
+        {
+          count += pricesByDate.Count;
+        }
+        return count;
+      }
+    }
+
+    private void ThreadProc()
+    {
+      int quantums = 0;
+      int quantumInterval = 1000;
+      long lastCount = 0;
+
+      while (threadRun)
+      {
+        Thread.Sleep(quantumInterval);
+        if(!threadRun)break;
+        quantums += quantumInterval;
+        if (quantums > cacheCycle)
+        {
+          quantums = 0;
+          lock (thisLock)
+          {
+            lastCount = Count();
+            MDTrace.WriteLine(LogLevel.DEBUG, $"[GLPriceCache:ThreadProc] Symbols: {priceCache.Keys.Count}. Items in cache: {Utility.FormatNumber(lastCount,0,true)}.");
+          }
+        }
+      }
+
+      MDTrace.WriteLine(LogLevel.DEBUG, $"[GLPriceCache:ThreadProc] Thread ended. Items in cache:{Utility.FormatNumber(lastCount,0,true)}");
+    }
+  }
+}

MarketData/MarketDataLib/Generator/GainLoss/ActiveGainLossGenerator.cs

@@ -22,9 +22,9 @@ namespace MarketData.Generator.GainLoss
     public GainLossCollection GenerateGainLoss(PortfolioTrades portfolioTrades,DateTime? maxDateRef=null)
     {
       if (null == portfolioTrades || 0 == portfolioTrades.Count) return null;
-      LocalPriceCache.GetInstance().Add(portfolioTrades);
+      GLPriceCache.GetInstance().Add(portfolioTrades);
       DateTime minTradeDate = portfolioTrades.GetMinTradeDate();
-      DateTime maxDate=LocalPriceCache.GetInstance().GetLatestDate();
+      DateTime maxDate=GLPriceCache.GetInstance().GetLatestDate();
       if(null!=maxDateRef)maxDate=maxDateRef.Value;
       Dictionary<DateTime,GainLossItem> gainLoss = new Dictionary<DateTime, GainLossItem>();
       DateGenerator dateGenerator = new DateGenerator();
@@ -42,11 +42,11 @@ namespace MarketData.Generator.GainLoss
           gainLoss.Add(holdingDate, new GainLossItem(holdingDate, 0,0,false));
           continue;
         }
-        if(!LocalPriceCache.GetInstance().ContainsPrice(openTrades.Symbols,holdingDate))
+        if(!GLPriceCache.GetInstance().ContainsPrice(openTrades.Symbols,holdingDate))
 				{
           if(holdingDate.Date.Equals(maxDate))
           {
-            LocalPriceCache.GetInstance().Add(openTrades.Symbols,holdingDate);
+            GLPriceCache.GetInstance().Add(openTrades.Symbols,holdingDate);
           }else continue;
 				}
         foreach (PortfolioTrade portfolioTrade in openTrades)

MarketData/MarketDataLib/Generator/GainLoss/GainLossGenerator.cs

@@ -22,9 +22,9 @@ namespace MarketData.Generator.GainLoss
     public TotalGainLossCollection GenerateTotalGainLoss(PortfolioTrades portfolioTrades,DateTime? maxDateRef=null)
     {
       if (null == portfolioTrades || 0 == portfolioTrades.Count) return null;
-      LocalPriceCache.GetInstance().Add(portfolioTrades);
+      GLPriceCache.GetInstance().Add(portfolioTrades);
       DateTime minTradeDate = portfolioTrades.GetMinTradeDate();
-      DateTime maxDate=LocalPriceCache.GetInstance().GetLatestDate();
+      DateTime maxDate=GLPriceCache.GetInstance().GetLatestDate();
       if(null!=maxDateRef)maxDate=maxDateRef.Value;
       Dictionary<DateTime,TotalGainLossItem> gainLossCollection = new Dictionary<DateTime, TotalGainLossItem>();
       DateGenerator dateGenerator = new DateGenerator();
@@ -74,9 +74,9 @@ namespace MarketData.Generator.GainLoss
     public TotalGainLossCollection GenerateTotalGainLossWithDividends(PortfolioTrades portfolioTrades,DividendPayments dividendPayments,DateTime? maxDateRef=null)
     {
       if (null == portfolioTrades || 0 == portfolioTrades.Count) return null;
-      LocalPriceCache.GetInstance().Add(portfolioTrades);
+      GLPriceCache.GetInstance().Add(portfolioTrades);
       DateTime minTradeDate = portfolioTrades.GetMinTradeDate();
-      DateTime maxDate=LocalPriceCache.GetInstance().GetLatestDate();
+      DateTime maxDate=GLPriceCache.GetInstance().GetLatestDate();
       if(null!=maxDateRef)maxDate=maxDateRef.Value;
       Dictionary<DateTime,TotalGainLossItem> gainLossCollection = new Dictionary<DateTime, TotalGainLossItem>();
       DateGenerator dateGenerator = new DateGenerator();

MarketData/MarketDataLib/Generator/GainLoss/GainLossGeneratorCum.cs

@@ -19,12 +19,12 @@ namespace MarketData.Generator.GainLoss
       DateGenerator dateGenerator=new DateGenerator();
       ModelPerformanceSeries performanceSeries=new ModelPerformanceSeries();
       List<TotalGainLossItem> gainLossList=new List<TotalGainLossItem>();
-      LocalPriceCache.GetInstance().Add(portfolioTrades);
+      GLPriceCache.GetInstance().Add(portfolioTrades);
       try
       {
         if(!ValidatePortfolioTrades(portfolioTrades))return null;
         DateTime minDate=portfolioTrades.GetMinTradeDate();
-        DateTime maxDate = LocalPriceCache.GetInstance().GetLatestDate();
+        DateTime maxDate = GLPriceCache.GetInstance().GetLatestDate();
         if(null!=maxDateRef) maxDate=maxDateRef.Value;
         double prevGainLoss=double.NaN;
         List<DateTime> historicalDates=dateGenerator.GenerateHistoricalDates(minDate,maxDate);
@@ -46,7 +46,7 @@ namespace MarketData.Generator.GainLoss
           foreach(PortfolioTrade openPosition in openPositions)
           {
             exposure+=openPosition.Shares*openPosition.Price;
-            Price price=LocalPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate);
+            Price price=GLPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate);
             if(null==price)
             {
               MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",openPosition.Symbol,currentDate.ToShortDateString()));
@@ -98,12 +98,12 @@ namespace MarketData.Generator.GainLoss
       DateGenerator dateGenerator=new DateGenerator();
       ModelPerformanceSeries performanceSeries=new ModelPerformanceSeries();
       List<TotalGainLossItem> gainLossList=new List<TotalGainLossItem>();
-      LocalPriceCache.GetInstance().Add(portfolioTrades);
+      GLPriceCache.GetInstance().Add(portfolioTrades);
       try
       {
         if(!ValidatePortfolioTrades(portfolioTrades)) return null;
         DateTime minDate=portfolioTrades.Min(x => x.TradeDate);
-        DateTime maxDate = LocalPriceCache.GetInstance().GetLatestDate();
+        DateTime maxDate = GLPriceCache.GetInstance().GetLatestDate();
         double prevGainLoss=double.NaN;
         List<DateTime> historicalDates=dateGenerator.GenerateHistoricalDates(minDate,maxDate);
 
@@ -123,7 +123,7 @@ namespace MarketData.Generator.GainLoss
           foreach(PortfolioTrade openPosition in openPositions)
           {
             exposure+=openPosition.Shares*openPosition.Price;
-            Price price=LocalPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate);
+            Price price=GLPriceCache.GetInstance().GetPrice(openPosition.Symbol,currentDate);
             if(null==price)
             {
               MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",openPosition.Symbol,currentDate.ToShortDateString()));

MarketData/MarketDataLib/Generator/GainLoss/GainLossHelper.cs

@@ -20,7 +20,7 @@ namespace MarketData.Generator.GainLoss
       if(holdingDate<portfolioTrade.TradeDate) return null;
       if(portfolioTrade.IsOpen||(portfolioTrade.IsClosed&&portfolioTrade.SellDate>holdingDate))
       {
-        Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
+        Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
         if(null==price)
         {
           MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate)));
@@ -35,7 +35,7 @@ namespace MarketData.Generator.GainLoss
       if(holdingDate<portfolioTrade.TradeDate) return null;
       if(portfolioTrade.IsOpen||(portfolioTrade.IsClosed&&portfolioTrade.SellDate>holdingDate))
       {
-        Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
+        Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
         if(null==price)
         {
           MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate)));
@@ -69,7 +69,7 @@ namespace MarketData.Generator.GainLoss
       {
         return (portfolioTrade.SellPrice*portfolioTrade.Shares)-(portfolioTrade.Price*portfolioTrade.Shares);
       }
-      Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
+      Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
       if(null==price)
       {
         MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate)));
@@ -83,7 +83,7 @@ namespace MarketData.Generator.GainLoss
       if(!portfolioTrade.SellDate.Equals(Utility.Epoch)&&holdingDate>portfolioTrade.SellDate) return null;
       // check to see if we bought and sold on the same date.
       if(portfolioTrade.SellDate.Equals(portfolioTrade.TradeDate)) return (portfolioTrade.SellPrice*portfolioTrade.Shares);
-      Price price=LocalPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
+      Price price=GLPriceCache.GetInstance().GetPrice(portfolioTrade.Symbol,holdingDate);
       if(null==price)
       {
         MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No price for {0} on {1}",portfolioTrade.Symbol,Utility.DateTimeToStringMMHDDHYYYY(holdingDate)));