Sean/ARM64
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Sean
2025-03-25 21:42:32 -04:00
parent c266eecfeb
commit 30c33d3cfd
247 changed files with 60107 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
MarketData/MarketDataLib/Cache/DividendHistoryCache.cs Executable file
View File

@@ -0,0 +1,148 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData;
using MarketData.Utils;
using MarketData.DataAccess;
namespace MarketData.Cache
{
// cache by symbol + div_ex_date
public class DividendHistoryByDixExYear:Dictionary<int,DividendHistory>
{
private int maxYear=int.MinValue;
private int minYear=int.MaxValue;
public DividendHistoryByDixExYear()
{
}
public new void Add(int key,DividendHistory dividendHistory)
{
base.Add(key,dividendHistory);
if(key>maxYear) maxYear=key;
if(key<minYear) minYear=key;
}
public int MaxYear
{
get { return maxYear; }
}
public int MinYear
{
get { return minYear; }
}
}
public class DividendHistoryCache
{
private Dictionary<String,DividendHistoryByDixExYear> dividendHistoryCache=new Dictionary<String,DividendHistoryByDixExYear>();
private static DividendHistoryCache instance=null;
private Object thisLock=new Object();
private DividendHistoryCache()
{
}
public void Dispose()
{
lock(thisLock)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[DividendHistoryCache:Dispose]Joined...");
instance=null;
}
}
public static DividendHistoryCache GetInstance()
{
lock(typeof(DividendHistoryCache))
{
if(null==instance) instance=new DividendHistoryCache();
return instance;
}
}
public void ClearCache()
{
lock(typeof(DividendHistoryCache))
{
dividendHistoryCache.Clear();
}
}
public void Add(DividendHistory dividendHistory)
{
lock(typeof(DividendHistoryCache))
{
int[] divExYears=dividendHistory.Select(x=>x.DivExDate.Year).Distinct().ToArray<int>();
String[] symbols=dividendHistory.Select(x=>x.Symbol).Distinct().ToArray<String>();
foreach(String symbol in symbols)
{
foreach(int divExYear in divExYears)
{
DividendHistory historyList=new DividendHistory(dividendHistory.Where(x => x.DivExDate.Year.Equals(divExYear)&&x.Symbol.Equals(symbol)).ToList());
Add(symbol,divExYear,historyList);
}
}
}
}
private void Add(String symbol,int divExYear,DividendHistory dividendHistory)
{
lock(typeof(DividendHistoryCache))
{
if(!dividendHistoryCache.ContainsKey(symbol))
{
DividendHistoryByDixExYear dividendHistoryByDivExYear=new DividendHistoryByDixExYear();
dividendHistoryByDivExYear.Add(divExYear,dividendHistory);
dividendHistoryCache.Add(symbol,dividendHistoryByDivExYear);
}
else
{
DividendHistoryByDixExYear dividendHistoryByDixExYear=dividendHistoryCache[symbol];
if(!dividendHistoryByDixExYear.ContainsKey(divExYear))
{
dividendHistoryByDixExYear.Add(divExYear,dividendHistory);
}
}
}
}
public DividendHistory GetDividendHistory(String symbol, int[] divExYears)
{
DividendHistory dividendHistory=new DividendHistory();
foreach(int divExYear in divExYears)
{
if(!DividendHistoryCache.GetInstance().ContainsDividendHistory(symbol,divExYear))
{
DividendHistory dividendHistoryList=DividendHistoryDA.GetDividendHistory(symbol,new int[] { divExYear });
if(0==dividendHistoryList.Count) DividendHistoryCache.GetInstance().Add(symbol,divExYear,dividendHistoryList);
else DividendHistoryCache.GetInstance().Add(dividendHistoryList);
dividendHistory.AddRange(dividendHistoryList);
}
else
{
dividendHistory.AddRange(DividendHistoryCache.GetInstance().GetDividendHistory(symbol,divExYear));
}
}
return dividendHistory;
}
private DividendHistory GetDividendHistory(String symbol,int divExYear)
{
lock(typeof(DividendHistoryCache))
{
if(!dividendHistoryCache.ContainsKey(symbol)) return null;
DividendHistoryByDixExYear dividendHistoryByDixExYear=dividendHistoryCache[symbol];
if(!dividendHistoryByDixExYear.ContainsKey(divExYear)) return null;
return dividendHistoryByDixExYear[divExYear];
}
}
public bool ContainsDividendHistory(String symbol,int divExYear)
{
lock(typeof(DividendHistoryCache))
{
if(!dividendHistoryCache.ContainsKey(symbol)) return false;
DividendHistoryByDixExYear dividendHistoryByDixExYear=dividendHistoryCache[symbol];
if(!dividendHistoryByDixExYear.ContainsKey(divExYear)) return false;
return true;
}
}
}
}

259
MarketData/MarketDataLib/Cache/GBPriceCache.cs Executable file
View File

@@ -0,0 +1,259 @@
using System;
using System.Collections.Generic;
using System.Text;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
using System.Linq;
using MarketData.Helper;
using MarketData.Numerical;
using System.Threading;
// This cache is mainly used by the models. It is a short lived cache that gets cleared out every 2 minutes.
// This cache will attempt to load a price from the database if it is found in the cache.
namespace MarketData.Cache
{
public class GBPriceCache
{
private Thread cacheMonitorThread=null;
private volatile bool threadRun=true;
private Object thisLock=new Object();
private Dictionary<String,PricesByDate> priceCache=new Dictionary<String,PricesByDate>(); // the main cache
private Dictionary<String,Price> realTimePriceCache=new Dictionary<String,Price>(); // short lived cache of realtime prices gets cleared out every cacheRefreshAfter(ms)
private Dictionary<String,bool> nullCache=new Dictionary<String,bool>();
private DateGenerator dateGenerator=new DateGenerator();
private static GBPriceCache priceCacheInstance=null;
private int cacheRefreshAfter=120000; // the cache will be cleaned up after 2 minutes
protected GBPriceCache()
{
cacheMonitorThread=new Thread(new ThreadStart(ThreadProc));
cacheMonitorThread.Start();
}
public static GBPriceCache GetInstance()
{
lock(typeof(GBPriceCache))
{
if(null==priceCacheInstance)
{
priceCacheInstance=new GBPriceCache();
}
return priceCacheInstance;
}
}
public void Clear()
{
lock(thisLock)
{
priceCache=new Dictionary<String,PricesByDate>();
realTimePriceCache=new Dictionary<String,Price>();
nullCache=new Dictionary<String,bool>();
}
}
public void Dispose()
{
lock(thisLock)
{
if(null==priceCacheInstance || false==threadRun)return;
threadRun=false;
if(null!=cacheMonitorThread)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GBPriceCache:Dispose]Thread state is '{0}'. Joining main thread...",Utility.ThreadStateToString(cacheMonitorThread)));
cacheMonitorThread.Join(5000);
this.cacheMonitorThread=null;
}
MDTrace.WriteLine(LogLevel.DEBUG,"[GBPriceCache:Dispose] End.");
priceCacheInstance=null;
}
}
public void ClearCacheOnOrBefore(DateTime onOrBeforeDate,bool collect=false)
{
lock(thisLock)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Clearing GBPriceCache cache.");
List<String> symbols=new List<String>(priceCache.Keys);
foreach(String symbol in symbols)
{
PricesByDate pricesByDate=priceCache[symbol];
List<DateTime> symbolDates=new List<DateTime>(pricesByDate.Keys);
foreach(DateTime symbolDate in symbolDates)
{
if(symbolDate<onOrBeforeDate) pricesByDate.Remove(symbolDate);
}
}
MDTrace.WriteLine(LogLevel.DEBUG,"Calling garbage collector...");
if(collect) GC.Collect();
}
}
public Price GetPriceOrLatestAvailable(String symbol,DateTime date)
{
lock(thisLock)
{
Price price=GetPrice(symbol,date);
if(null!=price) return price;
DateTime latestPricingDate=PricingDA.GetLatestDateOnOrBefore(symbol,date);
price=GetPrice(symbol,latestPricingDate);
if(null!=price) return price;
price=PricingDA.GetPrice(symbol,latestPricingDate);
if(null!=price) AddPrice(price);
return price;
}
}
public Price GetRealtimePrice(String symbol)
{
if(realTimePriceCache.ContainsKey(symbol)) return realTimePriceCache[symbol];
Price price=MarketDataHelper.GetLatestPrice(symbol);
if(null!=price)
{
realTimePriceCache.Add(symbol,price);
}
return price;
}
public Price GetPrice(String symbol,DateTime date)
{
lock(thisLock)
{
date=date.Date;
if(!ContainsPrice(symbol,date))
{
String key=symbol+Utility.DateTimeToStringMMHDDHYYYY(date);
if(nullCache.ContainsKey(key)) return null;
Price price=PricingDA.GetPrice(symbol,date);
if(null==price)
{
nullCache.Add(key,true);
return price;
}
AddPrice(price);
}
if(!priceCache.ContainsKey(symbol)) return null;
PricesByDate pricesByDate=priceCache[symbol];
if(!pricesByDate.ContainsKey(date.Date)) return null;
return pricesByDate[date];
}
}
public Prices GetPrices(String symbol, DateTime earlierDate, DateTime laterDate)
{
DateGenerator dateGenerator = new DateGenerator();
if(laterDate<earlierDate)
{
DateTime tempDate = earlierDate;
earlierDate = laterDate;
laterDate=tempDate;
}
List<DateTime> datesList = dateGenerator.GenerateHistoricalDates(earlierDate, laterDate);
datesList = datesList.Where(x => x >= earlierDate).ToList();
return GetPrices(symbol, laterDate, datesList.Count);
}
// The most recent price is returned at the lowest index
public Prices GetPrices(String symbol,DateTime startDate,int dayCount)
{
lock(thisLock)
{
List<DateTime> historicalDates=dateGenerator.GenerateHistoricalDates(startDate,dayCount+60);
Prices prices=null;
List<DateTime> missingDates=null;
foreach(DateTime historicalDate in historicalDates)
{
if(!ContainsPrice(symbol,historicalDate))
{
String key=symbol+Utility.DateTimeToStringMMHDDHYYYY(historicalDate);
if(nullCache.ContainsKey(key)) continue;
if(null==missingDates)missingDates=new List<DateTime>();
missingDates.Add(historicalDate);
}
}
if(null!=missingDates)
{
DateTime minDate=(from DateTime date in missingDates select date).Min();
DateTime maxDate=(from DateTime date in missingDates select date).Max();
prices=PricingDA.GetPrices(symbol,maxDate,minDate);
foreach(Price price in prices) AddPrice(price);
prices=new Prices();
foreach(DateTime historicalDate in historicalDates)
{
if(!ContainsPrice(symbol,historicalDate))
{
String key=symbol+Utility.DateTimeToStringMMHDDHYYYY(historicalDate);
if(!nullCache.ContainsKey(key)) nullCache.Add(key,true);
}
else
{
if(!priceCache.ContainsKey(symbol)) continue;
PricesByDate pricesByDate=priceCache[symbol];
if(!pricesByDate.ContainsKey(historicalDate.Date)) continue;
prices.Add(pricesByDate[historicalDate]);
}
}
}
else
{
prices=new Prices();
foreach(DateTime historicalDate in historicalDates)
{
if(!priceCache.ContainsKey(symbol)) continue;
if(!priceCache[symbol].ContainsKey(historicalDate.Date))
{
continue;
}
prices.Add((priceCache[symbol])[historicalDate]);
}
}
return new Prices(prices.OrderByDescending(x => x.Date).ToList().Take(dayCount).ToList());
}
}
private void AddPrice(Price price)
{
lock(thisLock)
{
if(null==price) return;
PricesByDate pricesByDate=null;
if(!priceCache.ContainsKey(price.Symbol))
{
pricesByDate=new PricesByDate();
pricesByDate.Add(price.Date,price);
priceCache.Add(price.Symbol,pricesByDate);
}
else
{
pricesByDate=priceCache[price.Symbol];
if(pricesByDate.ContainsKey(price.Date.Date)) return;
pricesByDate.Add(price.Date.Date,price);
}
}
}
public bool ContainsPrice(String symbol,DateTime date)
{
if(!priceCache.ContainsKey(symbol)) return false;
PricesByDate pricesByDate=priceCache[symbol];
if(!pricesByDate.ContainsKey(date.Date)) return false;
return true;
}
private void ThreadProc()
{
int quantums=0;
int quantumInterval=1000;
while(threadRun)
{
Thread.Sleep(quantumInterval);
if(!threadRun) break;
quantums+=quantumInterval;
if(quantums>cacheRefreshAfter)
{
quantums=0;
lock(thisLock)
{
realTimePriceCache.Clear();
MDTrace.WriteLine(LogLevel.DEBUG,"Clearing GBPriceCache price cache.");
}
}
}
MDTrace.WriteLine(LogLevel.DEBUG,"[GBPriceCache:ThreadProc]Thread ended.");
}
}
}

306
MarketData/MarketDataLib/Cache/LocalPriceCache.cs Executable file
View File

@@ -0,0 +1,306 @@
using System;
using System.Linq;
using System.Collections.Generic;
using MarketData;
using MarketData.MarketDataModel;
using MarketData.Utils;
using MarketData.DataAccess;
using System.Threading;
// This cache is mainly used by gainloss generator. This cache is intended to be front loaded and then used.
// This cache will not attempt to load an item that is not found. It does have a Refresh() that will reload only the most recent pricing data from the database in order to
// maintain the most updated pricing.
namespace MarketData.Cache
{
public class LocalPriceCache
{
private Dictionary<String,PricesByDate> priceCache=new Dictionary<String,PricesByDate>();
private static LocalPriceCache instance=null;
private Thread cacheMonitorThread=null;
private volatile bool threadRun=true;
private int cacheCycle=300000;
// private int cacheRefreshAfter=60000;
private Object thisLock=new Object();
private LocalPriceCache()
{
cacheMonitorThread=new Thread(new ThreadStart(ThreadProc));
cacheMonitorThread.Start();
}
public void Clear()
{
lock(thisLock)
{
priceCache=new Dictionary<String,PricesByDate>();
}
}
public void Dispose()
{
lock(thisLock)
{
if(null==instance || false==threadRun)return;
threadRun=false;
if(null!=cacheMonitorThread)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LocalPriceCache:Dispose]Thread state is '{0}'. Joining main thread...",Utility.ThreadStateToString(cacheMonitorThread)));
cacheMonitorThread.Join(5000);
this.cacheMonitorThread=null;
}
MDTrace.WriteLine(LogLevel.DEBUG,"[LocalPriceCache:Dispose] End");
instance=null;
}
}
public static LocalPriceCache GetInstance()
{
lock(typeof(LocalPriceCache))
{
if(null==instance)instance=new LocalPriceCache();
return instance;
}
}
public void Refresh()
{
lock(typeof(LocalPriceCache))
{
List<String> symbols=new List<String>(priceCache.Keys);
foreach(String symbol in symbols)
{
PricesByDate symbolPrices=priceCache[symbol];
DateTime maxDateDb=PricingDA.GetLatestDate(symbol); // get the latest date in the database
DateTime maxDate=symbolPrices.MaxDate; // get the latest date in the cache
if(!maxDateDb.Date.Equals(maxDate.Date)) // if the cache date and the database date are not equal then reload the cache
{
Prices prices=PricingDA.GetPrices(symbol,symbolPrices.MinDate); // reload the prices for this symbol using the current minDate in the cache as a lower boundary
if(null==prices)continue; // if we can't load any prices for symbol then just continue
priceCache.Remove(symbol); // remove the pricing entries in the price cache for the symbol
priceCache.Add(symbol,prices.GetPricesByDate()); // reload the cache
}
else
{
Price price=PricingDA.GetPrice(symbol,maxDate); // the max date from the cache equals the max date from the database so just reload the latest price from the database
if(null==price)continue; // if no latest price then just continue
symbolPrices.Remove(maxDate); // remove the current price associated with the max date
symbolPrices.Add(maxDate,price); // reload the latest price for maxDate(symbol) into the cache
}
}
}
}
//public void Add(PortfolioTrades portfolioTrades)
//{
// lock(typeof(LocalPriceCache))
// {
// Profiler profiler=new Profiler();
// profiler.Start();
// List<String> symbols=portfolioTrades.Symbols;
// foreach(String symbol in symbols)
// {
// if(ContainsSymbol(symbol))continue;
// DateTime minDate=portfolioTrades.GetMinTradeDate(symbol);
// Prices prices=PricingDA.GetPrices(symbol,minDate);
// if(null==prices)continue;
// if(!priceCache.ContainsKey(symbol))priceCache.Add(symbol,prices.GetPricesByDate());
// }
// }
//}
// This version of Add(PortfolioTrades) will account for adding multiple lots at different times. So instead of just checking for the existance of the symbol in the cache
// we look to see if the symbol is in the cache and what dates are available. If the date range specified in the trade are not available then we load those date ranges.
// This is a brute force approach always maintaining the gap between successive TradeDates in th.e portfolio trades and the maximum date for the symbol in the database.
// So while it is inefficient in terms of memory usage it alleviates the need for figuring out contiguous price sections
public void Add(PortfolioTrades portfolioTrades)
{
lock(typeof(LocalPriceCache))
{
Profiler profiler=new Profiler();
profiler.Start();
List<String> symbols=portfolioTrades.Symbols;
foreach(String symbol in symbols)
{
DateTime minPortfolioTradeDate=portfolioTrades.GetMinTradeDate(symbol);
if(!ContainsSymbol(symbol))
{
Prices prices=PricingDA.GetPrices(symbol,minPortfolioTradeDate);
if(null==prices)continue;
foreach(Price price in prices)Add(price);
}
else
{
DateTime minCacheDate=GetMinCacheDate(symbol);
if(minPortfolioTradeDate<minCacheDate)
{
Prices prices=PricingDA.GetPrices(symbol,minCacheDate,minPortfolioTradeDate); // Fill the gap by retrieving prices starting at minCache date and going back in time to minPortfolioTradeDate
if(null==prices)continue;
foreach(Price price in prices)Add(price);
PricesByDate p=priceCache[symbol];
}
}
}
}
}
// This version tries to be more efficient by examing what prices are required for the particular lot and just loading those.
// The version above is very brute force and always maintains min-present for all symbols even if they are no longer held.
// So if this more streamlined version gives problems then revert back to the more brute force method.
//public void Add(PortfolioTrades portfolioTrades)
//{
// lock(typeof(LocalPriceCache))
// {
// Profiler profiler=new Profiler();
// profiler.Start();
// List<String> symbols=portfolioTrades.Symbols;
// foreach(String symbol in symbols)
// {
// DateTime minPortfolioTradeDate=portfolioTrades.GetMinTradeDate(symbol);
// DateTime maxPortfolioTradeDate=portfolioTrades.GetMaxTradeDate(symbol);
// if(!ContainsSymbol(symbol))
// {
// Prices prices=null;
// if(Utility.IsEpoch(maxPortfolioTradeDate))prices=PricingDA.GetPrices(symbol,minPortfolioTradeDate);
// else prices=PricingDA.GetPrices(symbol,maxPortfolioTradeDate,minPortfolioTradeDate);
// if(null==prices)continue;
// foreach(Price price in prices)Add(price);
// }
// else
// {
// DateTime minCacheDate=GetMinCacheDate(symbol);
// if(minPortfolioTradeDate<minCacheDate)
// {
// Prices prices=PricingDA.GetPrices(symbol,minCacheDate,minPortfolioTradeDate); // Fill the gap by retrieving prices starting at minCache date and going back in time to minPortfolioTradeDate
// if(null==prices)continue;
// foreach(Price price in prices)Add(price);
// }
// else // this is wrong because it reloads prices we already have. Is would be necessary to figure out the price gaps and then load only those items.
// {
// Prices prices=null;
// if(Utility.IsEpoch(maxPortfolioTradeDate))prices=PricingDA.GetPrices(symbol,minPortfolioTradeDate);
// else prices=PricingDA.GetPrices(symbol,maxPortfolioTradeDate,minPortfolioTradeDate);
// foreach(Price price in prices)Add(price);
// }
// }
// }
// }
//}
public void Add(Prices prices)
{
foreach(Price price in prices)
{
Add(price);
}
}
public void Add(List<String> symbols,DateTime pricingDate)
{
foreach(String symbol in symbols)
{
if(ContainsPrice(symbol,pricingDate))continue;
Price price=PricingDA.GetPrice(symbol,pricingDate);
if(null==price)continue;
Add(price);
}
}
public void Add(Price price)
{
lock(typeof(LocalPriceCache))
{
if(null==price)return;
if(ContainsPrice(price.Symbol,price.Date))return;
PricesByDate pricesByDate=null;
if(!priceCache.ContainsKey(price.Symbol))
{
pricesByDate=new PricesByDate();
pricesByDate.Add(price.Date,price);
priceCache.Add(price.Symbol,pricesByDate);
return;
}
pricesByDate=priceCache[price.Symbol];
if(pricesByDate.ContainsKey(price.Date))return;
pricesByDate.Add(price.Date,price);
}
}
public DateTime GetMinCacheDate(String symbol)
{
if(!ContainsSymbol(symbol))return Utility.Epoch;
PricesByDate symbolPrices=priceCache[symbol];
return symbolPrices.MinDate;
}
public void RemoveDate(DateTime date)
{
lock(typeof(LocalPriceCache))
{
List<String> symbols=new List<String>(priceCache.Keys);
foreach(String key in symbols)
{
PricesByDate pricesByDate=priceCache[key];
if(pricesByDate.ContainsKey(date))pricesByDate.Remove(date);
}
}
}
public Price GetPrice(String symbol,DateTime date)
{
lock(typeof(LocalPriceCache))
{
if(!priceCache.ContainsKey(symbol))return null;
PricesByDate pricesByDate=priceCache[symbol];
if(!pricesByDate.ContainsKey(date))return null;
return pricesByDate[date];
}
}
public bool ContainsPrice(String symbol,DateTime date)
{
lock(typeof(LocalPriceCache))
{
if(!priceCache.ContainsKey(symbol))return false;
PricesByDate pricesByDate=priceCache[symbol];
if(!pricesByDate.ContainsKey(date))return false;
return true;
}
}
public bool ContainsPrice(List<String> symbols,DateTime date)
{
lock(typeof(LocalPriceCache))
{
foreach(String symbol in symbols)if(!ContainsPrice(symbol,date))return false;
return true;
}
}
public bool ContainsSymbol(String symbol)
{
lock(typeof(LocalPriceCache))
{
if(priceCache.ContainsKey(symbol))return true;
return false;
}
}
public long Count()
{
long count=0;
List<String> symbols=priceCache.Keys.ToList();
foreach(String symbol in symbols)
{
PricesByDate pricesByDate=priceCache[symbol];
count+=pricesByDate.Count;
}
return count;
}
private void ThreadProc()
{
int quantums=0;
int quantumInterval=1000;
long lastCount=0;
while(threadRun)
{
Thread.Sleep(quantumInterval);
quantums+=quantumInterval;
if(quantums>cacheCycle)
{
quantums=0;
lock(thisLock)
{
lastCount=Count();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("LocalPriceCache Symbols: {0}. Items in cache: {1}.",priceCache.Keys.Count,Utility.FormatNumber(lastCount,0,true)));
}
}
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LocalPriceCache:ThreadProc]Thread ended. Items in cache:{0}",Utility.FormatNumber(lastCount,0,true)));
}
}
}