Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize the LocalPriceCache

Browse Source
This commit is contained in:
Sean
2025-04-04 16:37:29 -04:00
parent d311da99c5
commit a213dec5d5
2 changed files with 95 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
MarketDataLib/Cache/LocalPriceCache.cs
View File

@@ -1,7 +1,6 @@
using System;
using System.Linq;
using System.Collections.Generic;
using MarketData;
using MarketData.MarketDataModel;
using MarketData.Utils;
using MarketData.DataAccess;
@@ -19,7 +18,6 @@ namespace MarketData.Cache
private Thread cacheMonitorThread=null;
private volatile bool threadRun=true;
private int cacheCycle=300000;
// private int cacheRefreshAfter=60000;
private Object thisLock=new Object();
private LocalPriceCache()
@@ -27,6 +25,7 @@ namespace MarketData.Cache
cacheMonitorThread=new Thread(new ThreadStart(ThreadProc));
cacheMonitorThread.Start();
}
public void Clear()
{
lock(thisLock)
@@ -34,6 +33,7 @@ namespace MarketData.Cache
priceCache=new Dictionary<String,PricesByDate>();
}
}
public void Dispose()
{
lock(thisLock)
@@ -50,6 +50,7 @@ namespace MarketData.Cache
instance=null;
}
}
public static LocalPriceCache GetInstance()
{
lock(typeof(LocalPriceCache))
@@ -64,13 +65,15 @@ namespace MarketData.Cache
lock(typeof(LocalPriceCache))
{
List<String> symbols=new List<String>(priceCache.Keys);
Dictionary<String, DateTime> maxDbDates = PricingDA.GetLatestDates(symbols);
foreach(String symbol in symbols)
{
PricesByDate symbolPrices=priceCache[symbol];
DateTime maxDateDb=PricingDA.GetLatestDate(symbol); // get the latest date in the database
DateTime maxDate=symbolPrices.MaxDate; // get the latest date in the cache
if(!maxDateDb.Date.Equals(maxDate.Date)) // if the cache date and the database date are not equal then reload the cache
if(maxDbDates.ContainsKey(symbol) && !maxDbDates[symbol].Date.Equals(maxDate.Date)) // if the cache date and the database date are not equal then reload the cache
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Cache date and Database date for {symbol} are not equal, reloading cache. Cache Date:{maxDate.ToShortDateString()} Database Date:{maxDbDates[symbol].Date.ToShortDateString()}");
Prices prices=PricingDA.GetPrices(symbol,symbolPrices.MinDate); // reload the prices for this symbol using the current minDate in the cache as a lower boundary
if(null==prices)continue; // if we can't load any prices for symbol then just continue
priceCache.Remove(symbol); // remove the pricing entries in the price cache for the symbol
@@ -78,6 +81,7 @@ namespace MarketData.Cache
}
else
{
MDTrace.WriteLine(LogLevel.DEBUG,$"[LocalPriceCache] Fetching latest price from database for {symbol} on {maxDate.ToShortDateString()}");
Price price=PricingDA.GetPrice(symbol,maxDate); // the max date from the cache equals the max date from the database so just reload the latest price from the database
if(null==price)continue; // if no latest price then just continue
symbolPrices.Remove(maxDate); // remove the current price associated with the max date
@@ -86,23 +90,7 @@ namespace MarketData.Cache
}
}
}
//public void Add(PortfolioTrades portfolioTrades)
//{
// lock(typeof(LocalPriceCache))
// {
// Profiler profiler=new Profiler();
// profiler.Start();
// List<String> symbols=portfolioTrades.Symbols;
// foreach(String symbol in symbols)
// {
// if(ContainsSymbol(symbol))continue;
// DateTime minDate=portfolioTrades.GetMinTradeDate(symbol);
// Prices prices=PricingDA.GetPrices(symbol,minDate);
// if(null==prices)continue;
// if(!priceCache.ContainsKey(symbol))priceCache.Add(symbol,prices.GetPricesByDate());
// }
// }
//}
// This version of Add(PortfolioTrades) will account for adding multiple lots at different times. So instead of just checking for the existance of the symbol in the cache
// we look to see if the symbol is in the cache and what dates are available. If the date range specified in the trade are not available then we load those date ranges.
// This is a brute force approach always maintaining the gap between successive TradeDates in th.e portfolio trades and the maximum date for the symbol in the database.
@@ -137,48 +125,6 @@ namespace MarketData.Cache
}
}
}
// This version tries to be more efficient by examing what prices are required for the particular lot and just loading those.
// The version above is very brute force and always maintains min-present for all symbols even if they are no longer held.
// So if this more streamlined version gives problems then revert back to the more brute force method.
//public void Add(PortfolioTrades portfolioTrades)
//{
// lock(typeof(LocalPriceCache))
// {
// Profiler profiler=new Profiler();
// profiler.Start();
// List<String> symbols=portfolioTrades.Symbols;
// foreach(String symbol in symbols)
// {
// DateTime minPortfolioTradeDate=portfolioTrades.GetMinTradeDate(symbol);
// DateTime maxPortfolioTradeDate=portfolioTrades.GetMaxTradeDate(symbol);
// if(!ContainsSymbol(symbol))
// {
// Prices prices=null;
// if(Utility.IsEpoch(maxPortfolioTradeDate))prices=PricingDA.GetPrices(symbol,minPortfolioTradeDate);
// else prices=PricingDA.GetPrices(symbol,maxPortfolioTradeDate,minPortfolioTradeDate);
// if(null==prices)continue;
// foreach(Price price in prices)Add(price);
// }
// else
// {
// DateTime minCacheDate=GetMinCacheDate(symbol);
// if(minPortfolioTradeDate<minCacheDate)
// {
// Prices prices=PricingDA.GetPrices(symbol,minCacheDate,minPortfolioTradeDate); // Fill the gap by retrieving prices starting at minCache date and going back in time to minPortfolioTradeDate
// if(null==prices)continue;
// foreach(Price price in prices)Add(price);
// }
// else // this is wrong because it reloads prices we already have. Is would be necessary to figure out the price gaps and then load only those items.
// {
// Prices prices=null;
// if(Utility.IsEpoch(maxPortfolioTradeDate))prices=PricingDA.GetPrices(symbol,minPortfolioTradeDate);
// else prices=PricingDA.GetPrices(symbol,maxPortfolioTradeDate,minPortfolioTradeDate);
// foreach(Price price in prices)Add(price);
// }
// }
// }
// }
//}
public void Add(Prices prices)
{
@@ -187,6 +133,7 @@ namespace MarketData.Cache
Add(price);
}
}
public void Add(List<String> symbols,DateTime pricingDate)
{
foreach(String symbol in symbols)
@@ -197,6 +144,7 @@ namespace MarketData.Cache
Add(price);
}
}
public void Add(Price price)
{
lock(typeof(LocalPriceCache))
@@ -222,6 +170,7 @@ namespace MarketData.Cache
PricesByDate symbolPrices=priceCache[symbol];
return symbolPrices.MinDate;
}
public void RemoveDate(DateTime date)
{
lock(typeof(LocalPriceCache))
@@ -262,6 +211,7 @@ namespace MarketData.Cache
return true;
}
}
public bool ContainsSymbol(String symbol)
{
lock(typeof(LocalPriceCache))
@@ -270,6 +220,7 @@ namespace MarketData.Cache
return false;
}
}
public long Count()
{
long count=0;
@@ -281,6 +232,7 @@ namespace MarketData.Cache
}
return count;
}
private void ThreadProc()
{
int quantums=0;
@@ -296,11 +248,11 @@ namespace MarketData.Cache
lock(thisLock)
{
lastCount=Count();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("LocalPriceCache Symbols: {0}. Items in cache: {1}.",priceCache.Keys.Count,Utility.FormatNumber(lastCount,0,true)));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LocalPriceCache:ThreadProc] Symbols: {0}. Items in cache: {1}.",priceCache.Keys.Count,Utility.FormatNumber(lastCount,0,true)));
}
}
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LocalPriceCache:ThreadProc]Thread ended. Items in cache:{0}",Utility.FormatNumber(lastCount,0,true)));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LocalPriceCache:ThreadProc] Thread ended. Items in cache:{0}",Utility.FormatNumber(lastCount,0,true)));
}
}
}

111
MarketDataLib/DataAccess/PricingDA.cs
View File

@@ -1,10 +1,9 @@
using System;
using System.Text;
using System.Diagnostics;
using System.Collections.Generic;
using System.Text;
using MySql.Data.MySqlClient;
using MarketData.MarketDataModel;
using MarketData.Utils;
using System;
using System.Collections.Generic;
namespace MarketData.DataAccess
{
@@ -87,7 +86,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -129,7 +128,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -165,7 +164,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand) sqlCommand.Dispose();
if(null!=sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if(null!=sqlConnection) sqlConnection.Close();
}
}
@@ -211,7 +210,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -247,7 +246,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -284,10 +283,54 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand) sqlCommand.Dispose();
if(null!=sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if(null!=sqlConnection) sqlConnection.Close();
}
}
public static Dictionary<String,DateTime> GetLatestDates(List<String> symbols)
{
MySqlConnection sqlConnection = null;
MySqlDataReader sqlDataReader = null;
MySqlCommand sqlCommand = null;
String strQuery = null;
Dictionary<String,DateTime> latestDates = new Dictionary<String,DateTime>();
try
{
StringBuilder sb = new StringBuilder();
sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
sb.Append(" select symbol, max(date) as date ");
sb.Append(" from prices where symbol in").Append(SqlUtils.CreateInClause(symbols));
sb.Append(" group by symbol order by symbol");
strQuery = sb.ToString();
sqlCommand = new MySqlCommand(strQuery, sqlConnection);
sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
sqlDataReader = sqlCommand.ExecuteReader();
while (sqlDataReader.Read())
{
if(sqlDataReader.IsDBNull(0) || sqlDataReader.IsDBNull(1))continue;
String symbol = sqlDataReader.GetString(0);
DateTime latestDate = sqlDataReader.GetDateTime(1);
if(latestDates.ContainsKey(symbol))continue;
latestDates.Add(symbol, latestDate);
}
return latestDates;
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Query was {0}",strQuery));
return latestDates;
}
finally
{
if (null != sqlCommand) sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlConnection) sqlConnection.Close();
}
}
public static DateTime GetLatestDate(List<String> symbols)
{
MySqlConnection sqlConnection = null;
@@ -326,7 +369,7 @@ namespace MarketData.DataAccess
finally
{
if (null != sqlCommand) sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -362,7 +405,7 @@ namespace MarketData.DataAccess
finally
{
if (null != sqlCommand) sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -400,7 +443,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand) sqlCommand.Dispose();
if(null!=sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if(null!=sqlConnection) sqlConnection.Close();
}
}
@@ -437,7 +480,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -476,7 +519,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -513,7 +556,7 @@ namespace MarketData.DataAccess
finally
{
if (null != sqlCommand) sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -551,7 +594,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand) sqlCommand.Dispose();
if(null!=sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if(null!=sqlConnection) sqlConnection.Close();
}
}
@@ -588,7 +631,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -625,7 +668,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -670,7 +713,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -707,7 +750,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -743,7 +786,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -779,7 +822,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -819,7 +862,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -855,7 +898,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -940,7 +983,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -993,7 +1036,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -1048,7 +1091,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -1103,7 +1146,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -1157,7 +1200,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -1212,7 +1255,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand) sqlCommand.Dispose();
if(null!=sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if(null!=sqlConnection) sqlConnection.Close();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetPricesForward] Done, took {0}(ms)",profiler.End()));
}
@@ -1262,7 +1305,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -1320,7 +1363,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}
@@ -1370,7 +1413,7 @@ namespace MarketData.DataAccess
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
if (null != sqlConnection) sqlConnection.Close();
}
}