Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update GBPriceCache.

Browse Source
This commit is contained in:
Sean
2026-03-11 20:58:51 -04:00
parent aa13085fd3
commit 10e1c87ec1
1 changed files with 96 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
MarketDataLib/Cache/GBPriceCache.cs
View File

@@ -5,7 +5,6 @@ using System.Threading;
using MarketData.MarketDataModel;
using MarketData.Utils;
using MarketData.Helper;
using MarketData.Numerical;
using MarketData.DataAccess;
namespace MarketData.Cache
@@ -43,6 +42,7 @@ namespace MarketData.Cache
public class GBPriceCache : IDisposable
{
private static readonly int EVICTION_DAYCOUNT=252; // upon eviction trigger remove all data older than maxdate - evictionPolicyThreshholdDays
private Thread cacheMonitorThread = null;
private volatile bool threadRun = true;
private Object thisLock = new Object();
@@ -61,6 +61,11 @@ namespace MarketData.Cache
cacheMonitorThread.Start();
}
/// <summary>
/// GetInstance
/// </summary>
/// <returns></returns>
/// <exception cref="ObjectDisposedException"></exception>
public static GBPriceCache GetInstance()
{
lock (typeof(GBPriceCache))
@@ -81,6 +86,11 @@ namespace MarketData.Cache
}
}
/// <summary>
/// Shuts down the cache and stops the monitor thread. This is a full application-level
/// shutdown. Callers should always access the cache via GetInstance() and not hold
/// long-lived references to the instance.
/// </summary>
public void Dispose()
{
lock (thisLock)
@@ -97,40 +107,15 @@ namespace MarketData.Cache
}
}
public void ClearCacheOnOrBefore(DateTime onOrBeforeDate, bool collect = false)
{
lock (thisLock)
{
Dictionary<String, PricesByDate> newPriceCache = new Dictionary<String, PricesByDate>();
foreach (KeyValuePair<String, PricesByDate> entry in snapshot.PriceCache)
{
String symbol = entry.Key;
PricesByDate filteredPrices = new PricesByDate();
PricesByDate existingPrices = entry.Value;
foreach (KeyValuePair<DateTime, Price> kvp in existingPrices)
{
if (kvp.Key >= onOrBeforeDate)
{
filteredPrices.Add(kvp.Key, kvp.Value);
}
}
if (filteredPrices.Count > 0)
{
newPriceCache.Add(symbol, filteredPrices);
}
}
UpdateSnapshot(newPriceCache, snapshot.RealTimePriceCache, snapshot.NullCache);
if (collect) GC.Collect();
}
}
public Price GetPriceOrLatestAvailable(String symbol, DateTime date)
{
Price price = GetPrice(symbol, date);
if (null != price) return price;
DateTime latestPricingDate = PricingDataAccess.GetLatestDateOnOrBefore(symbol, date);
price = GetPrice(symbol, latestPricingDate);
if (null != price) return price;
fetchSemaphore.Wait();
try
{
@@ -140,7 +125,8 @@ namespace MarketData.Cache
{
fetchSemaphore.Release();
}
if (null !=price) AddPrice(price);
if (null != price) AddPrice(price);
return price;
}
@@ -152,27 +138,33 @@ namespace MarketData.Cache
}
Price price = MarketDataHelper.GetLatestPrice(symbol);
if (null != price)
{
Dictionary<String, Price> newRealtime = new Dictionary<String, Price>(snapshot.RealTimePriceCache);
newRealtime.Add(symbol, price);
UpdateSnapshot(snapshot.PriceCache, newRealtime, snapshot.NullCache);
}
return price;
}
public Price GetPrice(String symbol, DateTime date)
{
date = date.Date;
if (!ContainsPrice(symbol, date))
{
String key = symbol + Utility.DateTimeToStringMMHDDHYYYY(date);
if (snapshot.NullCache.ContainsKey(key))
{
return null;
}
fetchSemaphore.Wait();
Price price;
try
{
price = PricingDataAccess.GetPrice(symbol, date);
@@ -181,7 +173,8 @@ namespace MarketData.Cache
{
fetchSemaphore.Release();
}
if (null ==price)
if (null == price)
{
Dictionary<String, bool> newNullCache = new Dictionary<String, bool>(snapshot.NullCache);
newNullCache.Add(key, true);
@@ -191,23 +184,29 @@ namespace MarketData.Cache
AddPrice(price);
}
if (!snapshot.PriceCache.ContainsKey(symbol)) return null;
PricesByDate pricesByDate = snapshot.PriceCache[symbol];
if (!pricesByDate.ContainsKey(date)) return null;
return pricesByDate[date];
}
public Prices GetPrices(String symbol, DateTime earlierDate, DateTime laterDate)
{
DateGenerator localDateGenerator = new DateGenerator();
if (laterDate < earlierDate)
{
DateTime tempDate = earlierDate;
earlierDate = laterDate;
laterDate = tempDate;
}
List<DateTime> datesList = localDateGenerator.GenerateHistoricalDates(earlierDate, laterDate);
datesList = datesList.Where(x => x >= earlierDate).ToList();
return GetPrices(symbol, laterDate, datesList.Count);
}
@@ -215,17 +214,20 @@ namespace MarketData.Cache
{
List<DateTime> historicalDates = dateGenerator.GenerateHistoricalDates(startDate, dayCount + 60);
List<DateTime> missingDates = new List<DateTime>();
foreach (DateTime historicalDate in historicalDates)
{
if (!ContainsPrice(symbol, historicalDate))
{
String key = symbol + Utility.DateTimeToStringMMHDDHYYYY(historicalDate);
if (!snapshot.NullCache.ContainsKey(key))
{
missingDates.Add(historicalDate);
}
}
}
if (missingDates.Count > 0)
{
DateTime minDate = missingDates.Min();
@@ -233,6 +235,7 @@ namespace MarketData.Cache
fetchSemaphore.Wait();
Prices loadedPrices;
try
{
loadedPrices = PricingDataAccess.GetPrices(symbol, maxDate, minDate);
@@ -249,11 +252,14 @@ namespace MarketData.Cache
}
Prices prices = new Prices();
foreach (DateTime historicalDate in historicalDates)
{
if (!snapshot.PriceCache.ContainsKey(symbol)) continue;
PricesByDate pricesByDate = snapshot.PriceCache[symbol];
if (!pricesByDate.ContainsKey(historicalDate)) continue;
prices.Add(pricesByDate[historicalDate]);
}
@@ -268,17 +274,21 @@ namespace MarketData.Cache
lock (thisLock)
{
PricesByDate pricesByDate;
if (!snapshot.PriceCache.ContainsKey(price.Symbol))
{
pricesByDate = new PricesByDate();
pricesByDate.Add(price.Date, price);
Dictionary<String, PricesByDate> newCache = new Dictionary<String, PricesByDate>(snapshot.PriceCache);
newCache.Add(price.Symbol, pricesByDate);
UpdateSnapshot(newCache, snapshot.RealTimePriceCache, snapshot.NullCache);
}
else
{
pricesByDate = snapshot.PriceCache[price.Symbol];
if (!pricesByDate.ContainsKey(price.Date))
{
pricesByDate.Add(price.Date, price);
@@ -290,10 +300,47 @@ namespace MarketData.Cache
public bool ContainsPrice(String symbol, DateTime date)
{
if (!snapshot.PriceCache.ContainsKey(symbol)) return false;
PricesByDate pricesByDate = snapshot.PriceCache[symbol];
return pricesByDate.ContainsKey(date);
}
public void ClearCacheOnOrBefore(DateTime onOrBeforeDate, bool collect = false)
{
lock (thisLock)
{
UpdateSnapshot(BuildEvictedPriceCache(onOrBeforeDate), snapshot.RealTimePriceCache, new Dictionary<String, bool>());
if (collect) GC.Collect();
}
}
private Dictionary<String, PricesByDate> BuildEvictedPriceCache(DateTime onOrBeforeDate)
{
Dictionary<String, PricesByDate> newPriceCache = new Dictionary<String, PricesByDate>();
foreach (KeyValuePair<String, PricesByDate> entry in snapshot.PriceCache)
{
String symbol = entry.Key;
PricesByDate filteredPrices = new PricesByDate();
PricesByDate existingPrices = entry.Value;
foreach (KeyValuePair<DateTime, Price> kvp in existingPrices)
{
if (kvp.Key >= onOrBeforeDate)
{
filteredPrices.Add(kvp.Key, kvp.Value);
}
}
if (filteredPrices.Count > 0)
{
newPriceCache.Add(symbol, filteredPrices);
}
}
return newPriceCache;
}
private void ThreadProc()
{
int quantums = 0;
@@ -302,22 +349,37 @@ namespace MarketData.Cache
while (threadRun)
{
Thread.Sleep(quantumInterval);
if(!threadRun)break;
if (!threadRun) break;
quantums += quantumInterval;
if (quantums > cacheRefreshAfter)
{
quantums = 0;
lock (thisLock)
{
UpdateSnapshot(snapshot.PriceCache, new Dictionary<String, Price>(), snapshot.NullCache);
DateTime maxDate = snapshot.PriceCache.Values.SelectMany(p => p.Keys).DefaultIfEmpty(DateTime.MinValue).Max();
if (maxDate != DateTime.MinValue)
{
DateTime evictBefore = dateGenerator.GenerateHistoricalDates(maxDate, EVICTION_DAYCOUNT).Min();
int beforeCount = snapshot.PriceCache.Values.Sum(p => p.Count);
Dictionary<String, PricesByDate> newCache = BuildEvictedPriceCache(evictBefore);
int afterCount = newCache.Values.Sum(p => p.Count);
int removed = beforeCount - afterCount;
MDTrace.WriteLine(LogLevel.DEBUG, $"GBPriceCache eviction: removed {removed} prices (before={beforeCount}, after={afterCount}) on or before {evictBefore.ToShortDateString()}");
UpdateSnapshot(newCache, new Dictionary<String, Price>(), new Dictionary<String, bool>());
GC.Collect();
}
}
}
}
}
private void UpdateSnapshot(Dictionary<String, PricesByDate> newPriceCache,Dictionary<String, Price> newRealtimePriceCache, Dictionary<String, bool> newNullCache)
private void UpdateSnapshot(Dictionary<String, PricesByDate> newPriceCache, Dictionary<String, Price> newRealtimePriceCache, Dictionary<String, bool> newNullCache)
{
snapshot = new CacheSnapshot(newPriceCache, newRealtimePriceCache, newNullCache);
}
}
}
}