From 84ec9b31a3a81066a80a61690500f0f0829337ba Mon Sep 17 00:00:00 2001 From: Sean Date: Wed, 11 Mar 2026 19:02:48 -0400 Subject: [PATCH] Add an eviction policy to the GBPriceCache. --- .../Models/runcmmomentumbacktest.sh | 51 +++++++++++ .../MarketDataLib/Cache/GBPriceCache.cs | 85 ++++++++++++------- .../Integration/HttpNetRequest.cs | 21 +++-- 3 files changed, 122 insertions(+), 35 deletions(-) create mode 100755 MarketData/MarketData/Models/runcmmomentumbacktest.sh diff --git a/MarketData/MarketData/Models/runcmmomentumbacktest.sh b/MarketData/MarketData/Models/runcmmomentumbacktest.sh new file mode 100755 index 0000000..30dc99f --- /dev/null +++ b/MarketData/MarketData/Models/runcmmomentumbacktest.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# IDENTICAL RUN TO TEST8B. THE MODEL WAS RUN A 2ND TIME WITH DATABASE UPDATES TO BRING IT CURRENT. +# THE PREVIOUS DATABASE ONLY CONTAINED DATA THROUGH 07/2025. THE NEW IMPORT +# MAKES DATA AVAILABLE UP TO EARLY MARCH 2026 +# CNNClient.Model.CNNClient.Model.convnext upscaling 224 for the convneXt pretrained model and adding ^VIX volatility and using 90 days +# THIS TEST USED A 25% REWARD FOR A POSITIVE PREDICTION +# CNN TEST +export DOTNET_ROOT=/opt/dotnet + +CNNDAYCOUNT=90 +VERSION=05 +USEMAXPOSITIONBUCKETWEIGHT=TRUE +USEMAXPOSITIONBUCKETWEIGHTMAXWEIGHT=.65 +USEOVEREXTENDEDINDICATOR=TRUE +DAYS=10 +VIOLATIONS=1 +MARGINPERCENT=1.00 +STARTDATE=10-31-2019 +CNNREWARD=.25 +PATHSESSIONFILE="CM${STARTDATE}_OI_${DAYS}_${VIOLATIONS}_MPBW_65_USEOV${USEOVEREXTENDEDINDICATOR}_V${VERSION}.TXT" + +if [ -f "$PATHSESSIONFILE" ]; then + rm "$PATHSESSIONFILE" + echo "Deleted existing session file: $PATHSESSIONFILE" +else + echo "Session file not found, proceeding: $PATHSESSIONFILE" +fi + +# echo "RUNCMBACKTEST /USECNN:TRUE /USECNNCLIENT:TRUE /USECNNDAYCOUNT:${CNNDAYCOUNT} /USECNNHOST:127.0.0.1:5000 /USECNNREWARDPERCENTDECIMAL:${CNNREWARD} /USEMAXPOSITIONBUCKETWEIGHT:${USEMAXPOSITIONBUCKETWEIGHT} /USEMAXPOSITIONBUCKETWEIGHTMAXWEIGHT:${USEMAXPOSITIONBUCKETWEIGHTMAXWEIGHT} /STARTDATE:${STARTDATE} /MAXPOSITIONS:3 /INITIALCASH:10000 /HOLDINGPERIOD:3 /TARGETBETA:1 /SESSIONFILE:${PATHSESSIONFILE} /USEOVEREXTENDEDINDICATOR:${USEOVEREXTENDEDINDICATOR} /USEOVEREXTENDEDINDICATORDAYS:${DAYS} /USEOVEREXTENDEDINDICATORVIOLATIONTHRESHHOLD:${VIOLATIONS} /USEOVEREXTENDEDINDICATORMARGINPERCENT:${MARGINPERCENT}" + +/home/pi/ARM64/MarketData/MarketData/bin/Debug/net8.0/mk RUNCMBACKTEST \ + /USECNN:TRUE \ + /USECNNCLIENT:TRUE \ + /USECNNDAYCOUNT:${CNNDAYCOUNT} \ + /USECNNHOST:10.0.0.240:5000 \ + /USECNNREWARDPERCENTDECIMAL:${CNNREWARD} \ + /USEMAXPOSITIONBUCKETWEIGHT:${USEMAXPOSITIONBUCKETWEIGHT} \ + /USEMAXPOSITIONBUCKETWEIGHTMAXWEIGHT:${USEMAXPOSITIONBUCKETWEIGHTMAXWEIGHT} \ + /STARTDATE:${STARTDATE} \ + /MAXPOSITIONS:3 \ + /INITIALCASH:10000 \ + /HOLDINGPERIOD:3 \ + /TARGETBETA:1 \ + /SESSIONFILE:${PATHSESSIONFILE} \ + /USEOVEREXTENDEDINDICATOR:${USEOVEREXTENDEDINDICATOR} \ + /USEOVEREXTENDEDINDICATORDAYS:${DAYS} \ + /USEOVEREXTENDEDINDICATORVIOLATIONTHRESHHOLD:${VIOLATIONS} \ + /USEOVEREXTENDEDINDICATORMARGINPERCENT:${MARGINPERCENT} + +exit 0 diff --git a/MarketData/MarketDataLib/Cache/GBPriceCache.cs b/MarketData/MarketDataLib/Cache/GBPriceCache.cs index c6494c1..167c4ee 100755 --- a/MarketData/MarketDataLib/Cache/GBPriceCache.cs +++ b/MarketData/MarketDataLib/Cache/GBPriceCache.cs @@ -46,6 +46,7 @@ namespace MarketData.Cache private Thread cacheMonitorThread = null; private volatile bool threadRun = true; private Object thisLock = new Object(); + private static volatile bool isShutdown = false; private CacheSnapshot snapshot; private DateGenerator dateGenerator = new DateGenerator(); @@ -61,10 +62,16 @@ namespace MarketData.Cache cacheMonitorThread.Start(); } + /// + /// GetInstance + /// + /// + /// public static GBPriceCache GetInstance() { lock (typeof(GBPriceCache)) { + if (isShutdown) throw new ObjectDisposedException(nameof(GBPriceCache), "Cache has been shut down."); if (null == priceCacheInstance) { priceCacheInstance = new GBPriceCache(); @@ -81,11 +88,17 @@ namespace MarketData.Cache } } + /// + /// Shuts down the cache and stops the monitor thread. This is a full application-level + /// shutdown. Callers should always access the cache via GetInstance() and not hold + /// long-lived references to the instance. + /// public void Dispose() { lock (thisLock) { if (null == priceCacheInstance || !threadRun) return; + isShutdown = true; threadRun = false; if (null != cacheMonitorThread) { @@ -97,33 +110,6 @@ namespace MarketData.Cache } } - public void ClearCacheOnOrBefore(DateTime onOrBeforeDate, bool collect = false) - { - lock (thisLock) - { - Dictionary newPriceCache = new Dictionary(); - foreach (KeyValuePair entry in snapshot.PriceCache) - { - String symbol = entry.Key; - PricesByDate filteredPrices = new PricesByDate(); - PricesByDate existingPrices = entry.Value; - foreach (KeyValuePair kvp in existingPrices) - { - if (kvp.Key >= onOrBeforeDate) - { - filteredPrices.Add(kvp.Key, kvp.Value); - } - } - if (filteredPrices.Count > 0) - { - newPriceCache.Add(symbol, filteredPrices); - } - } - UpdateSnapshot(newPriceCache, snapshot.RealTimePriceCache, snapshot.NullCache); - if (collect) GC.Collect(); - } - } - public Price GetPriceOrLatestAvailable(String symbol, DateTime date) { Price price = GetPrice(symbol, date); @@ -294,6 +280,38 @@ namespace MarketData.Cache return pricesByDate.ContainsKey(date); } + public void ClearCacheOnOrBefore(DateTime onOrBeforeDate, bool collect = false) + { + lock (thisLock) + { + UpdateSnapshot(BuildEvictedPriceCache(onOrBeforeDate), snapshot.RealTimePriceCache, new Dictionary()); + if (collect) GC.Collect(); + } + } + + private Dictionary BuildEvictedPriceCache(DateTime onOrBeforeDate) + { + Dictionary newPriceCache = new Dictionary(); + foreach (KeyValuePair entry in snapshot.PriceCache) + { + String symbol = entry.Key; + PricesByDate filteredPrices = new PricesByDate(); + PricesByDate existingPrices = entry.Value; + foreach (KeyValuePair kvp in existingPrices) + { + if (kvp.Key >= onOrBeforeDate) + { + filteredPrices.Add(kvp.Key, kvp.Value); + } + } + if (filteredPrices.Count > 0) + { + newPriceCache.Add(symbol, filteredPrices); + } + } + return newPriceCache; + } + private void ThreadProc() { int quantums = 0; @@ -302,18 +320,25 @@ namespace MarketData.Cache while (threadRun) { Thread.Sleep(quantumInterval); - if(!threadRun)break; + if (!threadRun) break; quantums += quantumInterval; if (quantums > cacheRefreshAfter) { quantums = 0; lock (thisLock) { - UpdateSnapshot(snapshot.PriceCache, new Dictionary(), snapshot.NullCache); + DateTime maxDate = snapshot.PriceCache.Values.SelectMany(p => p.Keys).DefaultIfEmpty(DateTime.MinValue).Max(); + if (maxDate != DateTime.MinValue) + { + DateTime evictBefore = dateGenerator.GenerateHistoricalDates(maxDate, 252).Min(); + MDTrace.WriteLine(LogLevel.DEBUG, $"GBPriceCache, clearing cache on or before {evictBefore.ToShortDateString()}"); + UpdateSnapshot(BuildEvictedPriceCache(evictBefore), new Dictionary(), new Dictionary()); + GC.Collect(); + } } } } - } + } private void UpdateSnapshot(Dictionary newPriceCache,Dictionary newRealtimePriceCache, Dictionary newNullCache) { diff --git a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs index 5ef8b40..f9886a3 100755 --- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs +++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs @@ -1,3 +1,4 @@ +#pragma warning disable SYSLIB0014 using System.Net; using System.Text; using System.IO.Compression; @@ -1843,18 +1844,27 @@ namespace MarketData.Integration return webRequest; } + /// + /// IsMovedException - MovedPermanently, Found, RedirectKeepVerb, PermanentRedirect will all return a "Location" in the response + /// That we will forward the request to. + /// + /// + /// private static bool IsMovedException(WebException webException) { - if(webException.Status.Equals(WebExceptionStatus.ProtocolError)) + if (webException.Status.Equals(WebExceptionStatus.ProtocolError)) { HttpWebResponse response = webException.Response as HttpWebResponse; - if(response.StatusCode.Equals(HttpStatusCode.MovedPermanently)) + if (response.StatusCode.Equals(HttpStatusCode.MovedPermanently) || + response.StatusCode.Equals(HttpStatusCode.Found) || + response.StatusCode.Equals(HttpStatusCode.RedirectKeepVerb) || + response.StatusCode.Equals(HttpStatusCode.PermanentRedirect)) { - return true; + return true; } - } + } return false; - } + } /// /// Get the proxy for the specified service or null if it is not congfigured to use the proxy @@ -1921,3 +1931,4 @@ namespace MarketData.Integration } } } +#pragma warning restore SYSLIB0014