Add an eviction policy to the GBPriceCache.

2026-03-11 19:02:48 -04:00
parent 7e638cca05
commit 84ec9b31a3
3 changed files with 122 additions and 35 deletions
--- a/MarketData/MarketData/Models/runcmmomentumbacktest.sh
+++ b/MarketData/MarketData/Models/runcmmomentumbacktest.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# IDENTICAL RUN TO TEST8B. THE MODEL WAS RUN A 2ND TIME WITH DATABASE UPDATES TO BRING IT CURRENT.
+# THE PREVIOUS DATABASE ONLY CONTAINED DATA THROUGH 07/2025. THE NEW IMPORT
+# MAKES DATA AVAILABLE UP TO EARLY MARCH 2026
+# CNNClient.Model.CNNClient.Model.convnext upscaling 224 for the convneXt pretrained model and adding ^VIX volatility and using 90 days
+# THIS TEST USED A 25% REWARD FOR A POSITIVE PREDICTION
+# CNN TEST
+export DOTNET_ROOT=/opt/dotnet
+
+CNNDAYCOUNT=90
+VERSION=05
+USEMAXPOSITIONBUCKETWEIGHT=TRUE
+USEMAXPOSITIONBUCKETWEIGHTMAXWEIGHT=.65
+USEOVEREXTENDEDINDICATOR=TRUE
+DAYS=10
+VIOLATIONS=1
+MARGINPERCENT=1.00
+STARTDATE=10-31-2019
+CNNREWARD=.25
+PATHSESSIONFILE="CM${STARTDATE}_OI_${DAYS}_${VIOLATIONS}_MPBW_65_USEOV${USEOVEREXTENDEDINDICATOR}_V${VERSION}.TXT"
+
+if [ -f "$PATHSESSIONFILE" ]; then
+    rm "$PATHSESSIONFILE"
+    echo "Deleted existing session file: $PATHSESSIONFILE"
+else
+    echo "Session file not found, proceeding: $PATHSESSIONFILE"
+fi
+
+# echo "RUNCMBACKTEST /USECNN:TRUE /USECNNCLIENT:TRUE /USECNNDAYCOUNT:${CNNDAYCOUNT} /USECNNHOST:127.0.0.1:5000 /USECNNREWARDPERCENTDECIMAL:${CNNREWARD} /USEMAXPOSITIONBUCKETWEIGHT:${USEMAXPOSITIONBUCKETWEIGHT} /USEMAXPOSITIONBUCKETWEIGHTMAXWEIGHT:${USEMAXPOSITIONBUCKETWEIGHTMAXWEIGHT} /STARTDATE:${STARTDATE} /MAXPOSITIONS:3 /INITIALCASH:10000 /HOLDINGPERIOD:3 /TARGETBETA:1 /SESSIONFILE:${PATHSESSIONFILE} /USEOVEREXTENDEDINDICATOR:${USEOVEREXTENDEDINDICATOR} /USEOVEREXTENDEDINDICATORDAYS:${DAYS} /USEOVEREXTENDEDINDICATORVIOLATIONTHRESHHOLD:${VIOLATIONS} /USEOVEREXTENDEDINDICATORMARGINPERCENT:${MARGINPERCENT}"
+
+/home/pi/ARM64/MarketData/MarketData/bin/Debug/net8.0/mk RUNCMBACKTEST \
+    /USECNN:TRUE \
+    /USECNNCLIENT:TRUE \
+    /USECNNDAYCOUNT:${CNNDAYCOUNT} \
+    /USECNNHOST:10.0.0.240:5000 \
+    /USECNNREWARDPERCENTDECIMAL:${CNNREWARD} \
+    /USEMAXPOSITIONBUCKETWEIGHT:${USEMAXPOSITIONBUCKETWEIGHT} \
+    /USEMAXPOSITIONBUCKETWEIGHTMAXWEIGHT:${USEMAXPOSITIONBUCKETWEIGHTMAXWEIGHT} \
+    /STARTDATE:${STARTDATE} \
+    /MAXPOSITIONS:3 \
+    /INITIALCASH:10000 \
+    /HOLDINGPERIOD:3 \
+    /TARGETBETA:1 \
+    /SESSIONFILE:${PATHSESSIONFILE} \
+    /USEOVEREXTENDEDINDICATOR:${USEOVEREXTENDEDINDICATOR} \
+    /USEOVEREXTENDEDINDICATORDAYS:${DAYS} \
+    /USEOVEREXTENDEDINDICATORVIOLATIONTHRESHHOLD:${VIOLATIONS} \
+    /USEOVEREXTENDEDINDICATORMARGINPERCENT:${MARGINPERCENT}
+
+exit 0
--- a/MarketData/MarketDataLib/Cache/GBPriceCache.cs
+++ b/MarketData/MarketDataLib/Cache/GBPriceCache.cs
@@ -46,6 +46,7 @@ namespace MarketData.Cache
    private Thread cacheMonitorThread = null;
    private volatile bool threadRun = true;
    private Object thisLock = new Object();
+    private static volatile bool isShutdown = false;

    private CacheSnapshot snapshot;
    private DateGenerator dateGenerator = new DateGenerator();
@@ -61,10 +62,16 @@ namespace MarketData.Cache
      cacheMonitorThread.Start();
    }

+    /// <summary>
+    /// GetInstance 
+    /// </summary>
+    /// <returns></returns>
+    /// <exception cref="ObjectDisposedException"></exception>
    public static GBPriceCache GetInstance()
    {
      lock (typeof(GBPriceCache))
      {
+        if (isShutdown) throw new ObjectDisposedException(nameof(GBPriceCache), "Cache has been shut down.");
        if (null == priceCacheInstance)
        {
          priceCacheInstance = new GBPriceCache();
@@ -81,11 +88,17 @@ namespace MarketData.Cache
      }
    }

+    /// <summary>
+    /// Shuts down the cache and stops the monitor thread. This is a full application-level
+    /// shutdown. Callers should always access the cache via GetInstance() and not hold
+    /// long-lived references to the instance.
+    /// </summary>
    public void Dispose()
    {
      lock (thisLock)
      {
        if (null == priceCacheInstance || !threadRun) return;
+        isShutdown = true;
        threadRun = false;
        if (null != cacheMonitorThread)
        {
@@ -97,33 +110,6 @@ namespace MarketData.Cache
      }
    }

-    public void ClearCacheOnOrBefore(DateTime onOrBeforeDate, bool collect = false)
-    {
-      lock (thisLock)
-      {
-        Dictionary<String, PricesByDate> newPriceCache = new Dictionary<String, PricesByDate>();
-        foreach (KeyValuePair<String, PricesByDate> entry in snapshot.PriceCache)
-        {
-          String symbol = entry.Key;
-          PricesByDate filteredPrices = new PricesByDate();
-          PricesByDate existingPrices = entry.Value;
-          foreach (KeyValuePair<DateTime, Price> kvp in existingPrices)
-          {
-            if (kvp.Key >= onOrBeforeDate)
-            {
-              filteredPrices.Add(kvp.Key, kvp.Value);
-            }
-          }
-          if (filteredPrices.Count > 0)
-          {
-            newPriceCache.Add(symbol, filteredPrices);
-          }
-        }
-        UpdateSnapshot(newPriceCache, snapshot.RealTimePriceCache, snapshot.NullCache);
-        if (collect) GC.Collect();
-      }
-    }
-
    public Price GetPriceOrLatestAvailable(String symbol, DateTime date)
    {
      Price price = GetPrice(symbol, date);
@@ -294,6 +280,38 @@ namespace MarketData.Cache
      return pricesByDate.ContainsKey(date);
    }

+    public void ClearCacheOnOrBefore(DateTime onOrBeforeDate, bool collect = false)
+    {
+      lock (thisLock)
+      {
+        UpdateSnapshot(BuildEvictedPriceCache(onOrBeforeDate), snapshot.RealTimePriceCache, new Dictionary<String, bool>());                     
+        if (collect) GC.Collect();
+      }
+    }
+
+    private Dictionary<String, PricesByDate> BuildEvictedPriceCache(DateTime onOrBeforeDate)
+    {
+      Dictionary<String, PricesByDate> newPriceCache = new Dictionary<String, PricesByDate>();
+      foreach (KeyValuePair<String, PricesByDate> entry in snapshot.PriceCache)
+      {
+        String symbol = entry.Key;
+        PricesByDate filteredPrices = new PricesByDate();
+        PricesByDate existingPrices = entry.Value;
+        foreach (KeyValuePair<DateTime, Price> kvp in existingPrices)
+        {
+          if (kvp.Key >= onOrBeforeDate)
+          {
+            filteredPrices.Add(kvp.Key, kvp.Value);
+          }
+        }
+        if (filteredPrices.Count > 0)
+        {
+          newPriceCache.Add(symbol, filteredPrices);
+        }
+      }
+      return newPriceCache;
+    }      
+
    private void ThreadProc()
    {
      int quantums = 0;
@@ -302,18 +320,25 @@ namespace MarketData.Cache
      while (threadRun)
      {
        Thread.Sleep(quantumInterval);
-        if(!threadRun)break;
+        if (!threadRun) break;
        quantums += quantumInterval;
        if (quantums > cacheRefreshAfter)
        {
          quantums = 0;
          lock (thisLock)
          {
-            UpdateSnapshot(snapshot.PriceCache, new Dictionary<String, Price>(), snapshot.NullCache);
+            DateTime maxDate = snapshot.PriceCache.Values.SelectMany(p => p.Keys).DefaultIfEmpty(DateTime.MinValue).Max();
+            if (maxDate != DateTime.MinValue)
+            {
+              DateTime evictBefore = dateGenerator.GenerateHistoricalDates(maxDate, 252).Min();
+              MDTrace.WriteLine(LogLevel.DEBUG, $"GBPriceCache, clearing cache on or before {evictBefore.ToShortDateString()}");
+              UpdateSnapshot(BuildEvictedPriceCache(evictBefore), new Dictionary<String, Price>(), new Dictionary<String, bool>());
+              GC.Collect();
+            }
          }
        }
      }
-    }
+    }     

    private void UpdateSnapshot(Dictionary<String, PricesByDate> newPriceCache,Dictionary<String, Price> newRealtimePriceCache, Dictionary<String, bool> newNullCache)
    {
--- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs
+++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs
@@ -1,3 +1,4 @@
+#pragma warning disable SYSLIB0014
 using System.Net;
 using System.Text;
 using System.IO.Compression;
@@ -1843,18 +1844,27 @@ namespace MarketData.Integration
      return webRequest;
    }

+    /// <summary>
+    /// IsMovedException - MovedPermanently, Found, RedirectKeepVerb, PermanentRedirect will all return a "Location" in the response
+    /// That we will forward the request to.
+    /// </summary>
+    /// <param name="webException"></param>
+    /// <returns></returns>
    private static bool IsMovedException(WebException webException)
    {
-      if(webException.Status.Equals(WebExceptionStatus.ProtocolError))
+      if (webException.Status.Equals(WebExceptionStatus.ProtocolError))
      {
        HttpWebResponse response = webException.Response as HttpWebResponse;
-        if(response.StatusCode.Equals(HttpStatusCode.MovedPermanently))
+        if (response.StatusCode.Equals(HttpStatusCode.MovedPermanently) ||
+            response.StatusCode.Equals(HttpStatusCode.Found) ||
+            response.StatusCode.Equals(HttpStatusCode.RedirectKeepVerb) ||
+            response.StatusCode.Equals(HttpStatusCode.PermanentRedirect))
        {
-          return true;
+            return true;
        }
-      } 
+      }
      return false;
-    }
+    }    

    /// <summary>
    /// Get the proxy for the specified service or null if it is not congfigured to use the proxy
@@ -1921,3 +1931,4 @@ namespace MarketData.Integration
    }
  }  
 }
+#pragma warning restore SYSLIB0014