From 22b387a2e3bddb4721e8cb404bc75be47fb56de4 Mon Sep 17 00:00:00 2001 From: "Sean Kessler (Europa)" Date: Tue, 10 Mar 2026 21:43:40 -0400 Subject: [PATCH] Push MarketData Changes. --- MarketDataLib/CNNProcessing/CNNClient.cs | 2 +- MarketDataLib/CNNProcessing/CNNProcessor.cs | 137 +++++++++++++- MarketDataLib/CNNProcessing/DataProcessor.cs | 177 +++++++++++++++++- MarketDataLib/CNNProcessing/TestCase.cs | 2 +- .../CMMomentum/CMMomentumGenerator.cs | 42 ++++- 5 files changed, 343 insertions(+), 17 deletions(-) diff --git a/MarketDataLib/CNNProcessing/CNNClient.cs b/MarketDataLib/CNNProcessing/CNNClient.cs index 616f377..6e9d467 100644 --- a/MarketDataLib/CNNProcessing/CNNClient.cs +++ b/MarketDataLib/CNNProcessing/CNNClient.cs @@ -7,7 +7,7 @@ namespace MarketData.CNNProcessing { public class CNNClient { - public enum Model{resnet50,resnet50B,resnet50_20241024_270,inception,vgg16,lenet5,ping}; + public enum Model{resnet50,resnet50B,resnet50_20241024_270,inception,vgg16,lenet5,convnext,ping}; private static readonly string Alive="Alive"; private readonly HttpClient client = new HttpClient(); private string baseUrl; diff --git a/MarketDataLib/CNNProcessing/CNNProcessor.cs b/MarketDataLib/CNNProcessing/CNNProcessor.cs index a13b02b..5e8e7a6 100644 --- a/MarketDataLib/CNNProcessing/CNNProcessor.cs +++ b/MarketDataLib/CNNProcessing/CNNProcessor.cs @@ -3,27 +3,75 @@ using System.IO; using System.Collections.Generic; using MarketData.Utils; using System.Text; +using System.Globalization; namespace MarketData.CNNProcessing { public class CNNProcessor { - private static int dayCount=270; - private static int width=128; - private static int height=128; + private static int dayCount=270; // This is the default days + private static int width=128; // This is the default width + private static int height=128; // THis is the defaukt height private CNNProcessor() { } - public static void GenerateTraining() + /// + /// GenerateTraining - This is the new one. Please refer to the CNNImageProcessor project for information on how to call this method. + /// + /// This is the collection of avoid holdings + /// This is the collection of good holdings + /// The image dimensions. for example 224 for 224x224 or 128 for 128x128 + /// This is the number of histDays. For example I used 90 for convnext + /// The type. For example I used BollingerBandWithVIX which is a bollinger band with ^VIX overay for convnext + /// + public static void GenerateTraining(List avoid, List good, int dimension, int histDays,TestCase.GenerateType generateType=TestCase.GenerateType.BollingerBandWithVIX,String rootFolder=@"C:\boneyard\DeepLearning\ModelInputData\") + { + TestCases testCases=new TestCases(); + DataProcessor dataProcessor=new DataProcessor(); + dataProcessor.Width=dimension; + dataProcessor.Height=dimension; + dataProcessor.PenWidthArray=new float[]{.75f,1.00f,1.12f}; + + if(!rootFolder.EndsWith(@"\"))rootFolder+=@"\"; +// [0] Data - The avoid data + foreach(Holding holding in avoid) + { + testCases.Add(new TestCase(holding.Symbol,holding.PurchaseDate,histDays,TestCase.CaseType.Training,generateType)); + } + dataProcessor.SetOutputFolderPath(rootFolder+"0"); + dataProcessor.ClearFolderPath(); + dataProcessor.ProcessData(testCases); + testCases.Clear(); + +// [1] Data - The good data + foreach(Holding holding in good) + { + testCases.Add(new TestCase(holding.Symbol,holding.PurchaseDate,histDays,TestCase.CaseType.Training,generateType)); + } + dataProcessor.SetOutputFolderPath(rootFolder+"1"); + dataProcessor.ClearFolderPath(); + dataProcessor.ProcessData(testCases); + } + + /// + /// GenerateTraining - This is the old methof training the resnet model. Please see above + /// + /// + public static void GenerateTraining(String rootFolder=@"C:\boneyard\DeepLearning\ModelInputData\") { TestCases testCases=new TestCases(); DataProcessor dataProcessor=new DataProcessor(); dataProcessor.Width=width; dataProcessor.Height=height; - dataProcessor.PenWidthArray=new float[]{.50f,.75f,1.00f,1.12f,1.25f,1.31f,1.37f,1.50f,1.56f,1.62f,1.75f,1.87f,2.00f}; + // dataProcessor.PenWidthArray=new float[]{.50f,.75f,1.00f,1.12f,1.25f,1.31f,1.37f,1.50f,1.56f,1.62f,1.75f,1.87f,2.00f}; + // Testing with 20,000 images in each set so reducing this use of pens to just one. It was producing 260,000 images for each classification, + // takings many hours to build the datasets + dataProcessor.PenWidthArray=new float[]{.75f,1.00f,1.12f}; + + if(!rootFolder.EndsWith(@"\"))rootFolder+=@"\"; // [0] Data - The avoid data testCases.Add(new TestCase("CENX",DateTime.Parse("03/31/2022"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand)); testCases.Add(new TestCase("ICPT",DateTime.Parse("12/31/2019"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand)); @@ -56,8 +104,8 @@ namespace MarketData.CNNProcessing testCases.Add(new TestCase("INBX",DateTime.Parse("01/31/2024"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand)); testCases.Add(new TestCase("WYNN",DateTime.Parse("02/28/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand)); - - dataProcessor.SetOutputFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\0"); +// **** + dataProcessor.SetOutputFolderPath(rootFolder+"0"); dataProcessor.ProcessData(testCases); testCases.Clear(); @@ -102,7 +150,8 @@ namespace MarketData.CNNProcessing testCases.Add(new TestCase("DOCU",DateTime.Parse("05/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand)); testCases.Add(new TestCase("SIG",DateTime.Parse("10/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand)); - dataProcessor.SetOutputFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\1"); +// *** + dataProcessor.SetOutputFolderPath(rootFolder+"1"); dataProcessor.ProcessData(testCases); } @@ -204,4 +253,76 @@ namespace MarketData.CNNProcessing Console.WriteLine(""); } } + + public class Holding + { + public String Symbol {get;set;} + public DateTime PurchaseDate {get; set; } + public double PurchasePrice {get;set;} + public DateTime SellDate {get; set; } + public double SellPrice {get;set;} + public double GainLoss{ get; set;} + public double GainLossPercent {get;set;} + private static readonly string[] DateFormats = { "MM/dd/yyyy", "M/dd/yyyy", "M/d/yyyy" }; + private static readonly CultureInfo UsCulture = CultureInfo.GetCultureInfo("en-US"); + + public static String Heading + { + get + { + return "Symbol,Shares,Purchase Date,Purchase Price,Sell Date,Sell Price,Exposure,Beta,BetaMonths,SharpeRatio,RiskAdjustedWeight,RiskAdjustedAllocation,TargetBetaOverBeta,Score,CNN Prediction,Market Value,Gain Loss,Gain Loss (%)"; + } + } + + public String ToTestCase() + { + StringBuilder sb = new StringBuilder(); + sb.Append("testCases.Add(new TestCase(").Append("\"").Append(Symbol).Append("\"").Append(","); + sb.Append("DateTime.Parse(").Append("\"").Append(Utility.DateTimeToStringMMSDDSYYYY(PurchaseDate)).Append("\")").Append(","); + sb.Append("270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));"); + return sb.ToString(); + } + + public override String ToString() + { + StringBuilder sb = new StringBuilder(); + sb.Append(Symbol).Append(","); + sb.Append(","); // shares + sb.Append(PurchaseDate.ToShortDateString()).Append(","); + sb.Append(Utility.FormatNumber(PurchasePrice,3)).Append(","); + sb.Append(SellDate.ToShortDateString()).Append(","); + sb.Append(Utility.FormatNumber(SellPrice,3)).Append(","); + sb.Append(","); //exposure + sb.Append(","); //beta + sb.Append(","); //bta months + sb.Append(","); //sharpe ratio + sb.Append(","); //risk adjusted weight + sb.Append(","); //RiskAdjustedAllocation + sb.Append(","); //TargetBetaOverBeta + sb.Append(","); //Score + sb.Append(","); //CNNPrediction + sb.Append(","); //Market Value + sb.Append(Utility.FormatNumber(GainLoss,3)).Append(","); + sb.Append(Utility.FormatNumber(GainLossPercent,3)); + return sb.ToString(); + } + + + public static Holding FromString(string strLine) + { + string[] items = strLine.Split(','); + + Holding holding = new Holding(); + holding.Symbol = items[0]; + if(string.IsNullOrEmpty(holding.Symbol))return null; + holding.PurchaseDate = DateTime.ParseExact(items[2], DateFormats, UsCulture, DateTimeStyles.AssumeLocal); + holding.PurchasePrice = double.Parse(items[3], UsCulture); + holding.SellDate = DateTime.ParseExact(items[4], DateFormats, UsCulture, DateTimeStyles.AssumeLocal); + holding.SellPrice = double.Parse(items[5], UsCulture); + holding.GainLoss = double.Parse(items[16], UsCulture); + holding.GainLossPercent = double.Parse(items[17].TrimEnd('%'), UsCulture) / 100.0; + + return holding; + } + } } diff --git a/MarketDataLib/CNNProcessing/DataProcessor.cs b/MarketDataLib/CNNProcessing/DataProcessor.cs index 5ed95a1..fe6fbb2 100644 --- a/MarketDataLib/CNNProcessing/DataProcessor.cs +++ b/MarketDataLib/CNNProcessing/DataProcessor.cs @@ -24,6 +24,7 @@ namespace MarketData.CNNProcessing Height=height; PenWidth=2f; DrawingBrush=new SolidBrush(Color.Black); + DrawingBrushRed=new SolidBrush(Color.Red); FillBrush=new SolidBrush(Color.White); DrawPrice=true; UseGrayScale=false; @@ -59,6 +60,11 @@ namespace MarketData.CNNProcessing /// ///Gets/Sets the drawing brush brush public Brush DrawingBrush{get;set;} + /// + /// DrawingBrush + /// + ///Gets/Sets the drawing brush brush + public Brush DrawingBrushRed{get;set;} /// /// DrawBlack @@ -143,6 +149,29 @@ namespace MarketData.CNNProcessing this.strFolderPath=strFolderPath; if(!this.strFolderPath.EndsWith(@"\"))this.strFolderPath=this.strFolderPath+@"\"; } + + /// + /// ClearFolderPath + /// + ///The test cases + public void ClearFolderPath() + { + if(String.IsNullOrEmpty(strFolderPath))throw new InvalidDataException($"{nameof(strFolderPath)} cannot be null"); + if(!Directory.Exists(strFolderPath)) + { + Directory.CreateDirectory(strFolderPath); + } + else + { + String[] pathFileNames = Directory.GetFiles(strFolderPath); + Console.WriteLine($"Deleting {pathFileNames.Length} files from {strFolderPath}"); + foreach(String file in pathFileNames) + { + File.Delete(file); + } + } + } + public void ProcessData(TestCases testCases) { for(int index=0;index +/// ProcessBollingerBandData item - Draws Price, K, L and Volatility +/// +///Symbol + private void ProcessBollingerBandDataWithVolatility(TestCase testCase,float penWidth,double noise) + { + String symbolVolatility="^VIX"; + DateGenerator dateGenerator=new DateGenerator(); + + int daysInPeriod=dateGenerator.DaysBetweenActual(testCase.PurchaseDate,testCase.HistDate); + daysInPeriod+=60; + Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,daysInPeriod); + Prices volatilityPrices=PricingDA.GetPrices(symbolVolatility,testCase.PurchaseDate,daysInPeriod); + BollingerBands bollingerBands=BollingerBandGenerator.GenerateBollingerBands(prices); // we want to grab K, L, and Close + bollingerBands=new BollingerBands(bollingerBands.Where(x=>x.Date>=testCase.HistDate).ToList()); + float[] k=new float[bollingerBands.Count]; + float[] l=new float[bollingerBands.Count]; + float[] close=new float[bollingerBands.Count]; + +// Line up volatility dates with bollinger bands + DateTime minDate = bollingerBands.Min(x=>x.Date); + DateTime maxDate = bollingerBands.Max(x=>x.Date); + volatilityPrices = new Prices(volatilityPrices.Where(x=>x.Date<=maxDate && x.Date>=minDate).OrderBy(x=>x.Date).ToList()); // most historical date in lowest index + float[] v=volatilityPrices.GetPrices(); + float minV=Numerics.Min(ref v); // get the minimum volatility value + double minP=bollingerBands.Min(x=>x.Close); // get minimum price + double factor=minP/minV; // determine scaling factor + for(int index=0;index=0;index--) + { + BollingerBandElement bollingerBandElement=bollingerBands[index]; + k[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.K)*1000.00f; // put the data in log form + l[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.L)*1000.00f; // put the data in log form + close[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.Close)*1000.00f; // put the data in log form + } + Numerics.ZeroForNaNOrInfinity(ref k); + Numerics.ZeroForNaNOrInfinity(ref l); + Numerics.ZeroForNaNOrInfinity(ref close); + Numerics.ZeroForNaNOrInfinity(ref v); + float maxY=Math.Max(Math.Max(Numerics.Max(ref l),Math.Max(Numerics.Max(ref close),Numerics.Max(ref k))),Numerics.Max(ref v)); + float minY=Math.Min(Math.Min(Numerics.Min(ref l),Math.Min(Numerics.Min(ref close),Numerics.Min(ref k))),Numerics.Min(ref v))-5f; + float maxX=close.Length; + float minX=0.00f; + + Pen pen=new Pen(DrawingBrush,penWidth); + Pen redPen=new Pen(DrawingBrushRed,penWidth); + ImageHelper imageHelper=new ImageHelper(); + + PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY); + imageHelper.CreateImage(Width,Height,pointMapping); + imageHelper.Fill(FillBrush); + + LineSegments lineSegments=new LineSegments(); +// draw volatility + for(int index=0;index + /// Generate Bollinger Band Data + /// + /// + /// + /// + /// private void ProcessBollingerBandData(TestCase testCase,int movingAverageDays,float penWidth,double noise) { int bufferDays=60; @@ -377,6 +546,7 @@ namespace MarketData.CNNProcessing if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile)) { + MDTrace.WriteLine(LogLevel.DEBUG,$"Writing {testCase.LastPathFileName}"); if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName); if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName); else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName); @@ -426,6 +596,7 @@ namespace MarketData.CNNProcessing if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile)) { + MDTrace.WriteLine(LogLevel.DEBUG,$"Writing {testCase.LastPathFileName}"); if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName); if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName); else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName); diff --git a/MarketDataLib/CNNProcessing/TestCase.cs b/MarketDataLib/CNNProcessing/TestCase.cs index 67cae49..93e8325 100644 --- a/MarketDataLib/CNNProcessing/TestCase.cs +++ b/MarketDataLib/CNNProcessing/TestCase.cs @@ -14,7 +14,7 @@ namespace MarketData.CNNProcessing public class TestCase { public enum CaseType{Training,Test,Validation}; - public enum GenerateType{Price,BollingerBand}; + public enum GenerateType{Price,BollingerBand,BollingerBandWithVIX}; public enum OutputType{OutputFile,OutputStream} private readonly List streams=new List(); private readonly List pathFileNames=new List(); diff --git a/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs b/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs index a893435..ad8d0b6 100644 --- a/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs +++ b/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs @@ -5,6 +5,7 @@ using MarketData.MarketDataModel; using MarketData.Utils; using System; using System.Collections.Generic; +using System.IO; using System.Linq; namespace MarketData.Generator.CMMomentum @@ -132,6 +133,35 @@ namespace MarketData.Generator.CMMomentum } return true; } + // This method is made public in order that it can be tested + //public static bool PredictCandidate(CMCandidate cmCandidate,CMParams cmParams) + //{ + // try + // { + // CNNClient cnnClient=new CNNClient(cmParams.UseCNNHost); + // DataProcessor dataProcessor=new DataProcessor(); + // dataProcessor.Width=128; + // dataProcessor.Height=128; + // dataProcessor.PenWidth=1; + // TestCase testCase=new TestCase(cmCandidate.Symbol,cmCandidate.TradeDate,cmParams.UseCNNDayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBand,TestCase.OutputType.OutputStream); + // dataProcessor.ProcessData(testCase); + // String prediction = cnnClient.Predict(CNNClient.Model.resnet50_20241024_270,testCase.LastStream); + // prediction=prediction.Substring(prediction.IndexOf("-->")); + // int result=int.Parse(Utility.BetweenString(prediction,"[[","]")); + // if(1==result) + // { + // cmCandidate.Score*=(1.00+cmParams.UseCNNRewardPercentDecimal); // increase the score by the percentage indicated in the params settings + // cmCandidate.CNNPrediction=true; + // } + // return true; + // } + // catch(Exception exception) + // { + // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Error encountered calling convolutional model at {0}. Exception was {1}",cmParams.UseCNNHost,exception.ToString())); + // return false; + // } + //} + // This method is made public in order that it can be tested public static bool PredictCandidate(CMCandidate cmCandidate,CMParams cmParams) { @@ -139,12 +169,14 @@ namespace MarketData.Generator.CMMomentum { CNNClient cnnClient=new CNNClient(cmParams.UseCNNHost); DataProcessor dataProcessor=new DataProcessor(); - dataProcessor.Width=128; - dataProcessor.Height=128; + int imageDimensions=224; + dataProcessor.Width=imageDimensions; + dataProcessor.Height=imageDimensions; dataProcessor.PenWidth=1; - TestCase testCase=new TestCase(cmCandidate.Symbol,cmCandidate.TradeDate,cmParams.UseCNNDayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBand,TestCase.OutputType.OutputStream); + TestCase testCase=new TestCase(cmCandidate.Symbol,cmCandidate.TradeDate,cmParams.UseCNNDayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBandWithVIX,TestCase.OutputType.OutputStream); dataProcessor.ProcessData(testCase); - String prediction = cnnClient.Predict(CNNClient.Model.resnet50_20241024_270,testCase.LastStream); + Stream streamResult = cnnClient.ProcessImage(testCase.LastStream); // process the image through PIL + String prediction = cnnClient.Predict(CNNClient.Model.convnext,streamResult); prediction=prediction.Substring(prediction.IndexOf("-->")); int result=int.Parse(Utility.BetweenString(prediction,"[[","]")); if(1==result) @@ -160,5 +192,7 @@ namespace MarketData.Generator.CMMomentum return false; } } + + } }