diff --git a/MarketData/MarketDataLib/CNNProcessing/CNNClient.cs b/MarketData/MarketDataLib/CNNProcessing/CNNClient.cs index 5c90402..95b78cf 100755 --- a/MarketData/MarketDataLib/CNNProcessing/CNNClient.cs +++ b/MarketData/MarketDataLib/CNNProcessing/CNNClient.cs @@ -7,7 +7,7 @@ namespace MarketData.CNNProcessing { public class CNNClient { - public enum Model{resnet50,resnet50B,resnet50_20241024_270,inception,vgg16,lenet5,ping}; + public enum Model{resnet50,resnet50B,resnet50_20241024_270,inception,vgg16,lenet5,convnext,ping}; private static readonly string Alive="Alive"; private readonly HttpClient client = new HttpClient(); private string baseUrl; diff --git a/MarketData/MarketDataLib/CNNProcessing/DataProcessor.cs b/MarketData/MarketDataLib/CNNProcessing/DataProcessor.cs index a8de462..8aa00b0 100755 --- a/MarketData/MarketDataLib/CNNProcessing/DataProcessor.cs +++ b/MarketData/MarketDataLib/CNNProcessing/DataProcessor.cs @@ -12,6 +12,7 @@ namespace MarketData.CNNProcessing private String strFolderPath=default; private readonly SKColor colorBlack = new SKColor(0,0,0); private readonly SKColor colorWhite = new SKColor(255,255,255); + private readonly SKColor colorRed = new SKColor(255,0,0); public DataProcessor(int width=640,int height=480) { @@ -19,6 +20,7 @@ namespace MarketData.CNNProcessing Height=height; PenWidth=2f; DrawingBrush = colorBlack; + DrawingBrushRed = colorRed; FillBrush = colorWhite; DrawPrice=true; UseGrayScale=false; @@ -58,6 +60,12 @@ namespace MarketData.CNNProcessing ///Gets/Sets the drawing brush brush public SKColor DrawingBrush{get;set;} + /// + /// DrawingBrushRed + /// + ///Gets/Sets the drawing brush brush + public SKColor DrawingBrushRed{get;set;} + /// /// DrawBlack /// @@ -142,6 +150,28 @@ namespace MarketData.CNNProcessing if(!this.strFolderPath.EndsWith(@"/"))this.strFolderPath=this.strFolderPath+@"/"; } + /// + /// ClearFolderPath + /// + ///The test cases + public void ClearFolderPath() + { + if(String.IsNullOrEmpty(strFolderPath))throw new InvalidDataException($"{nameof(strFolderPath)} cannot be null"); + if(!Directory.Exists(strFolderPath)) + { + Directory.CreateDirectory(strFolderPath); + } + else + { + String[] pathFileNames = Directory.GetFiles(strFolderPath); + Console.WriteLine($"Deleting {pathFileNames.Length} files from {strFolderPath}"); + foreach(String file in pathFileNames) + { + File.Delete(file); + } + } + } + public void ProcessData(TestCases testCases) { for(int index=0;index + /// ProcessBollingerBandDataWithVolatility item - Draws Price, K, L and Volatility + /// + ///Symbol + private void ProcessBollingerBandDataWithVolatility(TestCase testCase,float penWidth,double noise) + { + String symbolVolatility="^VIX"; + DateGenerator dateGenerator=new DateGenerator(); + + int daysInPeriod=dateGenerator.DaysBetweenActual(testCase.PurchaseDate,testCase.HistDate); + daysInPeriod+=60; + Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,daysInPeriod); + Prices volatilityPrices=PricingDA.GetPrices(symbolVolatility,testCase.PurchaseDate,daysInPeriod); + BollingerBands bollingerBands=BollingerBandGenerator.GenerateBollingerBands(prices); // we want to grab K, L, and Close + bollingerBands=new BollingerBands(bollingerBands.Where(x=>x.Date>=testCase.HistDate).ToList()); + float[] k=new float[bollingerBands.Count]; + float[] l=new float[bollingerBands.Count]; + float[] close=new float[bollingerBands.Count]; + +// Line up volatility dates with bollinger bands + DateTime minDate = bollingerBands.Min(x=>x.Date); + DateTime maxDate = bollingerBands.Max(x=>x.Date); + volatilityPrices = new Prices(volatilityPrices.Where(x=>x.Date<=maxDate && x.Date>=minDate).OrderBy(x=>x.Date).ToList()); // most historical date in lowest index + float[] v=volatilityPrices.GetPrices(); + float minV=Numerics.Min(ref v); // get the minimum volatility value + double minP=bollingerBands.Min(x=>x.Close); // get minimum price + double factor=minP/minV; // determine scaling factor + for(int index=0;index=0;index--) + { + BollingerBandElement bollingerBandElement=bollingerBands[index]; + k[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.K)*1000.00f; // put the data in log form + l[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.L)*1000.00f; // put the data in log form + close[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.Close)*1000.00f; // put the data in log form + } + Numerics.ZeroForNaNOrInfinity(ref k); + Numerics.ZeroForNaNOrInfinity(ref l); + Numerics.ZeroForNaNOrInfinity(ref close); + Numerics.ZeroForNaNOrInfinity(ref v); + float maxY=Math.Max(Math.Max(Numerics.Max(ref l),Math.Max(Numerics.Max(ref close),Numerics.Max(ref k))),Numerics.Max(ref v)); + float minY=Math.Min(Math.Min(Numerics.Min(ref l),Math.Min(Numerics.Min(ref close),Numerics.Min(ref k))),Numerics.Min(ref v))-5f; + float maxX=close.Length; + float minX=0.00f; + + // Pen pen=new Pen(DrawingBrush,penWidth); + // Pen redPen=new Pen(DrawingBrushRed,penWidth); + ImageHelper imageHelper=new ImageHelper(); + + PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY); + imageHelper.CreateImage(Width,Height,pointMapping); + imageHelper.Fill(FillBrush); + + LineSegments lineSegments=new LineSegments(); +// draw volatility + for(int index=0;index /// ProcessPriceData item /// @@ -292,6 +445,7 @@ namespace MarketData.CNNProcessing if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile)) { + MDTrace.WriteLine(LogLevel.DEBUG,$"Writing {testCase.LastPathFileName}"); if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName); if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName); else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName); diff --git a/MarketData/MarketDataLib/CNNProcessing/TestCase.cs b/MarketData/MarketDataLib/CNNProcessing/TestCase.cs index 888942c..e2c2ae2 100755 --- a/MarketData/MarketDataLib/CNNProcessing/TestCase.cs +++ b/MarketData/MarketDataLib/CNNProcessing/TestCase.cs @@ -14,7 +14,7 @@ namespace MarketData.CNNProcessing public class TestCase { public enum CaseType{Training,Test,Validation}; - public enum GenerateType{Price,BollingerBand}; + public enum GenerateType{Price,BollingerBand,BollingerBandWithVIX}; public enum OutputType{OutputFile,OutputStream} private readonly List streams=new List(); private readonly List pathFileNames=new List(); diff --git a/MarketData/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs b/MarketData/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs index 9d9f08a..08bae79 100755 --- a/MarketData/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs +++ b/MarketData/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs @@ -129,19 +129,31 @@ namespace MarketData.Generator.CMMomentum } return true; } - // This method is made public in order that it can be tested + /// + /// PredictCandidate - 2026 convnext version - Be sure that the model parameters has 90 days in UseCNNDayCount + /// If we need to revert back to CNNClient.Model.resnet50_20241024_270 then uncomment the below method and update cmParams.UseCNNDayCount=270 + /// + /// + /// + /// public static bool PredictCandidate(CMCandidate cmCandidate,CMParams cmParams) { try { CNNClient cnnClient=new CNNClient(cmParams.UseCNNHost); DataProcessor dataProcessor=new DataProcessor(); - dataProcessor.Width=128; - dataProcessor.Height=128; + int imageDimensions=224; + dataProcessor.Width=imageDimensions; + dataProcessor.Height=imageDimensions; dataProcessor.PenWidth=1; - TestCase testCase=new TestCase(cmCandidate.Symbol,cmCandidate.TradeDate,cmParams.UseCNNDayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBand,TestCase.OutputType.OutputStream); + if(90!=cmParams.UseCNNDayCount) + { + throw new InvalidDataException("CNNClient.Model.convnext must be used with cmParams.UseCNNDayCount=90."); + } + TestCase testCase=new TestCase(cmCandidate.Symbol,cmCandidate.TradeDate,cmParams.UseCNNDayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBandWithVIX,TestCase.OutputType.OutputStream); dataProcessor.ProcessData(testCase); - String prediction = cnnClient.Predict(CNNClient.Model.resnet50_20241024_270,testCase.LastStream); + Stream streamResult = cnnClient.ProcessImage(testCase.LastStream); // process the image through PIL + String prediction = cnnClient.Predict(CNNClient.Model.convnext,streamResult); prediction=prediction.Substring(prediction.IndexOf("-->")); int result=int.Parse(Utility.BetweenString(prediction,"[[","]")); if(1==result) @@ -157,5 +169,34 @@ namespace MarketData.Generator.CMMomentum return false; } } + // Keep this until happy with the new model + // This method is made public in order that it can be tested + // public static bool PredictCandidate(CMCandidate cmCandidate,CMParams cmParams) + // { + // try + // { + // CNNClient cnnClient=new CNNClient(cmParams.UseCNNHost); + // DataProcessor dataProcessor=new DataProcessor(); + // dataProcessor.Width=128; + // dataProcessor.Height=128; + // dataProcessor.PenWidth=1; + // TestCase testCase=new TestCase(cmCandidate.Symbol,cmCandidate.TradeDate,cmParams.UseCNNDayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBand,TestCase.OutputType.OutputStream); + // dataProcessor.ProcessData(testCase); + // String prediction = cnnClient.Predict(CNNClient.Model.resnet50_20241024_270,testCase.LastStream); + // prediction=prediction.Substring(prediction.IndexOf("-->")); + // int result=int.Parse(Utility.BetweenString(prediction,"[[","]")); + // if(1==result) + // { + // cmCandidate.Score*=(1.00+cmParams.UseCNNRewardPercentDecimal); // increase the score by the percentage indicated in the params settings + // cmCandidate.CNNPrediction=true; + // } + // return true; + // } + // catch(Exception exception) + // { + // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Error encountered calling convolutional model at {0}. Exception was {1}",cmParams.UseCNNHost,exception.ToString())); + // return false; + // } + // } } }