diff --git a/MarketDataLib/CNNProcessing/CNNClient.cs b/MarketDataLib/CNNProcessing/CNNClient.cs
index 616f377..6e9d467 100644
--- a/MarketDataLib/CNNProcessing/CNNClient.cs
+++ b/MarketDataLib/CNNProcessing/CNNClient.cs
@@ -7,7 +7,7 @@ namespace MarketData.CNNProcessing
{
public class CNNClient
{
- public enum Model{resnet50,resnet50B,resnet50_20241024_270,inception,vgg16,lenet5,ping};
+ public enum Model{resnet50,resnet50B,resnet50_20241024_270,inception,vgg16,lenet5,convnext,ping};
private static readonly string Alive="Alive";
private readonly HttpClient client = new HttpClient();
private string baseUrl;
diff --git a/MarketDataLib/CNNProcessing/CNNProcessor.cs b/MarketDataLib/CNNProcessing/CNNProcessor.cs
index a13b02b..5e8e7a6 100644
--- a/MarketDataLib/CNNProcessing/CNNProcessor.cs
+++ b/MarketDataLib/CNNProcessing/CNNProcessor.cs
@@ -3,27 +3,75 @@ using System.IO;
using System.Collections.Generic;
using MarketData.Utils;
using System.Text;
+using System.Globalization;
namespace MarketData.CNNProcessing
{
public class CNNProcessor
{
- private static int dayCount=270;
- private static int width=128;
- private static int height=128;
+ private static int dayCount=270; // This is the default days
+ private static int width=128; // This is the default width
+ private static int height=128; // THis is the defaukt height
private CNNProcessor()
{
}
- public static void GenerateTraining()
+ ///
+ /// GenerateTraining - This is the new one. Please refer to the CNNImageProcessor project for information on how to call this method.
+ ///
+ /// This is the collection of avoid holdings
+ /// This is the collection of good holdings
+ /// The image dimensions. for example 224 for 224x224 or 128 for 128x128
+ /// This is the number of histDays. For example I used 90 for convnext
+ /// The type. For example I used BollingerBandWithVIX which is a bollinger band with ^VIX overay for convnext
+ ///
+ public static void GenerateTraining(List avoid, List good, int dimension, int histDays,TestCase.GenerateType generateType=TestCase.GenerateType.BollingerBandWithVIX,String rootFolder=@"C:\boneyard\DeepLearning\ModelInputData\")
+ {
+ TestCases testCases=new TestCases();
+ DataProcessor dataProcessor=new DataProcessor();
+ dataProcessor.Width=dimension;
+ dataProcessor.Height=dimension;
+ dataProcessor.PenWidthArray=new float[]{.75f,1.00f,1.12f};
+
+ if(!rootFolder.EndsWith(@"\"))rootFolder+=@"\";
+// [0] Data - The avoid data
+ foreach(Holding holding in avoid)
+ {
+ testCases.Add(new TestCase(holding.Symbol,holding.PurchaseDate,histDays,TestCase.CaseType.Training,generateType));
+ }
+ dataProcessor.SetOutputFolderPath(rootFolder+"0");
+ dataProcessor.ClearFolderPath();
+ dataProcessor.ProcessData(testCases);
+ testCases.Clear();
+
+// [1] Data - The good data
+ foreach(Holding holding in good)
+ {
+ testCases.Add(new TestCase(holding.Symbol,holding.PurchaseDate,histDays,TestCase.CaseType.Training,generateType));
+ }
+ dataProcessor.SetOutputFolderPath(rootFolder+"1");
+ dataProcessor.ClearFolderPath();
+ dataProcessor.ProcessData(testCases);
+ }
+
+ ///
+ /// GenerateTraining - This is the old methof training the resnet model. Please see above
+ ///
+ ///
+ public static void GenerateTraining(String rootFolder=@"C:\boneyard\DeepLearning\ModelInputData\")
{
TestCases testCases=new TestCases();
DataProcessor dataProcessor=new DataProcessor();
dataProcessor.Width=width;
dataProcessor.Height=height;
- dataProcessor.PenWidthArray=new float[]{.50f,.75f,1.00f,1.12f,1.25f,1.31f,1.37f,1.50f,1.56f,1.62f,1.75f,1.87f,2.00f};
+ // dataProcessor.PenWidthArray=new float[]{.50f,.75f,1.00f,1.12f,1.25f,1.31f,1.37f,1.50f,1.56f,1.62f,1.75f,1.87f,2.00f};
+ // Testing with 20,000 images in each set so reducing this use of pens to just one. It was producing 260,000 images for each classification,
+ // takings many hours to build the datasets
+ dataProcessor.PenWidthArray=new float[]{.75f,1.00f,1.12f};
+
+ if(!rootFolder.EndsWith(@"\"))rootFolder+=@"\";
// [0] Data - The avoid data
testCases.Add(new TestCase("CENX",DateTime.Parse("03/31/2022"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("ICPT",DateTime.Parse("12/31/2019"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
@@ -56,8 +104,8 @@ namespace MarketData.CNNProcessing
testCases.Add(new TestCase("INBX",DateTime.Parse("01/31/2024"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("WYNN",DateTime.Parse("02/28/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
-
- dataProcessor.SetOutputFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\0");
+// ****
+ dataProcessor.SetOutputFolderPath(rootFolder+"0");
dataProcessor.ProcessData(testCases);
testCases.Clear();
@@ -102,7 +150,8 @@ namespace MarketData.CNNProcessing
testCases.Add(new TestCase("DOCU",DateTime.Parse("05/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("SIG",DateTime.Parse("10/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
- dataProcessor.SetOutputFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\1");
+// ***
+ dataProcessor.SetOutputFolderPath(rootFolder+"1");
dataProcessor.ProcessData(testCases);
}
@@ -204,4 +253,76 @@ namespace MarketData.CNNProcessing
Console.WriteLine("");
}
}
+
+ public class Holding
+ {
+ public String Symbol {get;set;}
+ public DateTime PurchaseDate {get; set; }
+ public double PurchasePrice {get;set;}
+ public DateTime SellDate {get; set; }
+ public double SellPrice {get;set;}
+ public double GainLoss{ get; set;}
+ public double GainLossPercent {get;set;}
+ private static readonly string[] DateFormats = { "MM/dd/yyyy", "M/dd/yyyy", "M/d/yyyy" };
+ private static readonly CultureInfo UsCulture = CultureInfo.GetCultureInfo("en-US");
+
+ public static String Heading
+ {
+ get
+ {
+ return "Symbol,Shares,Purchase Date,Purchase Price,Sell Date,Sell Price,Exposure,Beta,BetaMonths,SharpeRatio,RiskAdjustedWeight,RiskAdjustedAllocation,TargetBetaOverBeta,Score,CNN Prediction,Market Value,Gain Loss,Gain Loss (%)";
+ }
+ }
+
+ public String ToTestCase()
+ {
+ StringBuilder sb = new StringBuilder();
+ sb.Append("testCases.Add(new TestCase(").Append("\"").Append(Symbol).Append("\"").Append(",");
+ sb.Append("DateTime.Parse(").Append("\"").Append(Utility.DateTimeToStringMMSDDSYYYY(PurchaseDate)).Append("\")").Append(",");
+ sb.Append("270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));");
+ return sb.ToString();
+ }
+
+ public override String ToString()
+ {
+ StringBuilder sb = new StringBuilder();
+ sb.Append(Symbol).Append(",");
+ sb.Append(","); // shares
+ sb.Append(PurchaseDate.ToShortDateString()).Append(",");
+ sb.Append(Utility.FormatNumber(PurchasePrice,3)).Append(",");
+ sb.Append(SellDate.ToShortDateString()).Append(",");
+ sb.Append(Utility.FormatNumber(SellPrice,3)).Append(",");
+ sb.Append(","); //exposure
+ sb.Append(","); //beta
+ sb.Append(","); //bta months
+ sb.Append(","); //sharpe ratio
+ sb.Append(","); //risk adjusted weight
+ sb.Append(","); //RiskAdjustedAllocation
+ sb.Append(","); //TargetBetaOverBeta
+ sb.Append(","); //Score
+ sb.Append(","); //CNNPrediction
+ sb.Append(","); //Market Value
+ sb.Append(Utility.FormatNumber(GainLoss,3)).Append(",");
+ sb.Append(Utility.FormatNumber(GainLossPercent,3));
+ return sb.ToString();
+ }
+
+
+ public static Holding FromString(string strLine)
+ {
+ string[] items = strLine.Split(',');
+
+ Holding holding = new Holding();
+ holding.Symbol = items[0];
+ if(string.IsNullOrEmpty(holding.Symbol))return null;
+ holding.PurchaseDate = DateTime.ParseExact(items[2], DateFormats, UsCulture, DateTimeStyles.AssumeLocal);
+ holding.PurchasePrice = double.Parse(items[3], UsCulture);
+ holding.SellDate = DateTime.ParseExact(items[4], DateFormats, UsCulture, DateTimeStyles.AssumeLocal);
+ holding.SellPrice = double.Parse(items[5], UsCulture);
+ holding.GainLoss = double.Parse(items[16], UsCulture);
+ holding.GainLossPercent = double.Parse(items[17].TrimEnd('%'), UsCulture) / 100.0;
+
+ return holding;
+ }
+ }
}
diff --git a/MarketDataLib/CNNProcessing/DataProcessor.cs b/MarketDataLib/CNNProcessing/DataProcessor.cs
index 5ed95a1..fe6fbb2 100644
--- a/MarketDataLib/CNNProcessing/DataProcessor.cs
+++ b/MarketDataLib/CNNProcessing/DataProcessor.cs
@@ -24,6 +24,7 @@ namespace MarketData.CNNProcessing
Height=height;
PenWidth=2f;
DrawingBrush=new SolidBrush(Color.Black);
+ DrawingBrushRed=new SolidBrush(Color.Red);
FillBrush=new SolidBrush(Color.White);
DrawPrice=true;
UseGrayScale=false;
@@ -59,6 +60,11 @@ namespace MarketData.CNNProcessing
///
///Gets/Sets the drawing brush brush
public Brush DrawingBrush{get;set;}
+ ///
+ /// DrawingBrush
+ ///
+ ///Gets/Sets the drawing brush brush
+ public Brush DrawingBrushRed{get;set;}
///
/// DrawBlack
@@ -143,6 +149,29 @@ namespace MarketData.CNNProcessing
this.strFolderPath=strFolderPath;
if(!this.strFolderPath.EndsWith(@"\"))this.strFolderPath=this.strFolderPath+@"\";
}
+
+ ///
+ /// ClearFolderPath
+ ///
+ ///The test cases
+ public void ClearFolderPath()
+ {
+ if(String.IsNullOrEmpty(strFolderPath))throw new InvalidDataException($"{nameof(strFolderPath)} cannot be null");
+ if(!Directory.Exists(strFolderPath))
+ {
+ Directory.CreateDirectory(strFolderPath);
+ }
+ else
+ {
+ String[] pathFileNames = Directory.GetFiles(strFolderPath);
+ Console.WriteLine($"Deleting {pathFileNames.Length} files from {strFolderPath}");
+ foreach(String file in pathFileNames)
+ {
+ File.Delete(file);
+ }
+ }
+ }
+
public void ProcessData(TestCases testCases)
{
for(int index=0;index
+/// ProcessBollingerBandData item - Draws Price, K, L and Volatility
+///
+///Symbol
+ private void ProcessBollingerBandDataWithVolatility(TestCase testCase,float penWidth,double noise)
+ {
+ String symbolVolatility="^VIX";
+ DateGenerator dateGenerator=new DateGenerator();
+
+ int daysInPeriod=dateGenerator.DaysBetweenActual(testCase.PurchaseDate,testCase.HistDate);
+ daysInPeriod+=60;
+ Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,daysInPeriod);
+ Prices volatilityPrices=PricingDA.GetPrices(symbolVolatility,testCase.PurchaseDate,daysInPeriod);
+ BollingerBands bollingerBands=BollingerBandGenerator.GenerateBollingerBands(prices); // we want to grab K, L, and Close
+ bollingerBands=new BollingerBands(bollingerBands.Where(x=>x.Date>=testCase.HistDate).ToList());
+ float[] k=new float[bollingerBands.Count];
+ float[] l=new float[bollingerBands.Count];
+ float[] close=new float[bollingerBands.Count];
+
+// Line up volatility dates with bollinger bands
+ DateTime minDate = bollingerBands.Min(x=>x.Date);
+ DateTime maxDate = bollingerBands.Max(x=>x.Date);
+ volatilityPrices = new Prices(volatilityPrices.Where(x=>x.Date<=maxDate && x.Date>=minDate).OrderBy(x=>x.Date).ToList()); // most historical date in lowest index
+ float[] v=volatilityPrices.GetPrices();
+ float minV=Numerics.Min(ref v); // get the minimum volatility value
+ double minP=bollingerBands.Min(x=>x.Close); // get minimum price
+ double factor=minP/minV; // determine scaling factor
+ for(int index=0;index=0;index--)
+ {
+ BollingerBandElement bollingerBandElement=bollingerBands[index];
+ k[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.K)*1000.00f; // put the data in log form
+ l[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.L)*1000.00f; // put the data in log form
+ close[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.Close)*1000.00f; // put the data in log form
+ }
+ Numerics.ZeroForNaNOrInfinity(ref k);
+ Numerics.ZeroForNaNOrInfinity(ref l);
+ Numerics.ZeroForNaNOrInfinity(ref close);
+ Numerics.ZeroForNaNOrInfinity(ref v);
+ float maxY=Math.Max(Math.Max(Numerics.Max(ref l),Math.Max(Numerics.Max(ref close),Numerics.Max(ref k))),Numerics.Max(ref v));
+ float minY=Math.Min(Math.Min(Numerics.Min(ref l),Math.Min(Numerics.Min(ref close),Numerics.Min(ref k))),Numerics.Min(ref v))-5f;
+ float maxX=close.Length;
+ float minX=0.00f;
+
+ Pen pen=new Pen(DrawingBrush,penWidth);
+ Pen redPen=new Pen(DrawingBrushRed,penWidth);
+ ImageHelper imageHelper=new ImageHelper();
+
+ PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY);
+ imageHelper.CreateImage(Width,Height,pointMapping);
+ imageHelper.Fill(FillBrush);
+
+ LineSegments lineSegments=new LineSegments();
+// draw volatility
+ for(int index=0;index
+ /// Generate Bollinger Band Data
+ ///
+ ///
+ ///
+ ///
+ ///
private void ProcessBollingerBandData(TestCase testCase,int movingAverageDays,float penWidth,double noise)
{
int bufferDays=60;
@@ -377,6 +546,7 @@ namespace MarketData.CNNProcessing
if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile))
{
+ MDTrace.WriteLine(LogLevel.DEBUG,$"Writing {testCase.LastPathFileName}");
if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName);
if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName);
else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName);
@@ -426,6 +596,7 @@ namespace MarketData.CNNProcessing
if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile))
{
+ MDTrace.WriteLine(LogLevel.DEBUG,$"Writing {testCase.LastPathFileName}");
if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName);
if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName);
else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName);
diff --git a/MarketDataLib/CNNProcessing/TestCase.cs b/MarketDataLib/CNNProcessing/TestCase.cs
index 67cae49..93e8325 100644
--- a/MarketDataLib/CNNProcessing/TestCase.cs
+++ b/MarketDataLib/CNNProcessing/TestCase.cs
@@ -14,7 +14,7 @@ namespace MarketData.CNNProcessing
public class TestCase
{
public enum CaseType{Training,Test,Validation};
- public enum GenerateType{Price,BollingerBand};
+ public enum GenerateType{Price,BollingerBand,BollingerBandWithVIX};
public enum OutputType{OutputFile,OutputStream}
private readonly List streams=new List();
private readonly List pathFileNames=new List();
diff --git a/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs b/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs
index a893435..ad8d0b6 100644
--- a/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs
+++ b/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs
@@ -5,6 +5,7 @@ using MarketData.MarketDataModel;
using MarketData.Utils;
using System;
using System.Collections.Generic;
+using System.IO;
using System.Linq;
namespace MarketData.Generator.CMMomentum
@@ -132,6 +133,35 @@ namespace MarketData.Generator.CMMomentum
}
return true;
}
+ // This method is made public in order that it can be tested
+ //public static bool PredictCandidate(CMCandidate cmCandidate,CMParams cmParams)
+ //{
+ // try
+ // {
+ // CNNClient cnnClient=new CNNClient(cmParams.UseCNNHost);
+ // DataProcessor dataProcessor=new DataProcessor();
+ // dataProcessor.Width=128;
+ // dataProcessor.Height=128;
+ // dataProcessor.PenWidth=1;
+ // TestCase testCase=new TestCase(cmCandidate.Symbol,cmCandidate.TradeDate,cmParams.UseCNNDayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBand,TestCase.OutputType.OutputStream);
+ // dataProcessor.ProcessData(testCase);
+ // String prediction = cnnClient.Predict(CNNClient.Model.resnet50_20241024_270,testCase.LastStream);
+ // prediction=prediction.Substring(prediction.IndexOf("-->"));
+ // int result=int.Parse(Utility.BetweenString(prediction,"[[","]"));
+ // if(1==result)
+ // {
+ // cmCandidate.Score*=(1.00+cmParams.UseCNNRewardPercentDecimal); // increase the score by the percentage indicated in the params settings
+ // cmCandidate.CNNPrediction=true;
+ // }
+ // return true;
+ // }
+ // catch(Exception exception)
+ // {
+ // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Error encountered calling convolutional model at {0}. Exception was {1}",cmParams.UseCNNHost,exception.ToString()));
+ // return false;
+ // }
+ //}
+
// This method is made public in order that it can be tested
public static bool PredictCandidate(CMCandidate cmCandidate,CMParams cmParams)
{
@@ -139,12 +169,14 @@ namespace MarketData.Generator.CMMomentum
{
CNNClient cnnClient=new CNNClient(cmParams.UseCNNHost);
DataProcessor dataProcessor=new DataProcessor();
- dataProcessor.Width=128;
- dataProcessor.Height=128;
+ int imageDimensions=224;
+ dataProcessor.Width=imageDimensions;
+ dataProcessor.Height=imageDimensions;
dataProcessor.PenWidth=1;
- TestCase testCase=new TestCase(cmCandidate.Symbol,cmCandidate.TradeDate,cmParams.UseCNNDayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBand,TestCase.OutputType.OutputStream);
+ TestCase testCase=new TestCase(cmCandidate.Symbol,cmCandidate.TradeDate,cmParams.UseCNNDayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBandWithVIX,TestCase.OutputType.OutputStream);
dataProcessor.ProcessData(testCase);
- String prediction = cnnClient.Predict(CNNClient.Model.resnet50_20241024_270,testCase.LastStream);
+ Stream streamResult = cnnClient.ProcessImage(testCase.LastStream); // process the image through PIL
+ String prediction = cnnClient.Predict(CNNClient.Model.convnext,streamResult);
prediction=prediction.Substring(prediction.IndexOf("-->"));
int result=int.Parse(Utility.BetweenString(prediction,"[[","]"));
if(1==result)
@@ -160,5 +192,7 @@ namespace MarketData.Generator.CMMomentum
return false;
}
}
+
+
}
}