diff --git a/Program.cs b/Program.cs index e13e19d..22e6aed 100644 --- a/Program.cs +++ b/Program.cs @@ -211,7 +211,7 @@ namespace CNNImageProcessor List holdings = new List(); DateGenerator dateGenerator = new DateGenerator(); DateTime startDate = DateTime.Parse("10/31/2019"); - DateTime endDate = DateTime.Parse("02/01/2026"); + DateTime endDate = DateTime.Parse("12/31/2025"); DateTime actualEndDate = endDate; DateTime analysisDate = DateTime.Now; @@ -282,7 +282,6 @@ namespace CNNImageProcessor } } - public static List ReadHoldings(String strPathFileName) { String strLine; @@ -301,6 +300,20 @@ namespace CNNImageProcessor return universe; } + public static void WriteHoldings(List holdings,String strPathFileName) + { + if(File.Exists(strPathFileName))File.Delete(strPathFileName); + StreamWriter outStream = new StreamWriter(strPathFileName); + outStream.WriteLine(Holding.Heading); + foreach(Holding holding in holdings) + { + outStream.WriteLine(holding); + } + outStream.Flush(); + outStream.Close(); + outStream.Dispose(); + } + public static (List avoid, List good) GenerateCodeTestCases(List universe) { double validationPercent=0.05; @@ -335,38 +348,56 @@ namespace CNNImageProcessor public static void GenerateTrainingImages(List avoid, List good) { + String cnnClientUrl="http://127.0.0.1:5000"; int imageSize=224; - int dayCount=90; // 90 - Console.WriteLine($"Generate training into {@"C:\Data"}"); - CNNProcessor.GenerateTraining(avoid, good, imageSize,dayCount, TestCase.GenerateType.BollingerBandWithVIX,@"C:\Data"); - ClearFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\0"); - ClearFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\1"); - if(!ProcessImages(@"C:\Data\0",@"C:\boneyard\DeepLearning\ModelInputData\0",imageSize)) // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\0 + int dayCount=90; + Console.WriteLine($"Generate training into {@"C:\boneyard\DeepLearning\ModelInputData"}"); + CNNProcessor.GenerateTraining(avoid, good, imageSize,dayCount, TestCase.GenerateType.BollingerBandWithVIX,@"C:\boneyard\DeepLearning\ModelInputData"); + ClearFolderPath(@"C:\boneyard\DeepLearning\Data\0"); + ClearFolderPath(@"C:\boneyard\DeepLearning\Data\1"); + CNNClient cnnClient = new CNNClient(cnnClientUrl); + if(!cnnClient.Ping()) { - Console.WriteLine($"Process image failed, is the server running?"); + Console.WriteLine($"CNN Server @ {cnnClientUrl} is not responding."); + return; } - if(!ProcessImages(@"C:\Data\1",@"C:\boneyard\DeepLearning\ModelInputData\1",imageSize)) // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\1 - { - Console.WriteLine($"Process image failed, is the server running?"); - } - Console.WriteLine("Please copy these files into the training folder."); + ProcessImages(@"C:\boneyard\DeepLearning\ModelInputData\0",@"C:\boneyard\DeepLearning\Data\0",imageSize,cnnClientUrl); // Process through PIL and put in C:\boneyard\DeepLearning\Data\0 + ProcessImages(@"C:\boneyard\DeepLearning\ModelInputData\1",@"C:\boneyard\DeepLearning\Data\1",imageSize,cnnClientUrl); // Process through PIL and put in C:\boneyard\DeepLearning\Data\1 + Console.WriteLine("Done."); } /// - /// This will generate images into C:\boneyard\DeepLearning\ModelInputData\0 and C:\boneyard\DeepLearning\ModelInputData\1 - /// You should then copy the generated images into C:\boneyard\DeepLearning\Data folder and then proceed to train tbe latest model - /// which at the time of writing this is model_sk_convnext_v1.py. After running the model you shoukd then run - /// verify_model_sk_convnext_v1.py. This will produce a validation score which at the time of writing is 99%. It will also produce - /// some output images including the confusion matrix. + /// This will generate images into C:\boneyard\DeepLearning\Data\0 and C:\boneyard\DeepLearning\Data\1 + /// You should then proceed to train tbe latest model which at the time of writing this is model_sk_convnext_v1.py. + /// After running the model you should then run verify_model_sk_convnext_v1.py. This will produce a validation score + /// which at the time of writing is 99%. It will also produce some output images including the confusion matrix. + /// + /// I am using WSL2 to perform the training because WSL2 is the only option for tensorflow with GPU. + /// To launch WSL2 at a command prompt type "wsl ~". If the enviroment is not set up then you can use the setup_tf_gpu.sh + /// script in CNN/Scripts folder to re-create the full Python3 environment and Tensorflow. The script will create the + /// virtual environment and install everything. It was used to create the current WSL enviroment. + /// To start the environment "source tf_gpu/bin/activate" + /// then type "code ." This will run VSCODE and attach to the WSL environment. + /// Train the model on EUPORIE laptop using the GPU card with WSL2. (Windows Subsystem For Linux). I am running Ubuntu1 22.04.2 + /// To launch WSL open up a command prompt, run powershell and type "wsl ~". + /// The folder structure will be /home/pi/CNN /home/pi/DeepLearning + /// You can access the folder structure through windows explorer. type "\\wsl$" in explorer and navigate to the folder. + /// drop in the Data and Model and run the model. + /// There is a shell script in the Scripts folder of the CNN project. setup_tf_gpu.sh + /// Copy the script to the CNN folder and run it from the CNN folder. + /// It will create the venv enviroment and install python 3.10 and tensorflow (gpu) + /// + /// ******************************************************************************************************************** /// /// /// static void Main(string[] args) { // The modified flow - //List holdings = GenerateTrades(); // generate a holding set from the CMMomentum monthly candidates - List holdings = ReadHoldings("holdings.csv"); // read a holding set that was previously generated + // List holdings = GenerateTrades(); // generate a new holding set from the CMMomentum monthly candidates + // WriteHoldings(holdings,"holdings.csv"); // save the list. The saved list can be read back in to save time in case reruns are necessary + List holdings = ReadHoldings("holdings.csv"); // read a holding set that was previously generated. You'll want to create a new set of holdings for retraiing (List avoid, List good)=GenerateCodeTestCases(holdings); // split the dataset into avoid and good - GenerateTrainingImages(avoid, good); + GenerateTrainingImages(avoid, good); // Generate the training images // Clear cache at the end GBPriceCache.GetInstance().Dispose(); }