Code cleanup

.gitignore

@@ -1,6 +1,7 @@
-**/obj/
+**/obj/
 **/bin/
 **/.vs/
 bin
 obj
 .vs
+/obj/Debug

Program.cs

@@ -63,7 +63,7 @@ namespace CNNImageProcessor
     /// </summary>
     /// <param name="sourcePath"></param>
     /// <param name="destinationPath"></param>
-    public static bool ProcessImages(String sourcePath, String destinationPath,int resizeTo,String cnnClientUrl="http://10.0.0.73:5000")
+    public static bool ProcessImages(String sourcePath, String destinationPath,int resizeTo,String cnnClientUrl="http://10.0.0.73:5000",bool useGrayScale=false)
     {
       String[] files = Directory.GetFiles(sourcePath,"*.jpg");
 
@@ -78,7 +78,7 @@ namespace CNNImageProcessor
         Console.WriteLine($"Processing {file}");
         ImageHelper imageHelper=new ImageHelper();
         imageHelper.LoadImage(file);
-//        imageHelper.ToGrayScale();
+        if(useGrayScale)imageHelper.ToGrayScale();
         imageHelper.Resize(resizeTo,resizeTo);
         Stream stream = imageHelper.ToStream();
         Stream processed = cnnClient.ProcessImage(stream);
@@ -211,7 +211,7 @@ namespace CNNImageProcessor
       List<Holding> holdings = new List<Holding>();
       DateGenerator dateGenerator = new DateGenerator();
       DateTime startDate = DateTime.Parse("10/31/2019");
-      DateTime endDate =  DateTime.Parse("02/01/2026");
+      DateTime endDate =  DateTime.Parse("12/31/2025");
       DateTime actualEndDate = endDate; 
       DateTime analysisDate = DateTime.Now;
 
@@ -282,7 +282,6 @@ namespace CNNImageProcessor
       }
 		}
 
-
     public static List<Holding> ReadHoldings(String strPathFileName)
     {
       String strLine;
@@ -301,10 +300,55 @@ namespace CNNImageProcessor
       return universe;
     }
 
+    public static void WriteHoldings(List<Holding> holdings,String strPathFileName)
+    {
+      if(File.Exists(strPathFileName))File.Delete(strPathFileName);
+      StreamWriter outStream = new StreamWriter(strPathFileName);
+      outStream.WriteLine(Holding.Heading);
+      foreach(Holding holding in holdings)
+      {
+        outStream.WriteLine(holding);
+      }
+      outStream.Flush();
+      outStream.Close();
+      outStream.Dispose();
+    }
+
+    //public static (List<Holding> avoid, List<Holding> good) GenerateCodeTestCases(List<Holding> universe)
+    //{
+    //  double validationPercent=0.05;
+    //  double validationPercentUnseen=0.50;
+    //  Console.WriteLine($"Read {universe.Count} holdings");
+
+    //  List<Holding> avoid = universe.Where(x=>x.GainLoss<-.05).ToList();
+    //  List<Holding> good=universe.Where(x=>x.GainLoss>.05).ToList();
+
+    //  int validationCount = (int)(validationPercent * universe.Count);
+
+    //  Random rng = new Random();
+    //  List<Holding> goodValidation = good.OrderBy(x => rng.Next()).Take(validationCount).ToList();
+    //  int goodUnseenCount = (int)(validationPercentUnseen * goodValidation.Count);
+    //  List<Holding> goodValidationUnseen = goodValidation.OrderBy(x => rng.Next()).Take(goodUnseenCount).ToList();
+    //  good.RemoveAll(x => goodValidationUnseen.Contains(x));
+    //  Console.WriteLine($"Validation sample size: {goodValidation.Count}");
+    //  Console.WriteLine($"Unseen validation removed from good: {goodValidationUnseen.Count}");
+    //  Console.WriteLine($"Remaining good count: {good.Count}");
+
+
+    //  List<Holding> avoidValidation = avoid.OrderBy(x => rng.Next()).Take(validationCount).ToList();
+    //  int avoidUnseenCount = (int)(validationPercentUnseen * avoidValidation.Count);
+    //  List<Holding> avoidValidationUnseen = avoidValidation.OrderBy(x => rng.Next()).Take(avoidUnseenCount).ToList();
+    //  avoid.RemoveAll(x => avoidValidationUnseen.Contains(x));
+    //  Console.WriteLine($"Validation sample size: {avoidValidation.Count}");
+    //  Console.WriteLine($"Unseen validation removed from avoid: {avoidValidationUnseen.Count}");
+    //  Console.WriteLine($"Remaining avoid count: {avoid.Count}");
+
+    //  return (avoid, good);
+    //}
+
     public static (List<Holding> avoid, List<Holding> good) GenerateCodeTestCases(List<Holding> universe)
     {
       double validationPercent=0.05;
-      double validationPercentUnseen=0.50;
       Console.WriteLine($"Read {universe.Count} holdings");
 
       List<Holding> avoid = universe.Where(x=>x.GainLoss<-.05).ToList();
@@ -314,59 +358,70 @@ namespace CNNImageProcessor
 
       Random rng = new Random();
       List<Holding> goodValidation = good.OrderBy(x => rng.Next()).Take(validationCount).ToList();
-      int goodUnseenCount = (int)(validationPercentUnseen * goodValidation.Count);
-      List<Holding> goodValidationUnseen = goodValidation.OrderBy(x => rng.Next()).Take(goodUnseenCount).ToList();
-      good.RemoveAll(x => goodValidationUnseen.Contains(x));
       Console.WriteLine($"Validation sample size: {goodValidation.Count}");
-      Console.WriteLine($"Unseen validation removed from good: {goodValidationUnseen.Count}");
       Console.WriteLine($"Remaining good count: {good.Count}");
 
 
       List<Holding> avoidValidation = avoid.OrderBy(x => rng.Next()).Take(validationCount).ToList();
-      int avoidUnseenCount = (int)(validationPercentUnseen * avoidValidation.Count);
-      List<Holding> avoidValidationUnseen = avoidValidation.OrderBy(x => rng.Next()).Take(avoidUnseenCount).ToList();
-      avoid.RemoveAll(x => avoidValidationUnseen.Contains(x));
       Console.WriteLine($"Validation sample size: {avoidValidation.Count}");
-      Console.WriteLine($"Unseen validation removed from avoid: {avoidValidationUnseen.Count}");
       Console.WriteLine($"Remaining avoid count: {avoid.Count}");
 
       return (avoid, good);
     }
 
+
     public static void GenerateTrainingImages(List<Holding> avoid, List<Holding> good)
     {
+      String cnnClientUrl="http://127.0.0.1:5000";
       int imageSize=224;
-      int dayCount=90;  // 90
+      int dayCount=90;  
-      Console.WriteLine($"Generate training into {@"C:\Data"}");
+      Console.WriteLine($"Generate training into {@"C:\boneyard\DeepLearning\ModelInputData"}");
-      CNNProcessor.GenerateTraining(avoid, good, imageSize,dayCount, TestCase.GenerateType.BollingerBandWithVIX,@"C:\Data");
+      CNNProcessor.GenerateTraining(avoid, good, imageSize,dayCount, TestCase.GenerateType.BollingerBandWithVIX,@"C:\boneyard\DeepLearning\ModelInputData");
-      ClearFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\0");
+      ClearFolderPath(@"C:\boneyard\DeepLearning\Data\0");
-      ClearFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\1");
+      ClearFolderPath(@"C:\boneyard\DeepLearning\Data\1");
-      if(!ProcessImages(@"C:\Data\0",@"C:\boneyard\DeepLearning\ModelInputData\0",imageSize))  // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\0
+      CNNClient cnnClient = new CNNClient(cnnClientUrl);
+      if(!cnnClient.Ping())
       {
-        Console.WriteLine($"Process image failed, is the server running?");
+        Console.WriteLine($"CNN Server @ {cnnClientUrl} is not responding.");
+        return;
       }
-      if(!ProcessImages(@"C:\Data\1",@"C:\boneyard\DeepLearning\ModelInputData\1",imageSize))  // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\1                                                                             
+      ProcessImages(@"C:\boneyard\DeepLearning\ModelInputData\0",@"C:\boneyard\DeepLearning\Data\0",imageSize,cnnClientUrl,false);  // Process through PIL and put in C:\boneyard\DeepLearning\Data\0
-      {
+      ProcessImages(@"C:\boneyard\DeepLearning\ModelInputData\1",@"C:\boneyard\DeepLearning\Data\1",imageSize,cnnClientUrl,false);  // Process through PIL and put in C:\boneyard\DeepLearning\Data\1                                                                             
-        Console.WriteLine($"Process image failed, is the server running?");
+      Console.WriteLine("Done.");
-      }
-      Console.WriteLine("Please copy these files into the training folder.");
     }
 
     /// <summary>
-    /// This will generate images into  C:\boneyard\DeepLearning\ModelInputData\0 and  C:\boneyard\DeepLearning\ModelInputData\1
+    /// This will generate images into  C:\boneyard\DeepLearning\Data\0 and  C:\boneyard\DeepLearning\Data\1
-    /// You should then copy the generated images into C:\boneyard\DeepLearning\Data folder and then proceed to train tbe latest model
+    /// You should then proceed to train tbe latest model which at the time of writing this is model_sk_convnext_v1.py.  
-    /// which at the time of writing this is model_sk_convnext_v1.py.  After running the model you shoukd then run
+    /// After running the model you should then run verify_model_sk_convnext_v1.py.  This will produce a validation score 
-    /// verify_model_sk_convnext_v1.py.  This will produce a validation score which at the time of writing is 99%.  It will also produce
+    /// which at the time of writing is 99%.  It will also produce some output images including the confusion matrix.
-    /// some output images including the confusion matrix.
+    /// 
+    /// I am using WSL2 to perform the training because WSL2 is the only option for tensorflow with GPU.
+    /// To launch WSL2 at a command prompt type "wsl ~".   If the enviroment is not set up then you can use the setup_tf_gpu.sh
+    /// script in CNN/Scripts folder to re-create the full Python3 environment and Tensorflow.  The script will create the 
+    /// virtual environment and install everything.  It was used to create the current WSL enviroment.
+    /// To start the environment "source tf_gpu/bin/activate"
+    /// then type "code ."  This will run VSCODE and attach to the WSL environment.
+    /// Train the model on EUPORIE laptop using the GPU card with WSL2.  (Windows Subsystem For Linux).  I am running Ubuntu1 22.04.2
+    /// To launch WSL open up a command prompt, run powershell and type "wsl ~".
+    /// The folder structure will be /home/pi/CNN  /home/pi/DeepLearning
+    /// You can access the folder structure through windows explorer.  type "\\wsl$" in explorer and navigate to the folder.
+    /// drop in the Data and Model and run the model.
+    /// There is a shell script in the Scripts folder of the CNN project. setup_tf_gpu.sh  
+    /// Copy the script to the CNN folder and run it from the CNN folder.  
+    /// It will create the venv enviroment and install python 3.10 and tensorflow (gpu)
+    /// 
+    /// ********************************************************************************************************************    /// 
     /// </summary>
     /// <param name="args"></param>
     static void Main(string[] args)
     {
 // The modified flow
-      //List<Holding> holdings = GenerateTrades();   // generate a holding set from the CMMomentum monthly candidates
+ //     List<Holding> holdings = GenerateTrades();   // generate a new holding set from the CMMomentum monthly candidates
-      List<Holding> holdings = ReadHoldings("holdings.csv");  // read a holding set that was previously generated
+ //     WriteHoldings(holdings,"holdings.csv");      // save the list.  The saved list can be read back in to save time in case reruns are necessary
+      List<Holding> holdings = ReadHoldings("holdings.csv");  // read a holding set that was previously generated.  You'll want to create a new set of holdings for retraiing
       (List<Holding> avoid, List<Holding> good)=GenerateCodeTestCases(holdings);   // split the dataset into avoid and good
-      GenerateTrainingImages(avoid, good);
+      GenerateTrainingImages(avoid, good);    // Generate the training images
 // Clear cache at the end
       GBPriceCache.GetInstance().Dispose();
     }