Code cleanup

.gitignore

@@ -1,6 +1,7 @@
-**/obj/
+**/obj/
 **/bin/
 **/.vs/
 bin
 obj
 .vs
+/obj/Debug

App.config

@@ -1,6 +1,29 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0"?>
 <configuration>
-    <startup> 
+	<appSettings>
-        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2"/>
+		<add key="market_data" value="Database=market_data;Datasource=localhost;Username=root;Password=dbas"/>
-    </startup>
+		<add key="portfolio_data" value="Database=portfolio_data;Datasource=localhost;Username=root;Password=dbas"/>
+		<add key="user_data" value="Database=user_data;Datasource=localhost;Username=root;Password=dbas"/>
+		<add key="sms_smtpaddress" value="smtp.gmail.com"/>
+		<add key="sms_smsusername" value="skessler1964@gmail.com"/>
+		<add key="sms_smspassword" value="xjfo isnf gmyi zovr"/>
+		<!--<add key="sms_smsrecipients" value="6315252496@vtext.com"/>-->
+		<add key="sms_smsrecipients" value="skessler1964sms@gmail.com"/>
+		<add key="proxy_address" value="http://127.0.0.1:8182"/>
+		<add key="proxy_GetLatestPriceYahoo" value="false"/>
+		<add key="proxy_GetLatestPriceFidelity" value="true"/>
+		<add key="proxy_GetLatestPriceBigCharts" value="false"/>
+		<add key="proxy_GetETFHoldings" value="false"/>
+		<add key="proxy_GetAnalystPriceTargetYahoo" value="true"/>
+		<add key="proxy_GetDailyPrices" value="false"/>
+		<add key="proxy_GetFundamentalEx" value="false"/>
+		<add key="proxy_GetDividendHistory" value="false"/>
+		<add key="proxy_GetAnalystPriceTargetMarketBeat" value="false"/>
+		<add key="proxy_GetCompanyHeadlinesSeekingAlphaV1" value="true"/>
+		<add key="proxy_GetCompanyHeadlinesSeekingAlphaV2" value="true"/>
+	</appSettings>
+	<startup>
+		<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2"/>
+	</startup>
 </configuration>
+

CNNImageProcessor.csproj

@@ -9,7 +9,7 @@
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>CNNImageProcessor</RootNamespace>
     <AssemblyName>CNNImageProcessor</AssemblyName>
-    <TargetFrameworkVersion>v4.6.2</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <TargetFrameworkProfile />
   </PropertyGroup>

Program.cs

@@ -1,7 +1,15 @@
-using System;
+using MarketData.Cache;
-using System.IO;
 using MarketData.CNNProcessing;
+using MarketData.DataAccess;
+using MarketData.Generator.CMMomentum;
+using MarketData.MarketDataModel;
 using MarketData.Utils;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
 
 namespace CNNImageProcessor
 {
@@ -18,6 +26,72 @@ namespace CNNImageProcessor
       GenerateImageData(@"C:\DeepLearningImageTests\DeepLearningImageData\RawData1",@"c:\DeepLearningImageTests\DeepLearningImageData\Data\1");
     }
 
+    /// <summary>
+    /// Process all images in sourcePath through PIL on the CNNServer and save them to destinationFolder
+    /// </summary>
+    /// <param name="sourcePath"></param>
+    /// <param name="destinationPath"></param>
+    public static bool ProcessImages(String sourcePath, String destinationPath,String cnnClientUrl="http://10.0.0.73:5000")
+    {
+      String[] files = Directory.GetFiles(sourcePath,"*.jpg");
+
+      CNNClient cnnClient=new CNNClient(cnnClientUrl);
+      if(!cnnClient.Ping())
+      {
+        Console.WriteLine($"CNNServer at {cnnClientUrl} is not responding to ping");
+        return false;
+      }
+      foreach(String file in files)
+      {
+        Console.WriteLine($"Processing {file}");
+        ImageHelper imageHelper=new ImageHelper();
+        imageHelper.LoadImage(file);
+        imageHelper.ToGrayScale();
+        imageHelper.Resize(128,128);
+        Stream stream = imageHelper.ToStream();
+        Stream processed = cnnClient.ProcessImage(stream);
+        imageHelper.LoadImage(processed);
+        String pureFileName = Path.GetFileName(file);
+        String saveFileName = destinationPath + @"\" + pureFileName;
+        imageHelper.Save(saveFileName);
+      }
+      return true;
+    }
+
+    /// <summary>
+    /// Process all images in sourcePath through PIL on the CNNServer and save them to destinationFolder
+    /// </summary>
+    /// <param name="sourcePath"></param>
+    /// <param name="destinationPath"></param>
+    public static bool ProcessImages(String sourcePath, String destinationPath,int resizeTo,String cnnClientUrl="http://10.0.0.73:5000",bool useGrayScale=false)
+    {
+      String[] files = Directory.GetFiles(sourcePath,"*.jpg");
+
+      CNNClient cnnClient=new CNNClient(cnnClientUrl);
+      if(!cnnClient.Ping())
+      {
+        Console.WriteLine($"CNNServer at {cnnClientUrl} is not responding to ping");
+        return false;
+      }
+      foreach(String file in files)
+      {
+        Console.WriteLine($"Processing {file}");
+        ImageHelper imageHelper=new ImageHelper();
+        imageHelper.LoadImage(file);
+        if(useGrayScale)imageHelper.ToGrayScale();
+        imageHelper.Resize(resizeTo,resizeTo);
+        Stream stream = imageHelper.ToStream();
+        Stream processed = cnnClient.ProcessImage(stream);
+        imageHelper.LoadImage(processed);
+        String pureFileName = Path.GetFileName(file);
+        String saveFileName = destinationPath + @"\" + pureFileName;
+        imageHelper.Save(saveFileName);
+      }
+      return true;
+    }
+
+
+
     public static void GenerateImageData(String inputFolder,String destinationFolder)
     {
       ImageHelper imageHelper = new ImageHelper();
@@ -98,6 +172,9 @@ namespace CNNImageProcessor
         }
     }
 
+    /// <summary>
+    /// Processes an image through PIL on the CNN Server
+    /// </summary>
     public static void ProcessImage()
     {
       ImageHelper imageHelper=new ImageHelper();
@@ -129,39 +206,224 @@ namespace CNNImageProcessor
       }
     }
 
-    public static void ProcessImages(String sourcePath, String destinationPath)
+    public static List<Holding> GenerateTrades()
     {
-      String[] files = Directory.GetFiles(sourcePath,"*.jpg");
+      List<Holding> holdings = new List<Holding>();
-      foreach(String file in files)
+      DateGenerator dateGenerator = new DateGenerator();
+      DateTime startDate = DateTime.Parse("10/31/2019");
+      DateTime endDate =  DateTime.Parse("12/31/2025");
+      DateTime actualEndDate = endDate; 
+      DateTime analysisDate = DateTime.Now;
+
+      String modelPathFileName = @"C:\boneyard\marketdata\bin\Debug\saferun\CM20191031.txt";
+      CMSessionParams sessionParams = CMSessionManager.RestoreSession(modelPathFileName);
+
+      startDate = dateGenerator.GetCurrentMonthEnd(startDate);
+      endDate = dateGenerator.GetCurrentMonthEnd(endDate);
+      actualEndDate = dateGenerator.GenerateHistoricalDate(endDate, 60);
+      DateTime runDate = startDate;
+
+      sessionParams.CMParams.UseCNN=false;  // don't use the model
+      sessionParams.CMParams.MaxPositions=100;   // take up to 100
+      while(runDate < actualEndDate)
       {
-        Console.WriteLine($"Processing {file}");
+        Console.WriteLine($"Running {runDate.ToShortDateString()}");
-        ImageHelper imageHelper=new ImageHelper();
+        DateTime sellDate = dateGenerator.DaysAddActual(runDate, 90);
-        imageHelper.LoadImage(file);
+
-        imageHelper.ToGrayScale();
+        CMGeneratorResult result = CMMomentumGenerator.GenerateCMCandidates(runDate, analysisDate, sessionParams.CMParams, new List<string>());
-        imageHelper.Resize(128,128);
+        Console.WriteLine($"Got {result.CMCandidates.Count} candidates for {runDate.ToShortDateString()}");
-        Stream stream = imageHelper.ToStream();
+        foreach (CMCandidate candidate in result.CMCandidates)
-        CNNClient cnnClient = new CNNClient("http://10.0.0.73:5000");
+        {
-        Stream processed = cnnClient.ProcessImage(stream);
+          Holding holding = new Holding();
-        imageHelper.LoadImage(processed);
+          holding.Symbol = candidate.Symbol;
-        String pureFileName = Path.GetFileName(file);
+          holding.PurchaseDate = runDate;
-        String saveFileName = destinationPath + @"\" + pureFileName;
+          holding.SellDate = sellDate;
-        imageHelper.Save(saveFileName);
+          Price purchasePrice = GBPriceCache.GetInstance().GetPrice(holding.Symbol, holding.PurchaseDate);
+          Price sellPrice = GBPriceCache.GetInstance().GetPrice(holding.Symbol, holding.SellDate);
+          if (null == purchasePrice || null == sellPrice) continue;
+          holding.PurchasePrice = purchasePrice.Close;
+          holding.SellPrice = sellPrice.Close;
+          holding.GainLoss = holding.SellPrice - holding.PurchasePrice;
+          holding.GainLossPercent = ((holding.SellPrice - holding.PurchasePrice) / holding.PurchasePrice);
+          holdings.Add(holding);
+        }
+        runDate = dateGenerator.DaysAddActual(runDate, 30);
+        runDate = dateGenerator.GetCurrentMonthEnd(runDate);
       }
+      return holdings;
     }
 
 
+    public static void GenerateTrainingImages()
+    {
+    // model training will happen on these folders C:\boneyard\DeepLearning\data\0      C:\boneyard\DeepLearning\data\1
+      CNNProcessor.GenerateTraining(@"C:\Data");    // This will generate into C:\Data\0 and C:\Data\1
+      ProcessImages(@"C:\Data\0",@"C:\boneyard\DeepLearning\ModelInputData\0");  // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\0
+      ProcessImages(@"C:\Data\1",@"C:\boneyard\DeepLearning\ModelInputData\1");  // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\1                                                                             
+    }
 
+
+		public static void ClearFolderPath(String strFolderPath)
+		{
+      Console.WriteLine($"Cleaning {strFolderPath}");
+      if(String.IsNullOrEmpty(strFolderPath))throw new InvalidDataException($"{nameof(strFolderPath)} cannot be null");
+      if(!Directory.Exists(strFolderPath))
+      {
+        Directory.CreateDirectory(strFolderPath);
+      }
+      else
+      {
+        String[] pathFileNames = Directory.GetFiles(strFolderPath);
+        Console.WriteLine($"Deleting {pathFileNames.Length} files from {strFolderPath}");
+        foreach(String file in pathFileNames)
+        {
+          File.Delete(file);
+        }
+      }
+		}
+
+    public static List<Holding> ReadHoldings(String strPathFileName)
+    {
+      String strLine;
+      List<Holding> universe = new List<Holding>();
+      StreamReader inStream = new StreamReader(strPathFileName);
+      inStream.ReadLine();  // header
+      while (null != (strLine = inStream.ReadLine()))
+      {
+        Holding holding = Holding.FromString(strLine);
+        if (null == holding) continue;
+        universe.Add(holding);
+      }
+      inStream.Close();
+      inStream.Dispose();
+      Console.WriteLine($"Read {universe.Count} holdings");
+      return universe;
+    }
+
+    public static void WriteHoldings(List<Holding> holdings,String strPathFileName)
+    {
+      if(File.Exists(strPathFileName))File.Delete(strPathFileName);
+      StreamWriter outStream = new StreamWriter(strPathFileName);
+      outStream.WriteLine(Holding.Heading);
+      foreach(Holding holding in holdings)
+      {
+        outStream.WriteLine(holding);
+      }
+      outStream.Flush();
+      outStream.Close();
+      outStream.Dispose();
+    }
+
+    //public static (List<Holding> avoid, List<Holding> good) GenerateCodeTestCases(List<Holding> universe)
+    //{
+    //  double validationPercent=0.05;
+    //  double validationPercentUnseen=0.50;
+    //  Console.WriteLine($"Read {universe.Count} holdings");
+
+    //  List<Holding> avoid = universe.Where(x=>x.GainLoss<-.05).ToList();
+    //  List<Holding> good=universe.Where(x=>x.GainLoss>.05).ToList();
+
+    //  int validationCount = (int)(validationPercent * universe.Count);
+
+    //  Random rng = new Random();
+    //  List<Holding> goodValidation = good.OrderBy(x => rng.Next()).Take(validationCount).ToList();
+    //  int goodUnseenCount = (int)(validationPercentUnseen * goodValidation.Count);
+    //  List<Holding> goodValidationUnseen = goodValidation.OrderBy(x => rng.Next()).Take(goodUnseenCount).ToList();
+    //  good.RemoveAll(x => goodValidationUnseen.Contains(x));
+    //  Console.WriteLine($"Validation sample size: {goodValidation.Count}");
+    //  Console.WriteLine($"Unseen validation removed from good: {goodValidationUnseen.Count}");
+    //  Console.WriteLine($"Remaining good count: {good.Count}");
+
+
+    //  List<Holding> avoidValidation = avoid.OrderBy(x => rng.Next()).Take(validationCount).ToList();
+    //  int avoidUnseenCount = (int)(validationPercentUnseen * avoidValidation.Count);
+    //  List<Holding> avoidValidationUnseen = avoidValidation.OrderBy(x => rng.Next()).Take(avoidUnseenCount).ToList();
+    //  avoid.RemoveAll(x => avoidValidationUnseen.Contains(x));
+    //  Console.WriteLine($"Validation sample size: {avoidValidation.Count}");
+    //  Console.WriteLine($"Unseen validation removed from avoid: {avoidValidationUnseen.Count}");
+    //  Console.WriteLine($"Remaining avoid count: {avoid.Count}");
+
+    //  return (avoid, good);
+    //}
+
+    public static (List<Holding> avoid, List<Holding> good) GenerateCodeTestCases(List<Holding> universe)
+    {
+      double validationPercent=0.05;
+      Console.WriteLine($"Read {universe.Count} holdings");
+
+      List<Holding> avoid = universe.Where(x=>x.GainLoss<-.05).ToList();
+      List<Holding> good=universe.Where(x=>x.GainLoss>.05).ToList();
+
+      int validationCount = (int)(validationPercent * universe.Count);
+
+      Random rng = new Random();
+      List<Holding> goodValidation = good.OrderBy(x => rng.Next()).Take(validationCount).ToList();
+      Console.WriteLine($"Validation sample size: {goodValidation.Count}");
+      Console.WriteLine($"Remaining good count: {good.Count}");
+
+
+      List<Holding> avoidValidation = avoid.OrderBy(x => rng.Next()).Take(validationCount).ToList();
+      Console.WriteLine($"Validation sample size: {avoidValidation.Count}");
+      Console.WriteLine($"Remaining avoid count: {avoid.Count}");
+
+      return (avoid, good);
+    }
+
+
+    public static void GenerateTrainingImages(List<Holding> avoid, List<Holding> good)
+    {
+      String cnnClientUrl="http://127.0.0.1:5000";
+      int imageSize=224;
+      int dayCount=90;  
+      Console.WriteLine($"Generate training into {@"C:\boneyard\DeepLearning\ModelInputData"}");
+      CNNProcessor.GenerateTraining(avoid, good, imageSize,dayCount, TestCase.GenerateType.BollingerBandWithVIX,@"C:\boneyard\DeepLearning\ModelInputData");
+      ClearFolderPath(@"C:\boneyard\DeepLearning\Data\0");
+      ClearFolderPath(@"C:\boneyard\DeepLearning\Data\1");
+      CNNClient cnnClient = new CNNClient(cnnClientUrl);
+      if(!cnnClient.Ping())
+      {
+        Console.WriteLine($"CNN Server @ {cnnClientUrl} is not responding.");
+        return;
+      }
+      ProcessImages(@"C:\boneyard\DeepLearning\ModelInputData\0",@"C:\boneyard\DeepLearning\Data\0",imageSize,cnnClientUrl,false);  // Process through PIL and put in C:\boneyard\DeepLearning\Data\0
+      ProcessImages(@"C:\boneyard\DeepLearning\ModelInputData\1",@"C:\boneyard\DeepLearning\Data\1",imageSize,cnnClientUrl,false);  // Process through PIL and put in C:\boneyard\DeepLearning\Data\1                                                                             
+      Console.WriteLine("Done.");
+    }
+
+    /// <summary>
+    /// This will generate images into  C:\boneyard\DeepLearning\Data\0 and  C:\boneyard\DeepLearning\Data\1
+    /// You should then proceed to train tbe latest model which at the time of writing this is model_sk_convnext_v1.py.  
+    /// After running the model you should then run verify_model_sk_convnext_v1.py.  This will produce a validation score 
+    /// which at the time of writing is 99%.  It will also produce some output images including the confusion matrix.
+    /// 
+    /// I am using WSL2 to perform the training because WSL2 is the only option for tensorflow with GPU.
+    /// To launch WSL2 at a command prompt type "wsl ~".   If the enviroment is not set up then you can use the setup_tf_gpu.sh
+    /// script in CNN/Scripts folder to re-create the full Python3 environment and Tensorflow.  The script will create the 
+    /// virtual environment and install everything.  It was used to create the current WSL enviroment.
+    /// To start the environment "source tf_gpu/bin/activate"
+    /// then type "code ."  This will run VSCODE and attach to the WSL environment.
+    /// Train the model on EUPORIE laptop using the GPU card with WSL2.  (Windows Subsystem For Linux).  I am running Ubuntu1 22.04.2
+    /// To launch WSL open up a command prompt, run powershell and type "wsl ~".
+    /// The folder structure will be /home/pi/CNN  /home/pi/DeepLearning
+    /// You can access the folder structure through windows explorer.  type "\\wsl$" in explorer and navigate to the folder.
+    /// drop in the Data and Model and run the model.
+    /// There is a shell script in the Scripts folder of the CNN project. setup_tf_gpu.sh  
+    /// Copy the script to the CNN folder and run it from the CNN folder.  
+    /// It will create the venv enviroment and install python 3.10 and tensorflow (gpu)
+    /// 
+    /// ********************************************************************************************************************    /// 
+    /// </summary>
+    /// <param name="args"></param>
     static void Main(string[] args)
     {
-      //        ProcessImages(@"C:\Data\0",@"C:\DeepLearningImageTests\DeepLearningImageData\Data\0");
+// The modified flow
-      //        ProcessImages(@"C:\Data\1",@"C:\DeepLearningImageTests\DeepLearningImageData\Data\1");
+ //     List<Holding> holdings = GenerateTrades();   // generate a new holding set from the CMMomentum monthly candidates
-
+ //     WriteHoldings(holdings,"holdings.csv");      // save the list.  The saved list can be read back in to save time in case reruns are necessary
-
+      List<Holding> holdings = ReadHoldings("holdings.csv");  // read a holding set that was previously generated.  You'll want to create a new set of holdings for retraiing
-      //        GenerateImageData();
+      (List<Holding> avoid, List<Holding> good)=GenerateCodeTestCases(holdings);   // split the dataset into avoid and good
-      //        TestCNN();
+      GenerateTrainingImages(avoid, good);    // Generate the training images
-      //        ProcessImage();
+// Clear cache at the end
-      //        CreateValidationImages(@"C:\2",@"C:\3");
+      GBPriceCache.GetInstance().Dispose();
-
     }
   }
 }