CNNImageProcessor commit changes ot support new CNN model.

App.config

@@ -1,6 +1,29 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0"?>
 <configuration>
+	<appSettings>
+		<add key="market_data" value="Database=market_data;Datasource=localhost;Username=root;Password=dbas"/>
+		<add key="portfolio_data" value="Database=portfolio_data;Datasource=localhost;Username=root;Password=dbas"/>
+		<add key="user_data" value="Database=user_data;Datasource=localhost;Username=root;Password=dbas"/>
+		<add key="sms_smtpaddress" value="smtp.gmail.com"/>
+		<add key="sms_smsusername" value="skessler1964@gmail.com"/>
+		<add key="sms_smspassword" value="xjfo isnf gmyi zovr"/>
+		<!--<add key="sms_smsrecipients" value="6315252496@vtext.com"/>-->
+		<add key="sms_smsrecipients" value="skessler1964sms@gmail.com"/>
+		<add key="proxy_address" value="http://127.0.0.1:8182"/>
+		<add key="proxy_GetLatestPriceYahoo" value="false"/>
+		<add key="proxy_GetLatestPriceFidelity" value="true"/>
+		<add key="proxy_GetLatestPriceBigCharts" value="false"/>
+		<add key="proxy_GetETFHoldings" value="false"/>
+		<add key="proxy_GetAnalystPriceTargetYahoo" value="true"/>
+		<add key="proxy_GetDailyPrices" value="false"/>
+		<add key="proxy_GetFundamentalEx" value="false"/>
+		<add key="proxy_GetDividendHistory" value="false"/>
+		<add key="proxy_GetAnalystPriceTargetMarketBeat" value="false"/>
+		<add key="proxy_GetCompanyHeadlinesSeekingAlphaV1" value="true"/>
+		<add key="proxy_GetCompanyHeadlinesSeekingAlphaV2" value="true"/>
+	</appSettings>
 	<startup>
-        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2"/>
+		<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2"/>
 	</startup>
 </configuration>
+

CNNImageProcessor.csproj

@@ -9,7 +9,7 @@
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>CNNImageProcessor</RootNamespace>
     <AssemblyName>CNNImageProcessor</AssemblyName>
-    <TargetFrameworkVersion>v4.6.2</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <TargetFrameworkProfile />
   </PropertyGroup>

Program.cs

@@ -1,7 +1,15 @@
-using System;
+using MarketData.Cache;
-using System.IO;
 using MarketData.CNNProcessing;
+using MarketData.DataAccess;
+using MarketData.Generator.CMMomentum;
+using MarketData.MarketDataModel;
 using MarketData.Utils;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
 
 namespace CNNImageProcessor
 {
@@ -18,6 +26,72 @@ namespace CNNImageProcessor
       GenerateImageData(@"C:\DeepLearningImageTests\DeepLearningImageData\RawData1",@"c:\DeepLearningImageTests\DeepLearningImageData\Data\1");
     }
 
+    /// <summary>
+    /// Process all images in sourcePath through PIL on the CNNServer and save them to destinationFolder
+    /// </summary>
+    /// <param name="sourcePath"></param>
+    /// <param name="destinationPath"></param>
+    public static bool ProcessImages(String sourcePath, String destinationPath,String cnnClientUrl="http://10.0.0.73:5000")
+    {
+      String[] files = Directory.GetFiles(sourcePath,"*.jpg");
+
+      CNNClient cnnClient=new CNNClient(cnnClientUrl);
+      if(!cnnClient.Ping())
+      {
+        Console.WriteLine($"CNNServer at {cnnClientUrl} is not responding to ping");
+        return false;
+      }
+      foreach(String file in files)
+      {
+        Console.WriteLine($"Processing {file}");
+        ImageHelper imageHelper=new ImageHelper();
+        imageHelper.LoadImage(file);
+        imageHelper.ToGrayScale();
+        imageHelper.Resize(128,128);
+        Stream stream = imageHelper.ToStream();
+        Stream processed = cnnClient.ProcessImage(stream);
+        imageHelper.LoadImage(processed);
+        String pureFileName = Path.GetFileName(file);
+        String saveFileName = destinationPath + @"\" + pureFileName;
+        imageHelper.Save(saveFileName);
+      }
+      return true;
+    }
+
+    /// <summary>
+    /// Process all images in sourcePath through PIL on the CNNServer and save them to destinationFolder
+    /// </summary>
+    /// <param name="sourcePath"></param>
+    /// <param name="destinationPath"></param>
+    public static bool ProcessImages(String sourcePath, String destinationPath,int resizeTo,String cnnClientUrl="http://10.0.0.73:5000")
+    {
+      String[] files = Directory.GetFiles(sourcePath,"*.jpg");
+
+      CNNClient cnnClient=new CNNClient(cnnClientUrl);
+      if(!cnnClient.Ping())
+      {
+        Console.WriteLine($"CNNServer at {cnnClientUrl} is not responding to ping");
+        return false;
+      }
+      foreach(String file in files)
+      {
+        Console.WriteLine($"Processing {file}");
+        ImageHelper imageHelper=new ImageHelper();
+        imageHelper.LoadImage(file);
+//        imageHelper.ToGrayScale();
+        imageHelper.Resize(resizeTo,resizeTo);
+        Stream stream = imageHelper.ToStream();
+        Stream processed = cnnClient.ProcessImage(stream);
+        imageHelper.LoadImage(processed);
+        String pureFileName = Path.GetFileName(file);
+        String saveFileName = destinationPath + @"\" + pureFileName;
+        imageHelper.Save(saveFileName);
+      }
+      return true;
+    }
+
+
+
     public static void GenerateImageData(String inputFolder,String destinationFolder)
     {
       ImageHelper imageHelper = new ImageHelper();
@@ -98,6 +172,9 @@ namespace CNNImageProcessor
         }
     }
 
+    /// <summary>
+    /// Processes an image through PIL on the CNN Server
+    /// </summary>
     public static void ProcessImage()
     {
       ImageHelper imageHelper=new ImageHelper();
@@ -129,39 +206,169 @@ namespace CNNImageProcessor
       }
     }
 
-    public static void ProcessImages(String sourcePath, String destinationPath)
+    public static List<Holding> GenerateTrades()
     {
-      String[] files = Directory.GetFiles(sourcePath,"*.jpg");
+      List<Holding> holdings = new List<Holding>();
-      foreach(String file in files)
+      DateGenerator dateGenerator = new DateGenerator();
+      DateTime startDate = DateTime.Parse("10/31/2019");
+      DateTime endDate =  DateTime.Parse("02/01/2026");
+      DateTime actualEndDate = endDate; 
+      DateTime analysisDate = DateTime.Now;
+
+      String modelPathFileName = @"C:\boneyard\marketdata\bin\Debug\saferun\CM20191031.txt";
+      CMSessionParams sessionParams = CMSessionManager.RestoreSession(modelPathFileName);
+
+      startDate = dateGenerator.GetCurrentMonthEnd(startDate);
+      endDate = dateGenerator.GetCurrentMonthEnd(endDate);
+      actualEndDate = dateGenerator.GenerateHistoricalDate(endDate, 60);
+      DateTime runDate = startDate;
+
+      sessionParams.CMParams.UseCNN=false;  // don't use the model
+      sessionParams.CMParams.MaxPositions=100;   // take up to 100
+      while(runDate < actualEndDate)
       {
-        Console.WriteLine($"Processing {file}");
+        Console.WriteLine($"Running {runDate.ToShortDateString()}");
-        ImageHelper imageHelper=new ImageHelper();
+        DateTime sellDate = dateGenerator.DaysAddActual(runDate, 90);
-        imageHelper.LoadImage(file);
+
-        imageHelper.ToGrayScale();
+        CMGeneratorResult result = CMMomentumGenerator.GenerateCMCandidates(runDate, analysisDate, sessionParams.CMParams, new List<string>());
-        imageHelper.Resize(128,128);
+        Console.WriteLine($"Got {result.CMCandidates.Count} candidates for {runDate.ToShortDateString()}");
-        Stream stream = imageHelper.ToStream();
+        foreach (CMCandidate candidate in result.CMCandidates)
-        CNNClient cnnClient = new CNNClient("http://10.0.0.73:5000");
+        {
-        Stream processed = cnnClient.ProcessImage(stream);
+          Holding holding = new Holding();
-        imageHelper.LoadImage(processed);
+          holding.Symbol = candidate.Symbol;
-        String pureFileName = Path.GetFileName(file);
+          holding.PurchaseDate = runDate;
-        String saveFileName = destinationPath + @"\" + pureFileName;
+          holding.SellDate = sellDate;
-        imageHelper.Save(saveFileName);
+          Price purchasePrice = GBPriceCache.GetInstance().GetPrice(holding.Symbol, holding.PurchaseDate);
+          Price sellPrice = GBPriceCache.GetInstance().GetPrice(holding.Symbol, holding.SellDate);
+          if (null == purchasePrice || null == sellPrice) continue;
+          holding.PurchasePrice = purchasePrice.Close;
+          holding.SellPrice = sellPrice.Close;
+          holding.GainLoss = holding.SellPrice - holding.PurchasePrice;
+          holding.GainLossPercent = ((holding.SellPrice - holding.PurchasePrice) / holding.PurchasePrice);
+          holdings.Add(holding);
+        }
+        runDate = dateGenerator.DaysAddActual(runDate, 30);
+        runDate = dateGenerator.GetCurrentMonthEnd(runDate);
+      }
+      return holdings;
+    }
+
+
+    public static void GenerateTrainingImages()
+    {
+    // model training will happen on these folders C:\boneyard\DeepLearning\data\0      C:\boneyard\DeepLearning\data\1
+      CNNProcessor.GenerateTraining(@"C:\Data");    // This will generate into C:\Data\0 and C:\Data\1
+      ProcessImages(@"C:\Data\0",@"C:\boneyard\DeepLearning\ModelInputData\0");  // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\0
+      ProcessImages(@"C:\Data\1",@"C:\boneyard\DeepLearning\ModelInputData\1");  // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\1                                                                             
+    }
+
+
+		public static void ClearFolderPath(String strFolderPath)
+		{
+      Console.WriteLine($"Cleaning {strFolderPath}");
+      if(String.IsNullOrEmpty(strFolderPath))throw new InvalidDataException($"{nameof(strFolderPath)} cannot be null");
+      if(!Directory.Exists(strFolderPath))
+      {
+        Directory.CreateDirectory(strFolderPath);
+      }
+      else
+      {
+        String[] pathFileNames = Directory.GetFiles(strFolderPath);
+        Console.WriteLine($"Deleting {pathFileNames.Length} files from {strFolderPath}");
+        foreach(String file in pathFileNames)
+        {
+          File.Delete(file);
+        }
       }
 		}
 
 
+    public static List<Holding> ReadHoldings(String strPathFileName)
+    {
+      String strLine;
+      List<Holding> universe = new List<Holding>();
+      StreamReader inStream = new StreamReader(strPathFileName);
+      inStream.ReadLine();  // header
+      while (null != (strLine = inStream.ReadLine()))
+      {
+        Holding holding = Holding.FromString(strLine);
+        if (null == holding) continue;
+        universe.Add(holding);
+      }
+      inStream.Close();
+      inStream.Dispose();
+      Console.WriteLine($"Read {universe.Count} holdings");
+      return universe;
+    }
 
+    public static (List<Holding> avoid, List<Holding> good) GenerateCodeTestCases(List<Holding> universe)
+    {
+      double validationPercent=0.05;
+      double validationPercentUnseen=0.50;
+      Console.WriteLine($"Read {universe.Count} holdings");
+
+      List<Holding> avoid = universe.Where(x=>x.GainLoss<-.05).ToList();
+      List<Holding> good=universe.Where(x=>x.GainLoss>.05).ToList();
+
+      int validationCount = (int)(validationPercent * universe.Count);
+
+      Random rng = new Random();
+      List<Holding> goodValidation = good.OrderBy(x => rng.Next()).Take(validationCount).ToList();
+      int goodUnseenCount = (int)(validationPercentUnseen * goodValidation.Count);
+      List<Holding> goodValidationUnseen = goodValidation.OrderBy(x => rng.Next()).Take(goodUnseenCount).ToList();
+      good.RemoveAll(x => goodValidationUnseen.Contains(x));
+      Console.WriteLine($"Validation sample size: {goodValidation.Count}");
+      Console.WriteLine($"Unseen validation removed from good: {goodValidationUnseen.Count}");
+      Console.WriteLine($"Remaining good count: {good.Count}");
+
+
+      List<Holding> avoidValidation = avoid.OrderBy(x => rng.Next()).Take(validationCount).ToList();
+      int avoidUnseenCount = (int)(validationPercentUnseen * avoidValidation.Count);
+      List<Holding> avoidValidationUnseen = avoidValidation.OrderBy(x => rng.Next()).Take(avoidUnseenCount).ToList();
+      avoid.RemoveAll(x => avoidValidationUnseen.Contains(x));
+      Console.WriteLine($"Validation sample size: {avoidValidation.Count}");
+      Console.WriteLine($"Unseen validation removed from avoid: {avoidValidationUnseen.Count}");
+      Console.WriteLine($"Remaining avoid count: {avoid.Count}");
+
+      return (avoid, good);
+    }
+
+    public static void GenerateTrainingImages(List<Holding> avoid, List<Holding> good)
+    {
+      int imageSize=224;
+      int dayCount=90;  // 90
+      Console.WriteLine($"Generate training into {@"C:\Data"}");
+      CNNProcessor.GenerateTraining(avoid, good, imageSize,dayCount, TestCase.GenerateType.BollingerBandWithVIX,@"C:\Data");
+      ClearFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\0");
+      ClearFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\1");
+      if(!ProcessImages(@"C:\Data\0",@"C:\boneyard\DeepLearning\ModelInputData\0",imageSize))  // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\0
+      {
+        Console.WriteLine($"Process image failed, is the server running?");
+      }
+      if(!ProcessImages(@"C:\Data\1",@"C:\boneyard\DeepLearning\ModelInputData\1",imageSize))  // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\1                                                                             
+      {
+        Console.WriteLine($"Process image failed, is the server running?");
+      }
+      Console.WriteLine("Please copy these files into the training folder.");
+    }
+
+    /// <summary>
+    /// This will generate images into  C:\boneyard\DeepLearning\ModelInputData\0 and  C:\boneyard\DeepLearning\ModelInputData\1
+    /// You should then copy the generated images into C:\boneyard\DeepLearning\Data folder and then proceed to train tbe latest model
+    /// which at the time of writing this is model_sk_convnext_v1.py.  After running the model you shoukd then run
+    /// verify_model_sk_convnext_v1.py.  This will produce a validation score which at the time of writing is 99%.  It will also produce
+    /// some output images including the confusion matrix.
+    /// </summary>
+    /// <param name="args"></param>
     static void Main(string[] args)
     {
-      //        ProcessImages(@"C:\Data\0",@"C:\DeepLearningImageTests\DeepLearningImageData\Data\0");
+// The modified flow
-      //        ProcessImages(@"C:\Data\1",@"C:\DeepLearningImageTests\DeepLearningImageData\Data\1");
+      //List<Holding> holdings = GenerateTrades();   // generate a holding set from the CMMomentum monthly candidates
-
+      List<Holding> holdings = ReadHoldings("holdings.csv");  // read a holding set that was previously generated
-
+      (List<Holding> avoid, List<Holding> good)=GenerateCodeTestCases(holdings);   // split the dataset into avoid and good
-      //        GenerateImageData();
+      GenerateTrainingImages(avoid, good);
-      //        TestCNN();
+// Clear cache at the end
-      //        ProcessImage();
+      GBPriceCache.GetInstance().Dispose();
-      //        CreateValidationImages(@"C:\2",@"C:\3");
-
     }
   }
 }