CNNImageProcessor/Program.cs

using MarketData.Cache;
using MarketData.CNNProcessing;
using MarketData.DataAccess;
using MarketData.Generator.CMMomentum;
using MarketData.MarketDataModel;
using MarketData.Utils;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;

namespace CNNImageProcessor
{
  class Program
  {

// **************************************************************************************************************************************************
// **************************************************************************************************************************************************

    // You will then need to copy this data to C:\DeepLearningImageTests\DeepLearningImageData\Data
    public static void GenerateImageData()
    {
      GenerateImageData(@"C:\DeepLearningImageTests\DeepLearningImageData\RawData0",@"c:\DeepLearningImageTests\DeepLearningImageData\Data\0");
      GenerateImageData(@"C:\DeepLearningImageTests\DeepLearningImageData\RawData1",@"c:\DeepLearningImageTests\DeepLearningImageData\Data\1");
    }

    /// <summary>
    /// Process all images in sourcePath through PIL on the CNNServer and save them to destinationFolder
    /// </summary>
    /// <param name="sourcePath"></param>
    /// <param name="destinationPath"></param>
    public static bool ProcessImages(String sourcePath, String destinationPath,String cnnClientUrl="http://10.0.0.73:5000")
    {
      String[] files = Directory.GetFiles(sourcePath,"*.jpg");

      CNNClient cnnClient=new CNNClient(cnnClientUrl);
      if(!cnnClient.Ping())
      {
        Console.WriteLine($"CNNServer at {cnnClientUrl} is not responding to ping");
        return false;
      }
      foreach(String file in files)
      {
        Console.WriteLine($"Processing {file}");
        ImageHelper imageHelper=new ImageHelper();
        imageHelper.LoadImage(file);
        imageHelper.ToGrayScale();
        imageHelper.Resize(128,128);
        Stream stream = imageHelper.ToStream();
        Stream processed = cnnClient.ProcessImage(stream);
        imageHelper.LoadImage(processed);
        String pureFileName = Path.GetFileName(file);
        String saveFileName = destinationPath + @"\" + pureFileName;
        imageHelper.Save(saveFileName);
      }
      return true;
    }

    /// <summary>
    /// Process all images in sourcePath through PIL on the CNNServer and save them to destinationFolder
    /// </summary>
    /// <param name="sourcePath"></param>
    /// <param name="destinationPath"></param>
    public static bool ProcessImages(String sourcePath, String destinationPath,int resizeTo,String cnnClientUrl="http://10.0.0.73:5000")
    {
      String[] files = Directory.GetFiles(sourcePath,"*.jpg");

      CNNClient cnnClient=new CNNClient(cnnClientUrl);
      if(!cnnClient.Ping())
      {
        Console.WriteLine($"CNNServer at {cnnClientUrl} is not responding to ping");
        return false;
      }
      foreach(String file in files)
      {
        Console.WriteLine($"Processing {file}");
        ImageHelper imageHelper=new ImageHelper();
        imageHelper.LoadImage(file);
//        imageHelper.ToGrayScale();
        imageHelper.Resize(resizeTo,resizeTo);
        Stream stream = imageHelper.ToStream();
        Stream processed = cnnClient.ProcessImage(stream);
        imageHelper.LoadImage(processed);
        String pureFileName = Path.GetFileName(file);
        String saveFileName = destinationPath + @"\" + pureFileName;
        imageHelper.Save(saveFileName);
      }
      return true;
    }



    public static void GenerateImageData(String inputFolder,String destinationFolder)
    {
      ImageHelper imageHelper = new ImageHelper();
      String[] files = Directory.GetFiles(inputFolder, "*.jpg");
      foreach (String file in files)
      {
        try
        {
          String fileName = Path.GetFileName(file);
          String pathFileName = destinationFolder + @"\" + fileName;
          String pathFileNameL = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"L.jpg";
          String pathFileNameR = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"R.jpg";
          String pathFileNameU = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"U.jpg";
          String pathFileNameB1 = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"B1.jpg";
          String pathFileNameB2 = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"B2.jpg";
          String pathFileNameB3 = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"B3.jpg";
          String pathFileNameB4 = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"B4.jpg";

          Console.WriteLine(String.Format("Reading {0}", file));
          imageHelper.LoadImage(file);
          imageHelper.Resize(128, 128);

          ImageHelper bmpLeft=new ImageHelper(imageHelper);
          ImageHelper bmpRight=new ImageHelper(imageHelper);
          ImageHelper bmpUDown=null;
          bmpLeft.RotateLeft();
          bmpRight.RotateRight();
          bmpUDown=new ImageHelper(bmpRight);
          bmpUDown.RotateRight();

          //bmpLeft.ToGrayScale();
          //bmpRight.ToGrayScale();
          //bmpUDown.ToGrayScale();
          //imageHelper.ToGrayScale();

          imageHelper.Save(pathFileName);
          bmpLeft.Save(pathFileNameL);
          bmpRight.Save(pathFileNameR);
          bmpUDown.Save(pathFileNameU);

          ImageHelper bmpBlur1=new ImageHelper(imageHelper);
          ImageHelper bmpBlur2=new ImageHelper(bmpLeft);
          ImageHelper bmpBlur3=new ImageHelper(bmpRight);
          ImageHelper bmpBlur4=new ImageHelper(bmpUDown);
          bmpBlur1.Blur(1);
          bmpBlur2.Blur(1);
          bmpBlur3.Blur(1);
          bmpBlur4.Blur(1);
          bmpBlur1.Save(pathFileNameB1);
          bmpBlur2.Save(pathFileNameB2);
          bmpBlur3.Save(pathFileNameB3);
          bmpBlur4.Save(pathFileNameB4);

        }
        catch (Exception exception)
        {
          Console.WriteLine(exception.ToString());
        }
      }
    }

    public static void TestCNN()
    {
        ImageHelper imageHelper = null;
        Stream stream = null;

        String sourceFolder = @"C:\DeepLearningImageTests\DeepLearningImageData\Validation";
        String[] files = Directory.GetFiles(sourceFolder,"*.jpg");

        foreach(String file in files)
        {
          imageHelper=new ImageHelper();
          imageHelper.LoadImage(file);
          stream = imageHelper.ToStream();
          CNNClient cnnClient = new CNNClient("http://10.0.0.73:5000");
          String result = cnnClient.Predict(CNNClient.Model.vgg16,stream);
          Console.WriteLine(String.Format("Result:{0} File:{1}",result,file));
        }
    }

    /// <summary>
    /// Processes an image through PIL on the CNN Server
    /// </summary>
    public static void ProcessImage()
    {
      ImageHelper imageHelper=new ImageHelper();
      imageHelper.LoadImage(@"C:\DeepLearningImageTests\DeepLearningImageData\RawData0\00de4729-6aa9-465e-906b-4c92bc24f7a9.jpg");
      Stream stream = imageHelper.ToStream();
      CNNClient cnnClient = new CNNClient("http://10.0.0.73:5000");
      Stream result = cnnClient.ProcessImage(stream);
      imageHelper.LoadImage(stream);
      imageHelper.Save(@"c:\2\image.jpg");
    }

    public static void CreateValidationImages(String sourcePath, String destinationPath)
    {
      String[] files = Directory.GetFiles(sourcePath,"*.jpg");
      foreach(String file in files)
      {
        Console.WriteLine($"Processing {file}");
        ImageHelper imageHelper=new ImageHelper();
        imageHelper.LoadImage(file);
        imageHelper.ToGrayScale();
        imageHelper.Resize(128,128);
        Stream stream = imageHelper.ToStream();
        CNNClient cnnClient = new CNNClient("http://10.0.0.73:5000");
        Stream processed = cnnClient.ProcessImage(stream);
        imageHelper.LoadImage(processed);
        String pureFileName = Path.GetFileName(file);
        String saveFileName = destinationPath + @"\" + pureFileName;
        imageHelper.Save(saveFileName);
      }
    }

    public static List<Holding> GenerateTrades()
    {
      List<Holding> holdings = new List<Holding>();
      DateGenerator dateGenerator = new DateGenerator();
      DateTime startDate = DateTime.Parse("10/31/2019");
      DateTime endDate =  DateTime.Parse("02/01/2026");
      DateTime actualEndDate = endDate; 
      DateTime analysisDate = DateTime.Now;

      String modelPathFileName = @"C:\boneyard\marketdata\bin\Debug\saferun\CM20191031.txt";
      CMSessionParams sessionParams = CMSessionManager.RestoreSession(modelPathFileName);

      startDate = dateGenerator.GetCurrentMonthEnd(startDate);
      endDate = dateGenerator.GetCurrentMonthEnd(endDate);
      actualEndDate = dateGenerator.GenerateHistoricalDate(endDate, 60);
      DateTime runDate = startDate;

      sessionParams.CMParams.UseCNN=false;  // don't use the model
      sessionParams.CMParams.MaxPositions=100;   // take up to 100
      while(runDate < actualEndDate)
      {
        Console.WriteLine($"Running {runDate.ToShortDateString()}");
        DateTime sellDate = dateGenerator.DaysAddActual(runDate, 90);

        CMGeneratorResult result = CMMomentumGenerator.GenerateCMCandidates(runDate, analysisDate, sessionParams.CMParams, new List<string>());
        Console.WriteLine($"Got {result.CMCandidates.Count} candidates for {runDate.ToShortDateString()}");
        foreach (CMCandidate candidate in result.CMCandidates)
        {
          Holding holding = new Holding();
          holding.Symbol = candidate.Symbol;
          holding.PurchaseDate = runDate;
          holding.SellDate = sellDate;
          Price purchasePrice = GBPriceCache.GetInstance().GetPrice(holding.Symbol, holding.PurchaseDate);
          Price sellPrice = GBPriceCache.GetInstance().GetPrice(holding.Symbol, holding.SellDate);
          if (null == purchasePrice || null == sellPrice) continue;
          holding.PurchasePrice = purchasePrice.Close;
          holding.SellPrice = sellPrice.Close;
          holding.GainLoss = holding.SellPrice - holding.PurchasePrice;
          holding.GainLossPercent = ((holding.SellPrice - holding.PurchasePrice) / holding.PurchasePrice);
          holdings.Add(holding);
        }
        runDate = dateGenerator.DaysAddActual(runDate, 30);
        runDate = dateGenerator.GetCurrentMonthEnd(runDate);
      }
      return holdings;
    }


    public static void GenerateTrainingImages()
    {
    // model training will happen on these folders C:\boneyard\DeepLearning\data\0      C:\boneyard\DeepLearning\data\1
      CNNProcessor.GenerateTraining(@"C:\Data");    // This will generate into C:\Data\0 and C:\Data\1
      ProcessImages(@"C:\Data\0",@"C:\boneyard\DeepLearning\ModelInputData\0");  // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\0
      ProcessImages(@"C:\Data\1",@"C:\boneyard\DeepLearning\ModelInputData\1");  // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\1                                                                             
    }


		public static void ClearFolderPath(String strFolderPath)
		{
      Console.WriteLine($"Cleaning {strFolderPath}");
      if(String.IsNullOrEmpty(strFolderPath))throw new InvalidDataException($"{nameof(strFolderPath)} cannot be null");
      if(!Directory.Exists(strFolderPath))
      {
        Directory.CreateDirectory(strFolderPath);
      }
      else
      {
        String[] pathFileNames = Directory.GetFiles(strFolderPath);
        Console.WriteLine($"Deleting {pathFileNames.Length} files from {strFolderPath}");
        foreach(String file in pathFileNames)
        {
          File.Delete(file);
        }
      }
		}


    public static List<Holding> ReadHoldings(String strPathFileName)
    {
      String strLine;
      List<Holding> universe = new List<Holding>();
      StreamReader inStream = new StreamReader(strPathFileName);
      inStream.ReadLine();  // header
      while (null != (strLine = inStream.ReadLine()))
      {
        Holding holding = Holding.FromString(strLine);
        if (null == holding) continue;
        universe.Add(holding);
      }
      inStream.Close();
      inStream.Dispose();
      Console.WriteLine($"Read {universe.Count} holdings");
      return universe;
    }

    public static (List<Holding> avoid, List<Holding> good) GenerateCodeTestCases(List<Holding> universe)
    {
      double validationPercent=0.05;
      double validationPercentUnseen=0.50;
      Console.WriteLine($"Read {universe.Count} holdings");

      List<Holding> avoid = universe.Where(x=>x.GainLoss<-.05).ToList();
      List<Holding> good=universe.Where(x=>x.GainLoss>.05).ToList();

      int validationCount = (int)(validationPercent * universe.Count);

      Random rng = new Random();
      List<Holding> goodValidation = good.OrderBy(x => rng.Next()).Take(validationCount).ToList();
      int goodUnseenCount = (int)(validationPercentUnseen * goodValidation.Count);
      List<Holding> goodValidationUnseen = goodValidation.OrderBy(x => rng.Next()).Take(goodUnseenCount).ToList();
      good.RemoveAll(x => goodValidationUnseen.Contains(x));
      Console.WriteLine($"Validation sample size: {goodValidation.Count}");
      Console.WriteLine($"Unseen validation removed from good: {goodValidationUnseen.Count}");
      Console.WriteLine($"Remaining good count: {good.Count}");


      List<Holding> avoidValidation = avoid.OrderBy(x => rng.Next()).Take(validationCount).ToList();
      int avoidUnseenCount = (int)(validationPercentUnseen * avoidValidation.Count);
      List<Holding> avoidValidationUnseen = avoidValidation.OrderBy(x => rng.Next()).Take(avoidUnseenCount).ToList();
      avoid.RemoveAll(x => avoidValidationUnseen.Contains(x));
      Console.WriteLine($"Validation sample size: {avoidValidation.Count}");
      Console.WriteLine($"Unseen validation removed from avoid: {avoidValidationUnseen.Count}");
      Console.WriteLine($"Remaining avoid count: {avoid.Count}");

      return (avoid, good);
    }

    public static void GenerateTrainingImages(List<Holding> avoid, List<Holding> good)
    {
      int imageSize=224;
      int dayCount=90;  // 90
      Console.WriteLine($"Generate training into {@"C:\Data"}");
      CNNProcessor.GenerateTraining(avoid, good, imageSize,dayCount, TestCase.GenerateType.BollingerBandWithVIX,@"C:\Data");
      ClearFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\0");
      ClearFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\1");
      if(!ProcessImages(@"C:\Data\0",@"C:\boneyard\DeepLearning\ModelInputData\0",imageSize))  // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\0
      {
        Console.WriteLine($"Process image failed, is the server running?");
      }
      if(!ProcessImages(@"C:\Data\1",@"C:\boneyard\DeepLearning\ModelInputData\1",imageSize))  // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\1                                                                             
      {
        Console.WriteLine($"Process image failed, is the server running?");
      }
      Console.WriteLine("Please copy these files into the training folder.");
    }

    /// <summary>
    /// This will generate images into  C:\boneyard\DeepLearning\ModelInputData\0 and  C:\boneyard\DeepLearning\ModelInputData\1
    /// You should then copy the generated images into C:\boneyard\DeepLearning\Data folder and then proceed to train tbe latest model
    /// which at the time of writing this is model_sk_convnext_v1.py.  After running the model you shoukd then run
    /// verify_model_sk_convnext_v1.py.  This will produce a validation score which at the time of writing is 99%.  It will also produce
    /// some output images including the confusion matrix.
    /// </summary>
    /// <param name="args"></param>
    static void Main(string[] args)
    {
// The modified flow
      //List<Holding> holdings = GenerateTrades();   // generate a holding set from the CMMomentum monthly candidates
      List<Holding> holdings = ReadHoldings("holdings.csv");  // read a holding set that was previously generated
      (List<Holding> avoid, List<Holding> good)=GenerateCodeTestCases(holdings);   // split the dataset into avoid and good
      GenerateTrainingImages(avoid, good);
// Clear cache at the end
      GBPriceCache.GetInstance().Dispose();
    }
  }
}