406 lines
18 KiB
C#
406 lines
18 KiB
C#
using MarketData.Cache;
|
|
using MarketData.CNNProcessing;
|
|
using MarketData.DataAccess;
|
|
using MarketData.Generator.CMMomentum;
|
|
using MarketData.MarketDataModel;
|
|
using MarketData.Utils;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Globalization;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Text;
|
|
|
|
namespace CNNImageProcessor
|
|
{
|
|
class Program
|
|
{
|
|
|
|
// **************************************************************************************************************************************************
|
|
// **************************************************************************************************************************************************
|
|
|
|
// You will then need to copy this data to C:\DeepLearningImageTests\DeepLearningImageData\Data
|
|
public static void GenerateImageData()
|
|
{
|
|
GenerateImageData(@"C:\DeepLearningImageTests\DeepLearningImageData\RawData0",@"c:\DeepLearningImageTests\DeepLearningImageData\Data\0");
|
|
GenerateImageData(@"C:\DeepLearningImageTests\DeepLearningImageData\RawData1",@"c:\DeepLearningImageTests\DeepLearningImageData\Data\1");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Process all images in sourcePath through PIL on the CNNServer and save them to destinationFolder
|
|
/// </summary>
|
|
/// <param name="sourcePath"></param>
|
|
/// <param name="destinationPath"></param>
|
|
public static bool ProcessImages(String sourcePath, String destinationPath,String cnnClientUrl="http://10.0.0.73:5000")
|
|
{
|
|
String[] files = Directory.GetFiles(sourcePath,"*.jpg");
|
|
|
|
CNNClient cnnClient=new CNNClient(cnnClientUrl);
|
|
if(!cnnClient.Ping())
|
|
{
|
|
Console.WriteLine($"CNNServer at {cnnClientUrl} is not responding to ping");
|
|
return false;
|
|
}
|
|
foreach(String file in files)
|
|
{
|
|
Console.WriteLine($"Processing {file}");
|
|
ImageHelper imageHelper=new ImageHelper();
|
|
imageHelper.LoadImage(file);
|
|
imageHelper.ToGrayScale();
|
|
imageHelper.Resize(128,128);
|
|
Stream stream = imageHelper.ToStream();
|
|
Stream processed = cnnClient.ProcessImage(stream);
|
|
imageHelper.LoadImage(processed);
|
|
String pureFileName = Path.GetFileName(file);
|
|
String saveFileName = destinationPath + @"\" + pureFileName;
|
|
imageHelper.Save(saveFileName);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Process all images in sourcePath through PIL on the CNNServer and save them to destinationFolder
|
|
/// </summary>
|
|
/// <param name="sourcePath"></param>
|
|
/// <param name="destinationPath"></param>
|
|
public static bool ProcessImages(String sourcePath, String destinationPath,int resizeTo,String cnnClientUrl="http://10.0.0.73:5000")
|
|
{
|
|
String[] files = Directory.GetFiles(sourcePath,"*.jpg");
|
|
|
|
CNNClient cnnClient=new CNNClient(cnnClientUrl);
|
|
if(!cnnClient.Ping())
|
|
{
|
|
Console.WriteLine($"CNNServer at {cnnClientUrl} is not responding to ping");
|
|
return false;
|
|
}
|
|
foreach(String file in files)
|
|
{
|
|
Console.WriteLine($"Processing {file}");
|
|
ImageHelper imageHelper=new ImageHelper();
|
|
imageHelper.LoadImage(file);
|
|
// imageHelper.ToGrayScale();
|
|
imageHelper.Resize(resizeTo,resizeTo);
|
|
Stream stream = imageHelper.ToStream();
|
|
Stream processed = cnnClient.ProcessImage(stream);
|
|
imageHelper.LoadImage(processed);
|
|
String pureFileName = Path.GetFileName(file);
|
|
String saveFileName = destinationPath + @"\" + pureFileName;
|
|
imageHelper.Save(saveFileName);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
|
|
|
|
public static void GenerateImageData(String inputFolder,String destinationFolder)
|
|
{
|
|
ImageHelper imageHelper = new ImageHelper();
|
|
String[] files = Directory.GetFiles(inputFolder, "*.jpg");
|
|
foreach (String file in files)
|
|
{
|
|
try
|
|
{
|
|
String fileName = Path.GetFileName(file);
|
|
String pathFileName = destinationFolder + @"\" + fileName;
|
|
String pathFileNameL = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"L.jpg";
|
|
String pathFileNameR = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"R.jpg";
|
|
String pathFileNameU = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"U.jpg";
|
|
String pathFileNameB1 = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"B1.jpg";
|
|
String pathFileNameB2 = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"B2.jpg";
|
|
String pathFileNameB3 = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"B3.jpg";
|
|
String pathFileNameB4 = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"B4.jpg";
|
|
|
|
Console.WriteLine(String.Format("Reading {0}", file));
|
|
imageHelper.LoadImage(file);
|
|
imageHelper.Resize(128, 128);
|
|
|
|
ImageHelper bmpLeft=new ImageHelper(imageHelper);
|
|
ImageHelper bmpRight=new ImageHelper(imageHelper);
|
|
ImageHelper bmpUDown=null;
|
|
bmpLeft.RotateLeft();
|
|
bmpRight.RotateRight();
|
|
bmpUDown=new ImageHelper(bmpRight);
|
|
bmpUDown.RotateRight();
|
|
|
|
//bmpLeft.ToGrayScale();
|
|
//bmpRight.ToGrayScale();
|
|
//bmpUDown.ToGrayScale();
|
|
//imageHelper.ToGrayScale();
|
|
|
|
imageHelper.Save(pathFileName);
|
|
bmpLeft.Save(pathFileNameL);
|
|
bmpRight.Save(pathFileNameR);
|
|
bmpUDown.Save(pathFileNameU);
|
|
|
|
ImageHelper bmpBlur1=new ImageHelper(imageHelper);
|
|
ImageHelper bmpBlur2=new ImageHelper(bmpLeft);
|
|
ImageHelper bmpBlur3=new ImageHelper(bmpRight);
|
|
ImageHelper bmpBlur4=new ImageHelper(bmpUDown);
|
|
bmpBlur1.Blur(1);
|
|
bmpBlur2.Blur(1);
|
|
bmpBlur3.Blur(1);
|
|
bmpBlur4.Blur(1);
|
|
bmpBlur1.Save(pathFileNameB1);
|
|
bmpBlur2.Save(pathFileNameB2);
|
|
bmpBlur3.Save(pathFileNameB3);
|
|
bmpBlur4.Save(pathFileNameB4);
|
|
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
Console.WriteLine(exception.ToString());
|
|
}
|
|
}
|
|
}
|
|
|
|
public static void TestCNN()
|
|
{
|
|
ImageHelper imageHelper = null;
|
|
Stream stream = null;
|
|
|
|
String sourceFolder = @"C:\DeepLearningImageTests\DeepLearningImageData\Validation";
|
|
String[] files = Directory.GetFiles(sourceFolder,"*.jpg");
|
|
|
|
foreach(String file in files)
|
|
{
|
|
imageHelper=new ImageHelper();
|
|
imageHelper.LoadImage(file);
|
|
stream = imageHelper.ToStream();
|
|
CNNClient cnnClient = new CNNClient("http://10.0.0.73:5000");
|
|
String result = cnnClient.Predict(CNNClient.Model.vgg16,stream);
|
|
Console.WriteLine(String.Format("Result:{0} File:{1}",result,file));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Processes an image through PIL on the CNN Server
|
|
/// </summary>
|
|
public static void ProcessImage()
|
|
{
|
|
ImageHelper imageHelper=new ImageHelper();
|
|
imageHelper.LoadImage(@"C:\DeepLearningImageTests\DeepLearningImageData\RawData0\00de4729-6aa9-465e-906b-4c92bc24f7a9.jpg");
|
|
Stream stream = imageHelper.ToStream();
|
|
CNNClient cnnClient = new CNNClient("http://10.0.0.73:5000");
|
|
Stream result = cnnClient.ProcessImage(stream);
|
|
imageHelper.LoadImage(stream);
|
|
imageHelper.Save(@"c:\2\image.jpg");
|
|
}
|
|
|
|
public static void CreateValidationImages(String sourcePath, String destinationPath)
|
|
{
|
|
String[] files = Directory.GetFiles(sourcePath,"*.jpg");
|
|
foreach(String file in files)
|
|
{
|
|
Console.WriteLine($"Processing {file}");
|
|
ImageHelper imageHelper=new ImageHelper();
|
|
imageHelper.LoadImage(file);
|
|
imageHelper.ToGrayScale();
|
|
imageHelper.Resize(128,128);
|
|
Stream stream = imageHelper.ToStream();
|
|
CNNClient cnnClient = new CNNClient("http://10.0.0.73:5000");
|
|
Stream processed = cnnClient.ProcessImage(stream);
|
|
imageHelper.LoadImage(processed);
|
|
String pureFileName = Path.GetFileName(file);
|
|
String saveFileName = destinationPath + @"\" + pureFileName;
|
|
imageHelper.Save(saveFileName);
|
|
}
|
|
}
|
|
|
|
public static List<Holding> GenerateTrades()
|
|
{
|
|
List<Holding> holdings = new List<Holding>();
|
|
DateGenerator dateGenerator = new DateGenerator();
|
|
DateTime startDate = DateTime.Parse("10/31/2019");
|
|
DateTime endDate = DateTime.Parse("12/31/2025");
|
|
DateTime actualEndDate = endDate;
|
|
DateTime analysisDate = DateTime.Now;
|
|
|
|
String modelPathFileName = @"C:\boneyard\marketdata\bin\Debug\saferun\CM20191031.txt";
|
|
CMSessionParams sessionParams = CMSessionManager.RestoreSession(modelPathFileName);
|
|
|
|
startDate = dateGenerator.GetCurrentMonthEnd(startDate);
|
|
endDate = dateGenerator.GetCurrentMonthEnd(endDate);
|
|
actualEndDate = dateGenerator.GenerateHistoricalDate(endDate, 60);
|
|
DateTime runDate = startDate;
|
|
|
|
sessionParams.CMParams.UseCNN=false; // don't use the model
|
|
sessionParams.CMParams.MaxPositions=100; // take up to 100
|
|
while(runDate < actualEndDate)
|
|
{
|
|
Console.WriteLine($"Running {runDate.ToShortDateString()}");
|
|
DateTime sellDate = dateGenerator.DaysAddActual(runDate, 90);
|
|
|
|
CMGeneratorResult result = CMMomentumGenerator.GenerateCMCandidates(runDate, analysisDate, sessionParams.CMParams, new List<string>());
|
|
Console.WriteLine($"Got {result.CMCandidates.Count} candidates for {runDate.ToShortDateString()}");
|
|
foreach (CMCandidate candidate in result.CMCandidates)
|
|
{
|
|
Holding holding = new Holding();
|
|
holding.Symbol = candidate.Symbol;
|
|
holding.PurchaseDate = runDate;
|
|
holding.SellDate = sellDate;
|
|
Price purchasePrice = GBPriceCache.GetInstance().GetPrice(holding.Symbol, holding.PurchaseDate);
|
|
Price sellPrice = GBPriceCache.GetInstance().GetPrice(holding.Symbol, holding.SellDate);
|
|
if (null == purchasePrice || null == sellPrice) continue;
|
|
holding.PurchasePrice = purchasePrice.Close;
|
|
holding.SellPrice = sellPrice.Close;
|
|
holding.GainLoss = holding.SellPrice - holding.PurchasePrice;
|
|
holding.GainLossPercent = ((holding.SellPrice - holding.PurchasePrice) / holding.PurchasePrice);
|
|
holdings.Add(holding);
|
|
}
|
|
runDate = dateGenerator.DaysAddActual(runDate, 30);
|
|
runDate = dateGenerator.GetCurrentMonthEnd(runDate);
|
|
}
|
|
return holdings;
|
|
}
|
|
|
|
|
|
public static void GenerateTrainingImages()
|
|
{
|
|
// model training will happen on these folders C:\boneyard\DeepLearning\data\0 C:\boneyard\DeepLearning\data\1
|
|
CNNProcessor.GenerateTraining(@"C:\Data"); // This will generate into C:\Data\0 and C:\Data\1
|
|
ProcessImages(@"C:\Data\0",@"C:\boneyard\DeepLearning\ModelInputData\0"); // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\0
|
|
ProcessImages(@"C:\Data\1",@"C:\boneyard\DeepLearning\ModelInputData\1"); // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\1
|
|
}
|
|
|
|
|
|
public static void ClearFolderPath(String strFolderPath)
|
|
{
|
|
Console.WriteLine($"Cleaning {strFolderPath}");
|
|
if(String.IsNullOrEmpty(strFolderPath))throw new InvalidDataException($"{nameof(strFolderPath)} cannot be null");
|
|
if(!Directory.Exists(strFolderPath))
|
|
{
|
|
Directory.CreateDirectory(strFolderPath);
|
|
}
|
|
else
|
|
{
|
|
String[] pathFileNames = Directory.GetFiles(strFolderPath);
|
|
Console.WriteLine($"Deleting {pathFileNames.Length} files from {strFolderPath}");
|
|
foreach(String file in pathFileNames)
|
|
{
|
|
File.Delete(file);
|
|
}
|
|
}
|
|
}
|
|
|
|
public static List<Holding> ReadHoldings(String strPathFileName)
|
|
{
|
|
String strLine;
|
|
List<Holding> universe = new List<Holding>();
|
|
StreamReader inStream = new StreamReader(strPathFileName);
|
|
inStream.ReadLine(); // header
|
|
while (null != (strLine = inStream.ReadLine()))
|
|
{
|
|
Holding holding = Holding.FromString(strLine);
|
|
if (null == holding) continue;
|
|
universe.Add(holding);
|
|
}
|
|
inStream.Close();
|
|
inStream.Dispose();
|
|
Console.WriteLine($"Read {universe.Count} holdings");
|
|
return universe;
|
|
}
|
|
|
|
public static void WriteHoldings(List<Holding> holdings,String strPathFileName)
|
|
{
|
|
if(File.Exists(strPathFileName))File.Delete(strPathFileName);
|
|
StreamWriter outStream = new StreamWriter(strPathFileName);
|
|
outStream.WriteLine(Holding.Heading);
|
|
foreach(Holding holding in holdings)
|
|
{
|
|
outStream.WriteLine(holding);
|
|
}
|
|
outStream.Flush();
|
|
outStream.Close();
|
|
outStream.Dispose();
|
|
}
|
|
|
|
public static (List<Holding> avoid, List<Holding> good) GenerateCodeTestCases(List<Holding> universe)
|
|
{
|
|
double validationPercent=0.05;
|
|
double validationPercentUnseen=0.50;
|
|
Console.WriteLine($"Read {universe.Count} holdings");
|
|
|
|
List<Holding> avoid = universe.Where(x=>x.GainLoss<-.05).ToList();
|
|
List<Holding> good=universe.Where(x=>x.GainLoss>.05).ToList();
|
|
|
|
int validationCount = (int)(validationPercent * universe.Count);
|
|
|
|
Random rng = new Random();
|
|
List<Holding> goodValidation = good.OrderBy(x => rng.Next()).Take(validationCount).ToList();
|
|
int goodUnseenCount = (int)(validationPercentUnseen * goodValidation.Count);
|
|
List<Holding> goodValidationUnseen = goodValidation.OrderBy(x => rng.Next()).Take(goodUnseenCount).ToList();
|
|
good.RemoveAll(x => goodValidationUnseen.Contains(x));
|
|
Console.WriteLine($"Validation sample size: {goodValidation.Count}");
|
|
Console.WriteLine($"Unseen validation removed from good: {goodValidationUnseen.Count}");
|
|
Console.WriteLine($"Remaining good count: {good.Count}");
|
|
|
|
|
|
List<Holding> avoidValidation = avoid.OrderBy(x => rng.Next()).Take(validationCount).ToList();
|
|
int avoidUnseenCount = (int)(validationPercentUnseen * avoidValidation.Count);
|
|
List<Holding> avoidValidationUnseen = avoidValidation.OrderBy(x => rng.Next()).Take(avoidUnseenCount).ToList();
|
|
avoid.RemoveAll(x => avoidValidationUnseen.Contains(x));
|
|
Console.WriteLine($"Validation sample size: {avoidValidation.Count}");
|
|
Console.WriteLine($"Unseen validation removed from avoid: {avoidValidationUnseen.Count}");
|
|
Console.WriteLine($"Remaining avoid count: {avoid.Count}");
|
|
|
|
return (avoid, good);
|
|
}
|
|
|
|
public static void GenerateTrainingImages(List<Holding> avoid, List<Holding> good)
|
|
{
|
|
String cnnClientUrl="http://127.0.0.1:5000";
|
|
int imageSize=224;
|
|
int dayCount=90;
|
|
Console.WriteLine($"Generate training into {@"C:\boneyard\DeepLearning\ModelInputData"}");
|
|
CNNProcessor.GenerateTraining(avoid, good, imageSize,dayCount, TestCase.GenerateType.BollingerBandWithVIX,@"C:\boneyard\DeepLearning\ModelInputData");
|
|
ClearFolderPath(@"C:\boneyard\DeepLearning\Data\0");
|
|
ClearFolderPath(@"C:\boneyard\DeepLearning\Data\1");
|
|
CNNClient cnnClient = new CNNClient(cnnClientUrl);
|
|
if(!cnnClient.Ping())
|
|
{
|
|
Console.WriteLine($"CNN Server @ {cnnClientUrl} is not responding.");
|
|
return;
|
|
}
|
|
ProcessImages(@"C:\boneyard\DeepLearning\ModelInputData\0",@"C:\boneyard\DeepLearning\Data\0",imageSize,cnnClientUrl); // Process through PIL and put in C:\boneyard\DeepLearning\Data\0
|
|
ProcessImages(@"C:\boneyard\DeepLearning\ModelInputData\1",@"C:\boneyard\DeepLearning\Data\1",imageSize,cnnClientUrl); // Process through PIL and put in C:\boneyard\DeepLearning\Data\1
|
|
Console.WriteLine("Done.");
|
|
}
|
|
|
|
/// <summary>
|
|
/// This will generate images into C:\boneyard\DeepLearning\Data\0 and C:\boneyard\DeepLearning\Data\1
|
|
/// You should then proceed to train tbe latest model which at the time of writing this is model_sk_convnext_v1.py.
|
|
/// After running the model you should then run verify_model_sk_convnext_v1.py. This will produce a validation score
|
|
/// which at the time of writing is 99%. It will also produce some output images including the confusion matrix.
|
|
///
|
|
/// I am using WSL2 to perform the training because WSL2 is the only option for tensorflow with GPU.
|
|
/// To launch WSL2 at a command prompt type "wsl ~". If the enviroment is not set up then you can use the setup_tf_gpu.sh
|
|
/// script in CNN/Scripts folder to re-create the full Python3 environment and Tensorflow. The script will create the
|
|
/// virtual environment and install everything. It was used to create the current WSL enviroment.
|
|
/// To start the environment "source tf_gpu/bin/activate"
|
|
/// then type "code ." This will run VSCODE and attach to the WSL environment.
|
|
/// Train the model on EUPORIE laptop using the GPU card with WSL2. (Windows Subsystem For Linux). I am running Ubuntu1 22.04.2
|
|
/// To launch WSL open up a command prompt, run powershell and type "wsl ~".
|
|
/// The folder structure will be /home/pi/CNN /home/pi/DeepLearning
|
|
/// You can access the folder structure through windows explorer. type "\\wsl$" in explorer and navigate to the folder.
|
|
/// drop in the Data and Model and run the model.
|
|
/// There is a shell script in the Scripts folder of the CNN project. setup_tf_gpu.sh
|
|
/// Copy the script to the CNN folder and run it from the CNN folder.
|
|
/// It will create the venv enviroment and install python 3.10 and tensorflow (gpu)
|
|
///
|
|
/// ******************************************************************************************************************** ///
|
|
/// </summary>
|
|
/// <param name="args"></param>
|
|
static void Main(string[] args)
|
|
{
|
|
// The modified flow
|
|
// List<Holding> holdings = GenerateTrades(); // generate a new holding set from the CMMomentum monthly candidates
|
|
// WriteHoldings(holdings,"holdings.csv"); // save the list. The saved list can be read back in to save time in case reruns are necessary
|
|
List<Holding> holdings = ReadHoldings("holdings.csv"); // read a holding set that was previously generated. You'll want to create a new set of holdings for retraiing
|
|
(List<Holding> avoid, List<Holding> good)=GenerateCodeTestCases(holdings); // split the dataset into avoid and good
|
|
GenerateTrainingImages(avoid, good); // Generate the training images
|
|
// Clear cache at the end
|
|
GBPriceCache.GetInstance().Dispose();
|
|
}
|
|
}
|
|
}
|