Sean/CNNImageProcessor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: Sean:CNNImageProcessor_0001
Branches Tags
Sean:master Sean:CNNImageProcessor_0005 Sean:CNNImageProcessior_0004 Sean:CNNImageProcessor_0002 Sean:CNNImageProcessor_0001
..
compare: Sean:CNNImageProcessor_0002
Branches Tags
Sean:CNNImageProcessor_0005 Sean:master Sean:CNNImageProcessior_0004 Sean:CNNImageProcessor_0002 Sean:CNNImageProcessor_0001

1 Commits
CNNImagePr ... CNNImagePr

Author SHA1 Message Date
Sean
116733bdf4 Commit Latest 2026-03-10 21:37:14 -04:00
3 changed files with 32 additions and 262 deletions
Expand all files Collapse all files

31
App.config
View File

@@ -1,29 +1,6 @@
<?xml version="1.0"?>
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<appSettings>
<add key="market_data" value="Database=market_data;Datasource=localhost;Username=root;Password=dbas"/>
<add key="portfolio_data" value="Database=portfolio_data;Datasource=localhost;Username=root;Password=dbas"/>
<add key="user_data" value="Database=user_data;Datasource=localhost;Username=root;Password=dbas"/>
<add key="sms_smtpaddress" value="smtp.gmail.com"/>
<add key="sms_smsusername" value="skessler1964@gmail.com"/>
<add key="sms_smspassword" value="xjfo isnf gmyi zovr"/>
<!--<add key="sms_smsrecipients" value="6315252496@vtext.com"/>-->
<add key="sms_smsrecipients" value="skessler1964sms@gmail.com"/>
<add key="proxy_address" value="http://127.0.0.1:8182"/>
<add key="proxy_GetLatestPriceYahoo" value="false"/>
<add key="proxy_GetLatestPriceFidelity" value="true"/>
<add key="proxy_GetLatestPriceBigCharts" value="false"/>
<add key="proxy_GetETFHoldings" value="false"/>
<add key="proxy_GetAnalystPriceTargetYahoo" value="true"/>
<add key="proxy_GetDailyPrices" value="false"/>
<add key="proxy_GetFundamentalEx" value="false"/>
<add key="proxy_GetDividendHistory" value="false"/>
<add key="proxy_GetAnalystPriceTargetMarketBeat" value="false"/>
<add key="proxy_GetCompanyHeadlinesSeekingAlphaV1" value="true"/>
<add key="proxy_GetCompanyHeadlinesSeekingAlphaV2" value="true"/>
</appSettings>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2"/>
</startup>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2"/>
</startup>
</configuration>

2
CNNImageProcessor.csproj
View File

@@ -9,7 +9,7 @@
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>CNNImageProcessor</RootNamespace>
<AssemblyName>CNNImageProcessor</AssemblyName>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<TargetFrameworkVersion>v4.6.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkProfile />
</PropertyGroup>

261
Program.cs
View File

@@ -1,15 +1,7 @@
using MarketData.Cache;
using MarketData.CNNProcessing;
using MarketData.DataAccess;
using MarketData.Generator.CMMomentum;
using MarketData.MarketDataModel;
using MarketData.Utils;
using System;
using System.Collections.Generic;
using System.Globalization;
using System;
using System.IO;
using System.Linq;
using System.Text;
using MarketData.CNNProcessing;
using MarketData.Utils;
namespace CNNImageProcessor
{
@@ -26,72 +18,6 @@ namespace CNNImageProcessor
GenerateImageData(@"C:\DeepLearningImageTests\DeepLearningImageData\RawData1",@"c:\DeepLearningImageTests\DeepLearningImageData\Data\1");
}
/// <summary>
/// Process all images in sourcePath through PIL on the CNNServer and save them to destinationFolder
/// </summary>
/// <param name="sourcePath"></param>
/// <param name="destinationPath"></param>
public static bool ProcessImages(String sourcePath, String destinationPath,String cnnClientUrl="http://10.0.0.73:5000")
{
String[] files = Directory.GetFiles(sourcePath,"*.jpg");
CNNClient cnnClient=new CNNClient(cnnClientUrl);
if(!cnnClient.Ping())
{
Console.WriteLine($"CNNServer at {cnnClientUrl} is not responding to ping");
return false;
}
foreach(String file in files)
{
Console.WriteLine($"Processing {file}");
ImageHelper imageHelper=new ImageHelper();
imageHelper.LoadImage(file);
imageHelper.ToGrayScale();
imageHelper.Resize(128,128);
Stream stream = imageHelper.ToStream();
Stream processed = cnnClient.ProcessImage(stream);
imageHelper.LoadImage(processed);
String pureFileName = Path.GetFileName(file);
String saveFileName = destinationPath + @"\" + pureFileName;
imageHelper.Save(saveFileName);
}
return true;
}
/// <summary>
/// Process all images in sourcePath through PIL on the CNNServer and save them to destinationFolder
/// </summary>
/// <param name="sourcePath"></param>
/// <param name="destinationPath"></param>
public static bool ProcessImages(String sourcePath, String destinationPath,int resizeTo,String cnnClientUrl="http://10.0.0.73:5000")
{
String[] files = Directory.GetFiles(sourcePath,"*.jpg");
CNNClient cnnClient=new CNNClient(cnnClientUrl);
if(!cnnClient.Ping())
{
Console.WriteLine($"CNNServer at {cnnClientUrl} is not responding to ping");
return false;
}
foreach(String file in files)
{
Console.WriteLine($"Processing {file}");
ImageHelper imageHelper=new ImageHelper();
imageHelper.LoadImage(file);
// imageHelper.ToGrayScale();
imageHelper.Resize(resizeTo,resizeTo);
Stream stream = imageHelper.ToStream();
Stream processed = cnnClient.ProcessImage(stream);
imageHelper.LoadImage(processed);
String pureFileName = Path.GetFileName(file);
String saveFileName = destinationPath + @"\" + pureFileName;
imageHelper.Save(saveFileName);
}
return true;
}
public static void GenerateImageData(String inputFolder,String destinationFolder)
{
ImageHelper imageHelper = new ImageHelper();
@@ -172,9 +98,6 @@ namespace CNNImageProcessor
}
}
/// <summary>
/// Processes an image through PIL on the CNN Server
/// </summary>
public static void ProcessImage()
{
ImageHelper imageHelper=new ImageHelper();
@@ -206,169 +129,39 @@ namespace CNNImageProcessor
}
}
public static List<Holding> GenerateTrades()
public static void ProcessImages(String sourcePath, String destinationPath)
{
List<Holding> holdings = new List<Holding>();
DateGenerator dateGenerator = new DateGenerator();
DateTime startDate = DateTime.Parse("10/31/2019");
DateTime endDate = DateTime.Parse("02/01/2026");
DateTime actualEndDate = endDate;
DateTime analysisDate = DateTime.Now;
String modelPathFileName = @"C:\boneyard\marketdata\bin\Debug\saferun\CM20191031.txt";
CMSessionParams sessionParams = CMSessionManager.RestoreSession(modelPathFileName);
startDate = dateGenerator.GetCurrentMonthEnd(startDate);
endDate = dateGenerator.GetCurrentMonthEnd(endDate);
actualEndDate = dateGenerator.GenerateHistoricalDate(endDate, 60);
DateTime runDate = startDate;
sessionParams.CMParams.UseCNN=false; // don't use the model
sessionParams.CMParams.MaxPositions=100; // take up to 100
while(runDate < actualEndDate)
String[] files = Directory.GetFiles(sourcePath,"*.jpg");
foreach(String file in files)
{
Console.WriteLine($"Running {runDate.ToShortDateString()}");
DateTime sellDate = dateGenerator.DaysAddActual(runDate, 90);
CMGeneratorResult result = CMMomentumGenerator.GenerateCMCandidates(runDate, analysisDate, sessionParams.CMParams, new List<string>());
Console.WriteLine($"Got {result.CMCandidates.Count} candidates for {runDate.ToShortDateString()}");
foreach (CMCandidate candidate in result.CMCandidates)
{
Holding holding = new Holding();
holding.Symbol = candidate.Symbol;
holding.PurchaseDate = runDate;
holding.SellDate = sellDate;
Price purchasePrice = GBPriceCache.GetInstance().GetPrice(holding.Symbol, holding.PurchaseDate);
Price sellPrice = GBPriceCache.GetInstance().GetPrice(holding.Symbol, holding.SellDate);
if (null == purchasePrice || null == sellPrice) continue;
holding.PurchasePrice = purchasePrice.Close;
holding.SellPrice = sellPrice.Close;
holding.GainLoss = holding.SellPrice - holding.PurchasePrice;
holding.GainLossPercent = ((holding.SellPrice - holding.PurchasePrice) / holding.PurchasePrice);
holdings.Add(holding);
}
runDate = dateGenerator.DaysAddActual(runDate, 30);
runDate = dateGenerator.GetCurrentMonthEnd(runDate);
Console.WriteLine($"Processing {file}");
ImageHelper imageHelper=new ImageHelper();
imageHelper.LoadImage(file);
imageHelper.ToGrayScale();
imageHelper.Resize(128,128);
Stream stream = imageHelper.ToStream();
CNNClient cnnClient = new CNNClient("http://10.0.0.73:5000");
Stream processed = cnnClient.ProcessImage(stream);
imageHelper.LoadImage(processed);
String pureFileName = Path.GetFileName(file);
String saveFileName = destinationPath + @"\" + pureFileName;
imageHelper.Save(saveFileName);
}
return holdings;
}
public static void GenerateTrainingImages()
{
// model training will happen on these folders C:\boneyard\DeepLearning\data\0 C:\boneyard\DeepLearning\data\1
CNNProcessor.GenerateTraining(@"C:\Data"); // This will generate into C:\Data\0 and C:\Data\1
ProcessImages(@"C:\Data\0",@"C:\boneyard\DeepLearning\ModelInputData\0"); // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\0
ProcessImages(@"C:\Data\1",@"C:\boneyard\DeepLearning\ModelInputData\1"); // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\1
}
public static void ClearFolderPath(String strFolderPath)
{
Console.WriteLine($"Cleaning {strFolderPath}");
if(String.IsNullOrEmpty(strFolderPath))throw new InvalidDataException($"{nameof(strFolderPath)} cannot be null");
if(!Directory.Exists(strFolderPath))
{
Directory.CreateDirectory(strFolderPath);
}
else
{
String[] pathFileNames = Directory.GetFiles(strFolderPath);
Console.WriteLine($"Deleting {pathFileNames.Length} files from {strFolderPath}");
foreach(String file in pathFileNames)
{
File.Delete(file);
}
}
}
public static List<Holding> ReadHoldings(String strPathFileName)
{
String strLine;
List<Holding> universe = new List<Holding>();
StreamReader inStream = new StreamReader(strPathFileName);
inStream.ReadLine(); // header
while (null != (strLine = inStream.ReadLine()))
{
Holding holding = Holding.FromString(strLine);
if (null == holding) continue;
universe.Add(holding);
}
inStream.Close();
inStream.Dispose();
Console.WriteLine($"Read {universe.Count} holdings");
return universe;
}
public static (List<Holding> avoid, List<Holding> good) GenerateCodeTestCases(List<Holding> universe)
{
double validationPercent=0.05;
double validationPercentUnseen=0.50;
Console.WriteLine($"Read {universe.Count} holdings");
List<Holding> avoid = universe.Where(x=>x.GainLoss<-.05).ToList();
List<Holding> good=universe.Where(x=>x.GainLoss>.05).ToList();
int validationCount = (int)(validationPercent * universe.Count);
Random rng = new Random();
List<Holding> goodValidation = good.OrderBy(x => rng.Next()).Take(validationCount).ToList();
int goodUnseenCount = (int)(validationPercentUnseen * goodValidation.Count);
List<Holding> goodValidationUnseen = goodValidation.OrderBy(x => rng.Next()).Take(goodUnseenCount).ToList();
good.RemoveAll(x => goodValidationUnseen.Contains(x));
Console.WriteLine($"Validation sample size: {goodValidation.Count}");
Console.WriteLine($"Unseen validation removed from good: {goodValidationUnseen.Count}");
Console.WriteLine($"Remaining good count: {good.Count}");
List<Holding> avoidValidation = avoid.OrderBy(x => rng.Next()).Take(validationCount).ToList();
int avoidUnseenCount = (int)(validationPercentUnseen * avoidValidation.Count);
List<Holding> avoidValidationUnseen = avoidValidation.OrderBy(x => rng.Next()).Take(avoidUnseenCount).ToList();
avoid.RemoveAll(x => avoidValidationUnseen.Contains(x));
Console.WriteLine($"Validation sample size: {avoidValidation.Count}");
Console.WriteLine($"Unseen validation removed from avoid: {avoidValidationUnseen.Count}");
Console.WriteLine($"Remaining avoid count: {avoid.Count}");
return (avoid, good);
}
public static void GenerateTrainingImages(List<Holding> avoid, List<Holding> good)
{
int imageSize=224;
int dayCount=90; // 90
Console.WriteLine($"Generate training into {@"C:\Data"}");
CNNProcessor.GenerateTraining(avoid, good, imageSize,dayCount, TestCase.GenerateType.BollingerBandWithVIX,@"C:\Data");
ClearFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\0");
ClearFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\1");
if(!ProcessImages(@"C:\Data\0",@"C:\boneyard\DeepLearning\ModelInputData\0",imageSize)) // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\0
{
Console.WriteLine($"Process image failed, is the server running?");
}
if(!ProcessImages(@"C:\Data\1",@"C:\boneyard\DeepLearning\ModelInputData\1",imageSize)) // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\1
{
Console.WriteLine($"Process image failed, is the server running?");
}
Console.WriteLine("Please copy these files into the training folder.");
}
/// <summary>
/// This will generate images into C:\boneyard\DeepLearning\ModelInputData\0 and C:\boneyard\DeepLearning\ModelInputData\1
/// You should then copy the generated images into C:\boneyard\DeepLearning\Data folder and then proceed to train tbe latest model
/// which at the time of writing this is model_sk_convnext_v1.py. After running the model you shoukd then run
/// verify_model_sk_convnext_v1.py. This will produce a validation score which at the time of writing is 99%. It will also produce
/// some output images including the confusion matrix.
/// </summary>
/// <param name="args"></param>
static void Main(string[] args)
{
// The modified flow
//List<Holding> holdings = GenerateTrades(); // generate a holding set from the CMMomentum monthly candidates
List<Holding> holdings = ReadHoldings("holdings.csv"); // read a holding set that was previously generated
(List<Holding> avoid, List<Holding> good)=GenerateCodeTestCases(holdings); // split the dataset into avoid and good
GenerateTrainingImages(avoid, good);
// Clear cache at the end
GBPriceCache.GetInstance().Dispose();
// ProcessImages(@"C:\Data\0",@"C:\DeepLearningImageTests\DeepLearningImageData\Data\0");
// ProcessImages(@"C:\Data\1",@"C:\DeepLearningImageTests\DeepLearningImageData\Data\1");
// GenerateImageData();
// TestCNN();
// ProcessImage();
// CreateValidationImages(@"C:\2",@"C:\3");
}
}
}