Sean/CNNImageProcessor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: Sean:CNNImageProcessor_0001
Branches Tags
Sean:master Sean:CNNImageProcessor_0005 Sean:CNNImageProcessior_0004 Sean:CNNImageProcessor_0002 Sean:CNNImageProcessor_0001
..
compare: Sean:CNNImageProcessor_0002
Branches Tags
Sean:CNNImageProcessor_0005 Sean:master Sean:CNNImageProcessior_0004 Sean:CNNImageProcessor_0002 Sean:CNNImageProcessor_0001

1 Commits
CNNImagePr ... CNNImagePr

Author SHA1 Message Date
Sean
116733bdf4 Commit Latest 2026-03-10 21:37:14 -04:00
3 changed files with 32 additions and 262 deletions
Expand all files Collapse all files

27
App.config
View File

@@ -1,29 +1,6 @@
<?xml version="1.0"?> <?xml version="1.0" encoding="utf-8"?>
<configuration> <configuration>
<appSettings>
<add key="market_data" value="Database=market_data;Datasource=localhost;Username=root;Password=dbas"/>
<add key="portfolio_data" value="Database=portfolio_data;Datasource=localhost;Username=root;Password=dbas"/>
<add key="user_data" value="Database=user_data;Datasource=localhost;Username=root;Password=dbas"/>
<add key="sms_smtpaddress" value="smtp.gmail.com"/>
<add key="sms_smsusername" value="skessler1964@gmail.com"/>
<add key="sms_smspassword" value="xjfo isnf gmyi zovr"/>
<!--<add key="sms_smsrecipients" value="6315252496@vtext.com"/>-->
<add key="sms_smsrecipients" value="skessler1964sms@gmail.com"/>
<add key="proxy_address" value="http://127.0.0.1:8182"/>
<add key="proxy_GetLatestPriceYahoo" value="false"/>
<add key="proxy_GetLatestPriceFidelity" value="true"/>
<add key="proxy_GetLatestPriceBigCharts" value="false"/>
<add key="proxy_GetETFHoldings" value="false"/>
<add key="proxy_GetAnalystPriceTargetYahoo" value="true"/>
<add key="proxy_GetDailyPrices" value="false"/>
<add key="proxy_GetFundamentalEx" value="false"/>
<add key="proxy_GetDividendHistory" value="false"/>
<add key="proxy_GetAnalystPriceTargetMarketBeat" value="false"/>
<add key="proxy_GetCompanyHeadlinesSeekingAlphaV1" value="true"/>
<add key="proxy_GetCompanyHeadlinesSeekingAlphaV2" value="true"/>
</appSettings>
<startup> <startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2"/> <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2"/>
</startup> </startup>
</configuration> </configuration>

2
CNNImageProcessor.csproj
View File

@@ -9,7 +9,7 @@
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>CNNImageProcessor</RootNamespace> <RootNamespace>CNNImageProcessor</RootNamespace>
<AssemblyName>CNNImageProcessor</AssemblyName> <AssemblyName>CNNImageProcessor</AssemblyName>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion> <TargetFrameworkVersion>v4.6.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<TargetFrameworkProfile /> <TargetFrameworkProfile />
</PropertyGroup> </PropertyGroup>

261
Program.cs
View File

@@ -1,15 +1,7 @@
using MarketData.Cache; using System;
using MarketData.CNNProcessing;
using MarketData.DataAccess;
using MarketData.Generator.CMMomentum;
using MarketData.MarketDataModel;
using MarketData.Utils;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO; using System.IO;
using System.Linq; using MarketData.CNNProcessing;
using System.Text; using MarketData.Utils;
namespace CNNImageProcessor namespace CNNImageProcessor
{ {
@@ -26,72 +18,6 @@ namespace CNNImageProcessor
GenerateImageData(@"C:\DeepLearningImageTests\DeepLearningImageData\RawData1",@"c:\DeepLearningImageTests\DeepLearningImageData\Data\1"); GenerateImageData(@"C:\DeepLearningImageTests\DeepLearningImageData\RawData1",@"c:\DeepLearningImageTests\DeepLearningImageData\Data\1");
} }
/// <summary>
/// Process all images in sourcePath through PIL on the CNNServer and save them to destinationFolder
/// </summary>
/// <param name="sourcePath"></param>
/// <param name="destinationPath"></param>
public static bool ProcessImages(String sourcePath, String destinationPath,String cnnClientUrl="http://10.0.0.73:5000")
{
String[] files = Directory.GetFiles(sourcePath,"*.jpg");
CNNClient cnnClient=new CNNClient(cnnClientUrl);
if(!cnnClient.Ping())
{
Console.WriteLine($"CNNServer at {cnnClientUrl} is not responding to ping");
return false;
}
foreach(String file in files)
{
Console.WriteLine($"Processing {file}");
ImageHelper imageHelper=new ImageHelper();
imageHelper.LoadImage(file);
imageHelper.ToGrayScale();
imageHelper.Resize(128,128);
Stream stream = imageHelper.ToStream();
Stream processed = cnnClient.ProcessImage(stream);
imageHelper.LoadImage(processed);
String pureFileName = Path.GetFileName(file);
String saveFileName = destinationPath + @"\" + pureFileName;
imageHelper.Save(saveFileName);
}
return true;
}
/// <summary>
/// Process all images in sourcePath through PIL on the CNNServer and save them to destinationFolder
/// </summary>
/// <param name="sourcePath"></param>
/// <param name="destinationPath"></param>
public static bool ProcessImages(String sourcePath, String destinationPath,int resizeTo,String cnnClientUrl="http://10.0.0.73:5000")
{
String[] files = Directory.GetFiles(sourcePath,"*.jpg");
CNNClient cnnClient=new CNNClient(cnnClientUrl);
if(!cnnClient.Ping())
{
Console.WriteLine($"CNNServer at {cnnClientUrl} is not responding to ping");
return false;
}
foreach(String file in files)
{
Console.WriteLine($"Processing {file}");
ImageHelper imageHelper=new ImageHelper();
imageHelper.LoadImage(file);
// imageHelper.ToGrayScale();
imageHelper.Resize(resizeTo,resizeTo);
Stream stream = imageHelper.ToStream();
Stream processed = cnnClient.ProcessImage(stream);
imageHelper.LoadImage(processed);
String pureFileName = Path.GetFileName(file);
String saveFileName = destinationPath + @"\" + pureFileName;
imageHelper.Save(saveFileName);
}
return true;
}
public static void GenerateImageData(String inputFolder,String destinationFolder) public static void GenerateImageData(String inputFolder,String destinationFolder)
{ {
ImageHelper imageHelper = new ImageHelper(); ImageHelper imageHelper = new ImageHelper();
@@ -172,9 +98,6 @@ namespace CNNImageProcessor
} }
} }
/// <summary>
/// Processes an image through PIL on the CNN Server
/// </summary>
public static void ProcessImage() public static void ProcessImage()
{ {
ImageHelper imageHelper=new ImageHelper(); ImageHelper imageHelper=new ImageHelper();
@@ -206,169 +129,39 @@ namespace CNNImageProcessor
} }
} }
public static List<Holding> GenerateTrades() public static void ProcessImages(String sourcePath, String destinationPath)
{ {
List<Holding> holdings = new List<Holding>(); String[] files = Directory.GetFiles(sourcePath,"*.jpg");
DateGenerator dateGenerator = new DateGenerator(); foreach(String file in files)
DateTime startDate = DateTime.Parse("10/31/2019");
DateTime endDate = DateTime.Parse("02/01/2026");
DateTime actualEndDate = endDate;
DateTime analysisDate = DateTime.Now;
String modelPathFileName = @"C:\boneyard\marketdata\bin\Debug\saferun\CM20191031.txt";
CMSessionParams sessionParams = CMSessionManager.RestoreSession(modelPathFileName);
startDate = dateGenerator.GetCurrentMonthEnd(startDate);
endDate = dateGenerator.GetCurrentMonthEnd(endDate);
actualEndDate = dateGenerator.GenerateHistoricalDate(endDate, 60);
DateTime runDate = startDate;
sessionParams.CMParams.UseCNN=false; // don't use the model
sessionParams.CMParams.MaxPositions=100; // take up to 100
while(runDate < actualEndDate)
{ {
Console.WriteLine($"Running {runDate.ToShortDateString()}"); Console.WriteLine($"Processing {file}");
DateTime sellDate = dateGenerator.DaysAddActual(runDate, 90); ImageHelper imageHelper=new ImageHelper();
imageHelper.LoadImage(file);
CMGeneratorResult result = CMMomentumGenerator.GenerateCMCandidates(runDate, analysisDate, sessionParams.CMParams, new List<string>()); imageHelper.ToGrayScale();
Console.WriteLine($"Got {result.CMCandidates.Count} candidates for {runDate.ToShortDateString()}"); imageHelper.Resize(128,128);
foreach (CMCandidate candidate in result.CMCandidates) Stream stream = imageHelper.ToStream();
{ CNNClient cnnClient = new CNNClient("http://10.0.0.73:5000");
Holding holding = new Holding(); Stream processed = cnnClient.ProcessImage(stream);
holding.Symbol = candidate.Symbol; imageHelper.LoadImage(processed);
holding.PurchaseDate = runDate; String pureFileName = Path.GetFileName(file);
holding.SellDate = sellDate; String saveFileName = destinationPath + @"\" + pureFileName;
Price purchasePrice = GBPriceCache.GetInstance().GetPrice(holding.Symbol, holding.PurchaseDate); imageHelper.Save(saveFileName);
Price sellPrice = GBPriceCache.GetInstance().GetPrice(holding.Symbol, holding.SellDate);
if (null == purchasePrice || null == sellPrice) continue;
holding.PurchasePrice = purchasePrice.Close;
holding.SellPrice = sellPrice.Close;
holding.GainLoss = holding.SellPrice - holding.PurchasePrice;
holding.GainLossPercent = ((holding.SellPrice - holding.PurchasePrice) / holding.PurchasePrice);
holdings.Add(holding);
}
runDate = dateGenerator.DaysAddActual(runDate, 30);
runDate = dateGenerator.GetCurrentMonthEnd(runDate);
}
return holdings;
}
public static void GenerateTrainingImages()
{
// model training will happen on these folders C:\boneyard\DeepLearning\data\0 C:\boneyard\DeepLearning\data\1
CNNProcessor.GenerateTraining(@"C:\Data"); // This will generate into C:\Data\0 and C:\Data\1
ProcessImages(@"C:\Data\0",@"C:\boneyard\DeepLearning\ModelInputData\0"); // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\0
ProcessImages(@"C:\Data\1",@"C:\boneyard\DeepLearning\ModelInputData\1"); // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\1
}
public static void ClearFolderPath(String strFolderPath)
{
Console.WriteLine($"Cleaning {strFolderPath}");
if(String.IsNullOrEmpty(strFolderPath))throw new InvalidDataException($"{nameof(strFolderPath)} cannot be null");
if(!Directory.Exists(strFolderPath))
{
Directory.CreateDirectory(strFolderPath);
}
else
{
String[] pathFileNames = Directory.GetFiles(strFolderPath);
Console.WriteLine($"Deleting {pathFileNames.Length} files from {strFolderPath}");
foreach(String file in pathFileNames)
{
File.Delete(file);
}
} }
} }
public static List<Holding> ReadHoldings(String strPathFileName)
{
String strLine;
List<Holding> universe = new List<Holding>();
StreamReader inStream = new StreamReader(strPathFileName);
inStream.ReadLine(); // header
while (null != (strLine = inStream.ReadLine()))
{
Holding holding = Holding.FromString(strLine);
if (null == holding) continue;
universe.Add(holding);
}
inStream.Close();
inStream.Dispose();
Console.WriteLine($"Read {universe.Count} holdings");
return universe;
}
public static (List<Holding> avoid, List<Holding> good) GenerateCodeTestCases(List<Holding> universe)
{
double validationPercent=0.05;
double validationPercentUnseen=0.50;
Console.WriteLine($"Read {universe.Count} holdings");
List<Holding> avoid = universe.Where(x=>x.GainLoss<-.05).ToList();
List<Holding> good=universe.Where(x=>x.GainLoss>.05).ToList();
int validationCount = (int)(validationPercent * universe.Count);
Random rng = new Random();
List<Holding> goodValidation = good.OrderBy(x => rng.Next()).Take(validationCount).ToList();
int goodUnseenCount = (int)(validationPercentUnseen * goodValidation.Count);
List<Holding> goodValidationUnseen = goodValidation.OrderBy(x => rng.Next()).Take(goodUnseenCount).ToList();
good.RemoveAll(x => goodValidationUnseen.Contains(x));
Console.WriteLine($"Validation sample size: {goodValidation.Count}");
Console.WriteLine($"Unseen validation removed from good: {goodValidationUnseen.Count}");
Console.WriteLine($"Remaining good count: {good.Count}");
List<Holding> avoidValidation = avoid.OrderBy(x => rng.Next()).Take(validationCount).ToList();
int avoidUnseenCount = (int)(validationPercentUnseen * avoidValidation.Count);
List<Holding> avoidValidationUnseen = avoidValidation.OrderBy(x => rng.Next()).Take(avoidUnseenCount).ToList();
avoid.RemoveAll(x => avoidValidationUnseen.Contains(x));
Console.WriteLine($"Validation sample size: {avoidValidation.Count}");
Console.WriteLine($"Unseen validation removed from avoid: {avoidValidationUnseen.Count}");
Console.WriteLine($"Remaining avoid count: {avoid.Count}");
return (avoid, good);
}
public static void GenerateTrainingImages(List<Holding> avoid, List<Holding> good)
{
int imageSize=224;
int dayCount=90; // 90
Console.WriteLine($"Generate training into {@"C:\Data"}");
CNNProcessor.GenerateTraining(avoid, good, imageSize,dayCount, TestCase.GenerateType.BollingerBandWithVIX,@"C:\Data");
ClearFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\0");
ClearFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\1");
if(!ProcessImages(@"C:\Data\0",@"C:\boneyard\DeepLearning\ModelInputData\0",imageSize)) // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\0
{
Console.WriteLine($"Process image failed, is the server running?");
}
if(!ProcessImages(@"C:\Data\1",@"C:\boneyard\DeepLearning\ModelInputData\1",imageSize)) // Process through PIL and put in C:\boneyard\DeepLearning\ModelInputData\1
{
Console.WriteLine($"Process image failed, is the server running?");
}
Console.WriteLine("Please copy these files into the training folder.");
}
/// <summary>
/// This will generate images into C:\boneyard\DeepLearning\ModelInputData\0 and C:\boneyard\DeepLearning\ModelInputData\1
/// You should then copy the generated images into C:\boneyard\DeepLearning\Data folder and then proceed to train tbe latest model
/// which at the time of writing this is model_sk_convnext_v1.py. After running the model you shoukd then run
/// verify_model_sk_convnext_v1.py. This will produce a validation score which at the time of writing is 99%. It will also produce
/// some output images including the confusion matrix.
/// </summary>
/// <param name="args"></param>
static void Main(string[] args) static void Main(string[] args)
{ {
// The modified flow // ProcessImages(@"C:\Data\0",@"C:\DeepLearningImageTests\DeepLearningImageData\Data\0");
//List<Holding> holdings = GenerateTrades(); // generate a holding set from the CMMomentum monthly candidates // ProcessImages(@"C:\Data\1",@"C:\DeepLearningImageTests\DeepLearningImageData\Data\1");
List<Holding> holdings = ReadHoldings("holdings.csv"); // read a holding set that was previously generated
(List<Holding> avoid, List<Holding> good)=GenerateCodeTestCases(holdings); // split the dataset into avoid and good
GenerateTrainingImages(avoid, good); // GenerateImageData();
// Clear cache at the end // TestCNN();
GBPriceCache.GetInstance().Dispose(); // ProcessImage();
// CreateValidationImages(@"C:\2",@"C:\3");
} }
} }
} }