Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
20a2fa8dcbf7b5e01300b26df96cdecb6c30238f
marketdata/MarketDataLib/CNNProcessing/CNNProcessor.cs
Sean 9146dca17b Updating training October 2024
2024-10-26 12:26:46 -04:00

208 lines
15 KiB
C#
Raw Blame History

using System;
using System.IO;
using System.Collections.Generic;
using MarketData.Utils;
using System.Text;
namespace MarketData.CNNProcessing
{
public class CNNProcessor
{
private static int dayCount=270;
private static int width=128;
private static int height=128;
private CNNProcessor()
{
}
public static void GenerateTraining()
{
TestCases testCases=new TestCases();
DataProcessor dataProcessor=new DataProcessor();
dataProcessor.Width=width;
dataProcessor.Height=height;
dataProcessor.PenWidthArray=new float[]{.50f,.75f,1.00f,1.12f,1.25f,1.31f,1.37f,1.50f,1.56f,1.62f,1.75f,1.87f,2.00f};
// [0] Data - The avoid data
testCases.Add(new TestCase("CENX",DateTime.Parse("03/31/2022"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("ICPT",DateTime.Parse("12/31/2019"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("XMTR",DateTime.Parse("01/31/2024"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("CDXS",DateTime.Parse("11/30/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("EXTR",DateTime.Parse("08/31/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("DISCA",DateTime.Parse("02/26/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("DOCN",DateTime.Parse("10/29/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("CCL",DateTime.Parse("07/31/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("NET",DateTime.Parse("11/30/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("F",DateTime.Parse("01/31/2022"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("RCKT",DateTime.Parse("01/31/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("ADT",DateTime.Parse("11/29/2019"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("MOS",DateTime.Parse("03/31/2022"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("FSM",DateTime.Parse("05/31/2024"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("ITOS",DateTime.Parse("01/31/2022"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("ENPH",DateTime.Parse("01/29/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("GOSS",DateTime.Parse("10/29/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("KURA",DateTime.Parse("11/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("WIRE",DateTime.Parse("12/31/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("CUBI",DateTime.Parse("12/31/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("SWAV",DateTime.Parse("05/31/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("SPSC",DateTime.Parse("10/29/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("EPC",DateTime.Parse("01/31/2022"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("FTCH",DateTime.Parse("12/31/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("VC",DateTime.Parse("10/31/2019"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("VRT",DateTime.Parse("04/30/2024"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("LPG",DateTime.Parse("11/30/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("LI",DateTime.Parse("08/31/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("INBX",DateTime.Parse("01/31/2024"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("WYNN",DateTime.Parse("02/28/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
dataProcessor.SetOutputFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\0");
dataProcessor.ProcessData(testCases);
testCases.Clear();
// [1] Data - The good data
testCases.Add(new TestCase("CPG",DateTime.Parse("03/31/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("VCEL",DateTime.Parse("02/26/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("CLF",DateTime.Parse("01/29/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("XPO",DateTime.Parse("08/31/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("RH",DateTime.Parse("10/31/2019"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("NEM",DateTime.Parse("05/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("EXPI",DateTime.Parse("08/31/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("INTR",DateTime.Parse("07/31/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("TSLA",DateTime.Parse("08/31/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("WOW",DateTime.Parse("02/26/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("CLDR",DateTime.Parse("10/31/2019"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("SPOT",DateTime.Parse("03/31/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("ACLS",DateTime.Parse("02/28/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("COCO",DateTime.Parse("04/28/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("DAC",DateTime.Parse("04/30/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("CDE",DateTime.Parse("06/28/2024"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("UEC",DateTime.Parse("09/29/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("ERJ",DateTime.Parse("05/28/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("SITM",DateTime.Parse("08/31/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("CAVA",DateTime.Parse("06/28/2024"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("APGE",DateTime.Parse("02/29/2024"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("CPRI",DateTime.Parse("01/29/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("VST",DateTime.Parse("06/28/2024"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("PDD",DateTime.Parse("05/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("NVCR",DateTime.Parse("10/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("VITL",DateTime.Parse("04/30/2024"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("SIG",DateTime.Parse("03/31/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("YPF",DateTime.Parse("11/30/2022"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("BLBD",DateTime.Parse("03/28/2024"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("BAND",DateTime.Parse("06/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("COOP",DateTime.Parse("09/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("SPT",DateTime.Parse("07/30/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("NTLA",DateTime.Parse("12/31/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("PTON",DateTime.Parse("09/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("VRT",DateTime.Parse("07/31/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("NET",DateTime.Parse("08/31/2021"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("NVDA",DateTime.Parse("04/28/2023"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("DOCU",DateTime.Parse("05/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
testCases.Add(new TestCase("SIG",DateTime.Parse("10/30/2020"),270,TestCase.CaseType.Training,TestCase.GenerateType.BollingerBand));
dataProcessor.SetOutputFolderPath(@"C:\boneyard\DeepLearning\ModelInputData\1");
dataProcessor.ProcessData(testCases);
}
public static void GenerateGeneralizedData()
{
TestCases testCases=new TestCases();
DataProcessor dataProcessor=new DataProcessor();
dataProcessor.Width=width;
dataProcessor.Height=height;
dataProcessor.PenWidth=1;
string[,] s=new string[,]{{"CRTX","07/30/2021"},{"SBSW","12/31/2019"},{"CWH","01/31/2020"},{"DISCA","02/26/2021"},{"DNLI","12/31/2020"},{"SNAP","12/29/2023"},{"FSM","05/31/2024"},{"TSLA","11/30/2021"},{"SWAV","05/31/2023"},{"PCTY","09/30/2021"},{"DCBO","08/31/2021"},{"LPG","11/30/2023"},{"KOD","11/30/2020"},{"NUE","05/28/2021"},{"XPO","08/31/2023"},{"EDU","11/30/2023"},{"BAK","04/30/2021"},{"INSW","11/30/2022"},{"ACLS","02/28/2023"},{"VST","03/28/2024"},{"TRMD","11/30/2022"},{"APGE","02/29/2024"},{"ANF","02/29/2024"},{"BAND","06/30/2020"},{"COIN","12/29/2023"},{"MTDR","03/31/2021"}};
for(int index=0;index<s.Length/2;index++)
{
String symbol=s[index,0];
DateTime purchaseDate=DateTime.Parse(s[index,1]);
TestCase testCase=new TestCase(symbol,purchaseDate,dayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBand);
testCases.Add(testCase);
}
dataProcessor.SetOutputFolderPath(@"C:\boneyard\DeepLearning\IndividualValidationCases");
dataProcessor.ProcessData(testCases);
}
public static String Predict()
{
return "";
}
public static void TestPredictAPI()
{
String strPathOutputFile = @"C:\boneyard\DeepLearning\TestPredictAPIOutput\outputcases_europa.csv";
CNNClient cnnClient = new CNNClient("http://10.0.0.73:5000");
if (!cnnClient.Ping())
{
Console.WriteLine("Model server is not reachable.");
return;
}
DataProcessor dataProcessor=new DataProcessor();
dataProcessor.Width=width;
dataProcessor.Height=height;
dataProcessor.PenWidth=1;
if(File.Exists(strPathOutputFile))
{
try{File.Delete(strPathOutputFile);}
catch(Exception exception)
{
Console.WriteLine(String.Format("Exception:{0}",exception.ToString()));
throw;
}
}
StreamWriter streamWriter=new StreamWriter(new FileStream(strPathOutputFile,FileMode.CreateNew));
string[,] s=new string[,]{{"CRTX","07/30/2021"},{"SBSW","12/31/2019"},{"CWH","01/31/2020"},{"DISCA","02/26/2021"},{"DNLI","12/31/2020"},{"SNAP","12/29/2023"},{"FSM","05/31/2024"},{"TSLA","11/30/2021"},{"SWAV","05/31/2023"},{"PCTY","09/30/2021"},{"DCBO","08/31/2021"},{"LPG","11/30/2023"},{"KOD","11/30/2020"},{"NUE","05/28/2021"},{"XPO","08/31/2023"},{"EDU","11/30/2023"},{"BAK","04/30/2021"},{"INSW","11/30/2022"},{"ACLS","02/28/2023"},{"VST","03/28/2024"},{"TRMD","11/30/2022"},{"APGE","02/29/2024"},{"ANF","02/29/2024"},{"BAND","06/30/2020"},{"COIN","12/29/2023"},{"MTDR","03/31/2021"}};
streamWriter.WriteLine("Symbol,Date,Response,Response,Raw Response");
MDTrace.WriteLine(LogLevel.DEBUG,"Symbol,Date,Response,Response,Raw Response");
for(int index=0;index<s.Length/2;index++)
{
StringBuilder sb=new StringBuilder();
String symbol=s[index,0];
DateTime purchaseDate=DateTime.Parse(s[index,1]);
TestCase testCase=new TestCase(symbol,purchaseDate,dayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBand,TestCase.OutputType.OutputStream);
dataProcessor.ProcessData(testCase);
// testCase.Responses.Add(cnnClient.Predict(CNNClient.Model.resnet50,testCase.LastStream));
testCase.Responses.Add(cnnClient.Predict(CNNClient.Model.resnet50_20241024_270,testCase.LastStream));
String strPredictionResponse=testCase.LastResponse;
String strPredictionResult=strPredictionResponse.Substring(strPredictionResponse.IndexOf("-->")+3);
int result=int.Parse(Utility.BetweenString(strPredictionResult,"[[","]"));
String strPredictionValue=strPredictionResponse.Substring(0,strPredictionResponse.IndexOf("-->"));
double value=double.Parse(Utility.BetweenString(strPredictionValue,"[[","]"));
sb.Append(symbol);
sb.Append(",");
sb.Append(Utility.AddQuotes(Utility.DateTimeToStringMMSDDSYYYY(purchaseDate)));
sb.Append(",");
sb.Append(Utility.AddQuotes(Utility.FormatNumber(value,8,false))).Append(",").Append(Utility.AddQuotes(result.ToString()));
sb.Append(",").Append(Utility.AddQuotes(strPredictionResponse));
MDTrace.WriteLine(LogLevel.DEBUG,sb.ToString());
streamWriter.WriteLine(sb.ToString());
}
streamWriter.Flush();
streamWriter.Close();
streamWriter.Dispose();
}
public static void divide(double divisions,double seedMin,double seedMax)
{
double skip=(seedMax-seedMin)/divisions;
List<double> array=new List<double>();
for(double seed=seedMax;seed>=seedMin;seed-=skip)
{
if(array.Count==divisions)break;
array.Add(seed);
}
for(int index=array.Count-1;index>=0;index--)
{
Console.Write(String.Format("{0}f,",Utility.FormatNumber(array[index],6)));
}
Console.WriteLine("");
}
}
}
Reference in New Issue View Git Blame Copy Permalink