From d0cabd5382b9a6291a469ae12258114c59aff563 Mon Sep 17 00:00:00 2001 From: Sean Date: Fri, 25 Oct 2024 17:37:05 -0400 Subject: [PATCH] relocate CNN image processing code into CNNImageProcessor project --- Program.cs | 203 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 142 insertions(+), 61 deletions(-) diff --git a/Program.cs b/Program.cs index b244324..237c8ba 100644 --- a/Program.cs +++ b/Program.cs @@ -610,76 +610,155 @@ namespace MarketData return sb.ToString(); } - public static void GenerateImageData() - { - GenerateImageData(@"C:\DeepLearningImageData\0",@"c:\3\0"); - GenerateImageData(@"C:\DeepLearningImageData\1",@"c:\3\1"); - } + // All of the CNN image processing related example code that used to be here has been relocated to the CNNImageProcessing project which is a console + // application that does all of that stuff + // ********************************************************************************************************** - public static void GenerateImageData(String inputFolder,String destinationFolder) +// This nest section is a study for accumulating stocks where there was a price increase >= 15% <= 25% within some period (aw) +// [p1,p2,p3,p4,p5......pn-2,pn-1,pn] +// ------- ow ----- ---- aw ----] ow+aw = total span +// ow = observation window. X. The timeseries leading up to the event +// aw = action window. The timeseries containing the event + public static void GetObservations(int observationWindow=90, int actionWindow=30) { -// String destinationFolder = @"c:\3\1"; - ImageHelper imageHelper = new ImageHelper(); - String[] files = Directory.GetFiles(inputFolder, "*.jpg"); - foreach (String file in files) + try { - try + DateTime minDate = DateTime.Parse("01/01/2017"); + DateGenerator dateGenerator = new DateGenerator(); + String pathFileName = "observations.txt"; + int eventWindow = observationWindow + actionWindow; + + if(File.Exists(pathFileName))File.Delete(pathFileName); + FileStream outStream = new FileStream(pathFileName, FileMode.Create); + StreamWriter streamWriter = new StreamWriter(outStream); + streamWriter.WriteLine($"OW={observationWindow},AW={actionWindow}"); + + List symbols = PricingDA.GetSymbols(); + for(int index=0;index x.Date).ToList()); // sort the prices so the oldest is in the lowest index + Prices actionPrices = new Prices(orderedPrices.Skip(observationWindow).ToList()); // get the aw window + Price p1 = actionPrices[0]; // get the earlier price from the lowest index + Price p2 = actionPrices[actionPrices.Count()-1]; // get the recent price from the highest index + double periodReturnPercentDecimal = (p2.Close-p1.Close)/p1.Close; // calculate the return over this period + if(periodReturnPercentDecimal<.15 || periodReturnPercentDecimal>.25)continue; + float[] priceArray = orderedPrices.GetPrices(); // priceArray is ordered earliest date in lowest index, most recent date in highest index + streamWriter.WriteLine($"SYMBOL={currentSymbol},DATE={orderedPrices[0].Date.ToShortDateString()},PRICES={String.Join(",",priceArray)}"); + } } - catch (Exception exception) - { - Console.WriteLine(exception.ToString()); - } + streamWriter.Flush(); + streamWriter.Close(); + streamWriter.Dispose(); + + CompressObservations(pathFileName); + } + catch(Exception exception) + { + Console.WriteLine(exception.ToString()); } } -// ******************************************************************************* -// ********************************************************************************************************** + public static void CompressObservations(String pathFileName) + { + DateGenerator dateGenerator = new DateGenerator(); + List itemsToRemove=new List(); + List observations = LoadObservations(pathFileName); + String prevSymbol = null; + DateTime prevEventDate = Utility.Epoch; + + if(0==observations.Count)return; + Console.WriteLine($"Compressing {pathFileName}"); + for(int index=observations.Count-1;index>=0;index--) + { + String strLine = observations[index]; + + if(strLine.StartsWith("OW"))break; + String symbol=Utility.BetweenString(strLine,"SYMBOL=",","); + DateTime eventDate = DateTime.Parse(Utility.BetweenString(strLine,"DATE=",",")); + + if(null==prevSymbol) + { + prevSymbol=symbol; + prevEventDate = eventDate; + } + else if(prevSymbol.Equals(symbol)) + { + DateTime nextBusinessDay = dateGenerator.FindNextBusinessDay(prevEventDate); + if(nextBusinessDay.Date.Equals(eventDate.Date)) + { + observations[index]=null; + } + prevEventDate = eventDate; + } + else + { + prevSymbol=symbol; + prevEventDate = eventDate; + } + } + observations = observations.Where(x => !String.IsNullOrEmpty(x)).ToList(); + String extension = Path.GetExtension(pathFileName); + pathFileName = pathFileName.Replace(extension, null); + Console.WriteLine($"Writing {pathFileName}"); + WriteObservations(pathFileName+".compressed.txt",observations); + Console.WriteLine("Done."); + } + + public static List LoadObservations(String pathFileName) + { + List lines = new List(); + if(!File.Exists(pathFileName))return lines; + String strLine; + + FileStream inStream = new FileStream(pathFileName, FileMode.Open); + StreamReader streamReader = new StreamReader(inStream); + while(null!=(strLine=streamReader.ReadLine())) + { + lines.Add(strLine); + } + streamReader.Close(); + streamReader.Dispose(); + inStream.Close(); + inStream.Dispose(); + return lines; + } + + public static void WriteObservations(String pathFileName,List observations) + { + if(File.Exists(pathFileName)) + { + File.Delete(pathFileName); + } + + FileStream outStream = new FileStream(pathFileName, FileMode.Create); + StreamWriter streamWriter = new StreamWriter(outStream); + foreach(String strLine in observations) + { + streamWriter.WriteLine(strLine); + } + streamWriter.Flush(); + streamWriter.Close(); + streamWriter.Dispose(); + outStream.Close(); + outStream.Dispose(); + } + + + static int Main(string[] args) { try @@ -690,7 +769,9 @@ namespace MarketData Trace.Listeners.Add(new TextWriterTraceListener(strLogFile)); DateTime currentDate=DateTime.Now; -// GenerateImageData(); +// GetObservations(90,30); +// CompressObservations("Observations.txt"); + DateTime maxHolidayDate =HolidayDA.GetMaxHolidayDate(); if(currentDate>maxHolidayDate)