relocate CNN image processing code into CNNImageProcessor project
This commit is contained in:
203
Program.cs
203
Program.cs
@@ -610,76 +610,155 @@ namespace MarketData
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
public static void GenerateImageData()
|
||||
{
|
||||
GenerateImageData(@"C:\DeepLearningImageData\0",@"c:\3\0");
|
||||
GenerateImageData(@"C:\DeepLearningImageData\1",@"c:\3\1");
|
||||
}
|
||||
// All of the CNN image processing related example code that used to be here has been relocated to the CNNImageProcessing project which is a console
|
||||
// application that does all of that stuff
|
||||
// **********************************************************************************************************
|
||||
|
||||
public static void GenerateImageData(String inputFolder,String destinationFolder)
|
||||
// This nest section is a study for accumulating stocks where there was a price increase >= 15% <= 25% within some period (aw)
|
||||
// [p1,p2,p3,p4,p5......pn-2,pn-1,pn]
|
||||
// ------- ow ----- ---- aw ----] ow+aw = total span
|
||||
// ow = observation window. X. The timeseries leading up to the event
|
||||
// aw = action window. The timeseries containing the event
|
||||
public static void GetObservations(int observationWindow=90, int actionWindow=30)
|
||||
{
|
||||
// String destinationFolder = @"c:\3\1";
|
||||
ImageHelper imageHelper = new ImageHelper();
|
||||
String[] files = Directory.GetFiles(inputFolder, "*.jpg");
|
||||
foreach (String file in files)
|
||||
try
|
||||
{
|
||||
try
|
||||
DateTime minDate = DateTime.Parse("01/01/2017");
|
||||
DateGenerator dateGenerator = new DateGenerator();
|
||||
String pathFileName = "observations.txt";
|
||||
int eventWindow = observationWindow + actionWindow;
|
||||
|
||||
if(File.Exists(pathFileName))File.Delete(pathFileName);
|
||||
FileStream outStream = new FileStream(pathFileName, FileMode.Create);
|
||||
StreamWriter streamWriter = new StreamWriter(outStream);
|
||||
streamWriter.WriteLine($"OW={observationWindow},AW={actionWindow}");
|
||||
|
||||
List<String> symbols = PricingDA.GetSymbols();
|
||||
for(int index=0;index<symbols.Count();index++)
|
||||
{
|
||||
String fileName = Path.GetFileName(file);
|
||||
String pathFileName = destinationFolder + @"\" + fileName;
|
||||
String pathFileNameL = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"L.jpg";
|
||||
String pathFileNameR = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"R.jpg";
|
||||
String pathFileNameU = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"U.jpg";
|
||||
String pathFileNameB1 = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"B1.jpg";
|
||||
String pathFileNameB2 = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"B2.jpg";
|
||||
String pathFileNameB3 = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"B3.jpg";
|
||||
String pathFileNameB4 = destinationFolder + @"\" + Utility.BetweenString(fileName,null,".")+"B4.jpg";
|
||||
String currentSymbol = symbols[index];
|
||||
|
||||
Console.WriteLine(String.Format("Reading {0}", file));
|
||||
imageHelper.LoadImage(file);
|
||||
imageHelper.Resize(128, 128);
|
||||
|
||||
ImageHelper bmpLeft=new ImageHelper(imageHelper);
|
||||
ImageHelper bmpRight=new ImageHelper(imageHelper);
|
||||
ImageHelper bmpUDown=null;
|
||||
bmpLeft.RotateLeft();
|
||||
bmpRight.RotateRight();
|
||||
bmpUDown=new ImageHelper(bmpRight);
|
||||
bmpUDown.RotateRight();
|
||||
|
||||
bmpLeft.ToGrayScale();
|
||||
bmpRight.ToGrayScale();
|
||||
bmpUDown.ToGrayScale();
|
||||
imageHelper.ToGrayScale();
|
||||
|
||||
imageHelper.Save(pathFileName);
|
||||
bmpLeft.Save(pathFileNameL);
|
||||
bmpRight.Save(pathFileNameR);
|
||||
bmpUDown.Save(pathFileNameU);
|
||||
|
||||
ImageHelper bmpBlur1=new ImageHelper(imageHelper);
|
||||
ImageHelper bmpBlur2=new ImageHelper(bmpLeft);
|
||||
ImageHelper bmpBlur3=new ImageHelper(bmpRight);
|
||||
ImageHelper bmpBlur4=new ImageHelper(bmpUDown);
|
||||
bmpBlur1.Blur(3);
|
||||
bmpBlur2.Blur(3);
|
||||
bmpBlur3.Blur(3);
|
||||
bmpBlur4.Blur(3);
|
||||
bmpBlur1.Save(pathFileNameB1);
|
||||
bmpBlur2.Save(pathFileNameB2);
|
||||
bmpBlur3.Save(pathFileNameB3);
|
||||
bmpBlur4.Save(pathFileNameB4);
|
||||
CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(currentSymbol);
|
||||
if(null == companyProfile || !companyProfile.IsEquity)continue;
|
||||
|
||||
Console.WriteLine($"Working on {currentSymbol}");
|
||||
DateTime latestDate = PricingDA.GetLatestDate(currentSymbol);
|
||||
if(Utility.IsEpoch(latestDate) || latestDate<minDate)continue;
|
||||
while(true)
|
||||
{
|
||||
if(latestDate<minDate)break;
|
||||
Prices prices = GBPriceCache.GetInstance().GetPrices(currentSymbol,latestDate,eventWindow);
|
||||
if(null == prices || prices.Count < eventWindow || prices.Min()<=0.00)break;
|
||||
latestDate = dateGenerator.FindPrevBusinessDay(latestDate);
|
||||
Prices orderedPrices = new Prices(prices.OrderBy(x => x.Date).ToList()); // sort the prices so the oldest is in the lowest index
|
||||
Prices actionPrices = new Prices(orderedPrices.Skip(observationWindow).ToList()); // get the aw window
|
||||
Price p1 = actionPrices[0]; // get the earlier price from the lowest index
|
||||
Price p2 = actionPrices[actionPrices.Count()-1]; // get the recent price from the highest index
|
||||
double periodReturnPercentDecimal = (p2.Close-p1.Close)/p1.Close; // calculate the return over this period
|
||||
if(periodReturnPercentDecimal<.15 || periodReturnPercentDecimal>.25)continue;
|
||||
float[] priceArray = orderedPrices.GetPrices(); // priceArray is ordered earliest date in lowest index, most recent date in highest index
|
||||
streamWriter.WriteLine($"SYMBOL={currentSymbol},DATE={orderedPrices[0].Date.ToShortDateString()},PRICES={String.Join(",",priceArray)}");
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
Console.WriteLine(exception.ToString());
|
||||
}
|
||||
streamWriter.Flush();
|
||||
streamWriter.Close();
|
||||
streamWriter.Dispose();
|
||||
|
||||
CompressObservations(pathFileName);
|
||||
}
|
||||
catch(Exception exception)
|
||||
{
|
||||
Console.WriteLine(exception.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
// *******************************************************************************
|
||||
// **********************************************************************************************************
|
||||
public static void CompressObservations(String pathFileName)
|
||||
{
|
||||
DateGenerator dateGenerator = new DateGenerator();
|
||||
List<String> itemsToRemove=new List<String>();
|
||||
List<String> observations = LoadObservations(pathFileName);
|
||||
String prevSymbol = null;
|
||||
DateTime prevEventDate = Utility.Epoch;
|
||||
|
||||
if(0==observations.Count)return;
|
||||
Console.WriteLine($"Compressing {pathFileName}");
|
||||
for(int index=observations.Count-1;index>=0;index--)
|
||||
{
|
||||
String strLine = observations[index];
|
||||
|
||||
if(strLine.StartsWith("OW"))break;
|
||||
String symbol=Utility.BetweenString(strLine,"SYMBOL=",",");
|
||||
DateTime eventDate = DateTime.Parse(Utility.BetweenString(strLine,"DATE=",","));
|
||||
|
||||
if(null==prevSymbol)
|
||||
{
|
||||
prevSymbol=symbol;
|
||||
prevEventDate = eventDate;
|
||||
}
|
||||
else if(prevSymbol.Equals(symbol))
|
||||
{
|
||||
DateTime nextBusinessDay = dateGenerator.FindNextBusinessDay(prevEventDate);
|
||||
if(nextBusinessDay.Date.Equals(eventDate.Date))
|
||||
{
|
||||
observations[index]=null;
|
||||
}
|
||||
prevEventDate = eventDate;
|
||||
}
|
||||
else
|
||||
{
|
||||
prevSymbol=symbol;
|
||||
prevEventDate = eventDate;
|
||||
}
|
||||
}
|
||||
observations = observations.Where(x => !String.IsNullOrEmpty(x)).ToList();
|
||||
String extension = Path.GetExtension(pathFileName);
|
||||
pathFileName = pathFileName.Replace(extension, null);
|
||||
Console.WriteLine($"Writing {pathFileName}");
|
||||
WriteObservations(pathFileName+".compressed.txt",observations);
|
||||
Console.WriteLine("Done.");
|
||||
}
|
||||
|
||||
public static List<String> LoadObservations(String pathFileName)
|
||||
{
|
||||
List<String> lines = new List<string>();
|
||||
if(!File.Exists(pathFileName))return lines;
|
||||
String strLine;
|
||||
|
||||
FileStream inStream = new FileStream(pathFileName, FileMode.Open);
|
||||
StreamReader streamReader = new StreamReader(inStream);
|
||||
while(null!=(strLine=streamReader.ReadLine()))
|
||||
{
|
||||
lines.Add(strLine);
|
||||
}
|
||||
streamReader.Close();
|
||||
streamReader.Dispose();
|
||||
inStream.Close();
|
||||
inStream.Dispose();
|
||||
return lines;
|
||||
}
|
||||
|
||||
public static void WriteObservations(String pathFileName,List<String> observations)
|
||||
{
|
||||
if(File.Exists(pathFileName))
|
||||
{
|
||||
File.Delete(pathFileName);
|
||||
}
|
||||
|
||||
FileStream outStream = new FileStream(pathFileName, FileMode.Create);
|
||||
StreamWriter streamWriter = new StreamWriter(outStream);
|
||||
foreach(String strLine in observations)
|
||||
{
|
||||
streamWriter.WriteLine(strLine);
|
||||
}
|
||||
streamWriter.Flush();
|
||||
streamWriter.Close();
|
||||
streamWriter.Dispose();
|
||||
outStream.Close();
|
||||
outStream.Dispose();
|
||||
}
|
||||
|
||||
|
||||
|
||||
static int Main(string[] args)
|
||||
{
|
||||
try
|
||||
@@ -690,7 +769,9 @@ namespace MarketData
|
||||
Trace.Listeners.Add(new TextWriterTraceListener(strLogFile));
|
||||
DateTime currentDate=DateTime.Now;
|
||||
|
||||
// GenerateImageData();
|
||||
// GetObservations(90,30);
|
||||
// CompressObservations("Observations.txt");
|
||||
|
||||
|
||||
DateTime maxHolidayDate =HolidayDA.GetMaxHolidayDate();
|
||||
if(currentDate>maxHolidayDate)
|
||||
|
||||
Reference in New Issue
Block a user