444 lines
16 KiB
C#
444 lines
16 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using System.Threading.Tasks;
|
|
using System.Drawing;
|
|
using System.IO;
|
|
using MarketData.Numerical;
|
|
using MarketData.DataAccess;
|
|
using MarketData.MarketDataModel;
|
|
using MarketData.Generator;
|
|
using MarketData.Utils;
|
|
using System.Drawing.Drawing2D;
|
|
|
|
namespace MarketData.CNNProcessing
|
|
{
|
|
public class DataProcessor
|
|
{
|
|
private String strFolderPath=@"c:\1\";
|
|
public DataProcessor(int width=640,int height=480)
|
|
{
|
|
Width=width;
|
|
Height=height;
|
|
PenWidth=2f;
|
|
DrawingBrush=new SolidBrush(Color.Black);
|
|
FillBrush=new SolidBrush(Color.White);
|
|
DrawPrice=true;
|
|
UseGrayScale=false;
|
|
NoiseColor=Color.White;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Width
|
|
/// </summary>
|
|
///<param name="value">Gets/Sets the width of the output image</param>
|
|
public int Width{get;set;}
|
|
|
|
/// <summary>
|
|
/// Height
|
|
/// </summary>
|
|
///<param name="value">Gets/Sets the height of the output image</param>
|
|
public int Height{get;set;}
|
|
|
|
/// <summary>
|
|
/// PenWidth
|
|
/// </summary>
|
|
///<param name="value">Gets/Sets the width of the drawing pen</param>
|
|
public float PenWidth{get;set;}
|
|
|
|
/// <summary>
|
|
/// FillBrush
|
|
/// </summary>
|
|
///<param name="value">Gets/Sets the background brush</param>
|
|
public Brush FillBrush{get;set;}
|
|
|
|
/// <summary>
|
|
/// DrawingBrush
|
|
/// </summary>
|
|
///<param name="value">Gets/Sets the drawing brush brush</param>
|
|
public Brush DrawingBrush{get;set;}
|
|
|
|
/// <summary>
|
|
/// DrawBlack
|
|
/// </summary>
|
|
///<param name="value">Sets up the canvas to do black drawing</param>
|
|
public void DrawBlack()
|
|
{
|
|
DrawingBrush=new SolidBrush(Color.Black);
|
|
FillBrush=new SolidBrush(Color.White);
|
|
}
|
|
|
|
/// <summary>
|
|
/// IncludePrice
|
|
/// </summary>
|
|
///<param name="value">Indicates whether we want to include price data in the graph</param>
|
|
public bool DrawPrice{get; set;}
|
|
|
|
/// <summary>
|
|
/// UseGrayScale
|
|
/// </summary>
|
|
///<param name="value">Indicates whether we want to use grayscale.. default is B&W</param>
|
|
public bool UseGrayScale{get; set;}
|
|
|
|
/// <summary>
|
|
/// DrawWhite
|
|
/// </summary>
|
|
///<param name="value">Sets up the canvas to do white drawing</param>
|
|
public void DrawWhite()
|
|
{
|
|
DrawingBrush=new SolidBrush(Color.White);
|
|
FillBrush=new SolidBrush(Color.Black);
|
|
}
|
|
|
|
/// <summary>
|
|
/// MovingAverageArray
|
|
/// </summary>
|
|
///<param name="MovingAverageArray">Gets/Sets the moving average array</param>
|
|
public int[] MovingAverageArray{get; set; }
|
|
|
|
/// <summary>
|
|
/// PenWidthArray
|
|
/// </summary>
|
|
///<param name="PenWidthArray">Gets/Sets thepen width array</param>
|
|
public float[] PenWidthArray{get; set; }
|
|
|
|
/// <summary>
|
|
/// NoiseArray
|
|
/// </summary>
|
|
///<param name="NoiseArray">Gets/Sets the noise array</param>
|
|
public double[] NoiseArray{get; set; }
|
|
|
|
/// <summary>
|
|
/// NoiseColor
|
|
/// </summary>
|
|
///<param name="NoiseColor">Gets/Sets the noise color</param>
|
|
public Color NoiseColor{get; set; }
|
|
|
|
/// <summary>
|
|
/// UseNoise
|
|
/// </summary>
|
|
///<param name="seed">The seed value</param>
|
|
///<param name="increment">The increment</param>
|
|
///<param name="length">The size</param>
|
|
public void UseNoise(double seed, double increment, int length, Color noiseColor)
|
|
{
|
|
NoiseColor=noiseColor;
|
|
NoiseArray=new double[length+1];
|
|
NoiseArray[0]=0.00;
|
|
for(int index=1;index<length;index++)
|
|
{
|
|
NoiseArray[index]=seed;
|
|
seed+=increment;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// SetOutputFolderPath
|
|
/// </summary>
|
|
///<param name="testCases">The test cases</param>
|
|
public void SetOutputFolderPath(String strFolderPath)
|
|
{
|
|
this.strFolderPath=strFolderPath;
|
|
if(!this.strFolderPath.EndsWith(@"\"))this.strFolderPath=this.strFolderPath+@"\";
|
|
}
|
|
public void ProcessData(TestCases testCases)
|
|
{
|
|
for(int index=0;index<testCases.Count;index++)
|
|
{
|
|
TestCase testCase=testCases[index];
|
|
ProcessData(testCase,index);
|
|
}
|
|
}
|
|
/// <summary>
|
|
/// ProcessData item
|
|
/// </summary>
|
|
///<param name="testCases">The test cases</param>
|
|
public void ProcessData(TestCase testCase,int index=0)
|
|
{
|
|
if(null==PenWidthArray)PenWidthArray=new float[]{PenWidth};
|
|
if(null==NoiseArray)NoiseArray=new double[]{0.00};
|
|
if(testCase.TypeGenerate.Equals(TestCase.GenerateType.Price))
|
|
{
|
|
for(int penIndex=0;penIndex<PenWidthArray.Length;penIndex++)
|
|
{
|
|
float penWidth=PenWidthArray[penIndex];
|
|
for(int noiseIndex=0;noiseIndex<NoiseArray.Length;noiseIndex++)
|
|
{
|
|
double noise=NoiseArray[noiseIndex];
|
|
// String strPathFileName=String.Format("{0}{1}_{2}_{3}_{4}_{5}_{6}_{7}.jpg",strFolderPath,testCase.Symbol,index,penIndex,noiseIndex,testCase.TypeCase.ToString(),testCase.TypeGenerate.ToString(),Utility.DateToLong(testCase.PurchaseDate));
|
|
String strPathFileName=CreateFileName(strFolderPath,testCase.Symbol,testCase.DayCount,index,penIndex,noiseIndex,testCase.TypeCase,testCase.TypeGenerate,testCase.PurchaseDate);
|
|
testCase.PathFileNames.Add(strPathFileName);
|
|
ProcessPriceData(testCase,penWidth,noise);
|
|
}
|
|
}
|
|
}
|
|
else // Bollinger bands
|
|
{
|
|
if(null==MovingAverageArray)
|
|
{
|
|
for(int penIndex=0;penIndex<PenWidthArray.Length;penIndex++)
|
|
{
|
|
float penWidth=PenWidthArray[penIndex];
|
|
for(int noiseIndex=0;noiseIndex<NoiseArray.Length;noiseIndex++)
|
|
{
|
|
double noise=NoiseArray[noiseIndex];
|
|
// String strPathFileName=String.Format("{0}{1}_{2}_{3}_{4}_{5}_{6}_{7}.jpg",strFolderPath,testCase.Symbol,index,penIndex,noiseIndex,testCase.TypeCase.ToString(),testCase.TypeGenerate.ToString(),Utility.DateToLong(testCase.PurchaseDate));
|
|
String strPathFileName=CreateFileName(strFolderPath,testCase.Symbol,testCase.DayCount,index,penIndex,noiseIndex,testCase.TypeCase,testCase.TypeGenerate,testCase.PurchaseDate);
|
|
testCase.PathFileNames.Add(strPathFileName);
|
|
ProcessBollingerBandData(testCase,penWidth,noise);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for(int avgIndex=0;avgIndex<MovingAverageArray.Length;avgIndex++)
|
|
{
|
|
int movingAverage=MovingAverageArray[avgIndex];
|
|
|
|
for(int penIndex=0;penIndex<PenWidthArray.Length;penIndex++)
|
|
{
|
|
float penWidth=PenWidthArray[penIndex];
|
|
for(int noiseIndex=0;noiseIndex<NoiseArray.Length;noiseIndex++)
|
|
{
|
|
double noise=NoiseArray[noiseIndex];
|
|
// String strPathFileName=String.Format("{0}{1}_{2}_{3}_{4}_{5}_{6}_{7}.jpg",strFolderPath,testCase.Symbol,index,penIndex,noiseIndex,testCase.TypeCase.ToString(),testCase.TypeGenerate.ToString(),Utility.DateToLong(testCase.PurchaseDate));
|
|
String strPathFileName=CreateFileName(strFolderPath,testCase.Symbol,testCase.DayCount,index,penIndex,noiseIndex,testCase.TypeCase,testCase.TypeGenerate,testCase.PurchaseDate);
|
|
testCase.PathFileNames.Add(strPathFileName);
|
|
ProcessBollingerBandData(testCase,movingAverage,penWidth,noise);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private String CreateFileName(String strFolderPath,String symbol,int dayCount,int index,int penIndex,int noiseIndex,TestCase.CaseType caseType,TestCase.GenerateType generateType,DateTime purchaseDate)
|
|
{
|
|
return String.Format("{0}{1}_{2}_{3}_{4}_{5}_{6}_{7}_{8}d.jpg",strFolderPath,symbol,index,penIndex,noiseIndex,caseType.ToString(),generateType.ToString(),Utility.DateToLong(purchaseDate),dayCount);
|
|
}
|
|
|
|
private void ProcessBollingerBandData(TestCase testCase,int movingAverageDays,float penWidth,double noise)
|
|
{
|
|
int bufferDays=60;
|
|
DateGenerator dateGenerator=new DateGenerator();
|
|
|
|
int daysInPeriod=dateGenerator.DaysBetweenActual(testCase.PurchaseDate,testCase.HistDate);
|
|
if(movingAverageDays>bufferDays)daysInPeriod=movingAverageDays*2;
|
|
else daysInPeriod+=60;
|
|
Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,daysInPeriod);
|
|
BollingerBands bollingerBands=BollingerBandGenerator.GenerateBollingerBands(prices); // we want to grab K, L, and Close
|
|
bollingerBands=new BollingerBands(bollingerBands.Where(x=>x.Date>=testCase.HistDate).ToList());
|
|
float[] k=new float[bollingerBands.Count];
|
|
float[] l=new float[bollingerBands.Count];
|
|
float[] close=new float[bollingerBands.Count];
|
|
|
|
DMAPrices dmaPrices=MovingAverageGenerator.GenerateMovingAverage(prices, movingAverageDays);
|
|
DMAPricesByDate dmaPricesByDate=dmaPrices.GetDMAPricesByDate();
|
|
|
|
// populate the arrays in reverse order so that we have the most historical date in the lowest index
|
|
for(int index=bollingerBands.Count-1;index>=0;index--)
|
|
{
|
|
BollingerBandElement bollingerBandElement=bollingerBands[index];
|
|
if(!dmaPricesByDate.ContainsKey(bollingerBandElement.Date))continue;
|
|
k[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.K)*1000.00f;
|
|
l[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.L)*1000.00f;
|
|
double movingAverageClose=dmaPricesByDate[bollingerBandElement.Date].AVGPrice;
|
|
close[bollingerBands.Count-index-1]=(float)Math.Log(movingAverageClose)*1000.00f;
|
|
}
|
|
Numerics.ZeroForNaNOrInfinity(ref k);
|
|
Numerics.ZeroForNaNOrInfinity(ref l);
|
|
Numerics.ZeroForNaNOrInfinity(ref close);
|
|
float maxY=Math.Max(Numerics.Max(ref l),Math.Max(Numerics.Max(ref close),Numerics.Max(ref k)));
|
|
float minY=Math.Min(Numerics.Min(ref l),Math.Min(Numerics.Min(ref close),Numerics.Min(ref k)))-5f;
|
|
float maxX=close.Length;
|
|
float minX=0.00f;
|
|
|
|
Pen pen=new Pen(DrawingBrush,penWidth);
|
|
ImageHelper imageHelper=new ImageHelper();
|
|
|
|
PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY);
|
|
imageHelper.CreateImage(Width,Height,pointMapping);
|
|
imageHelper.Fill(FillBrush);
|
|
|
|
LineSegments lineSegments=new LineSegments();
|
|
for(int index=0;index<close.Length && DrawPrice;index++)
|
|
{
|
|
if(0==index)continue;
|
|
Point p1=new Point(index-1,(int)close[index-1]);
|
|
Point p2=new Point(index,(int)close[index]);
|
|
lineSegments.Add(p1,p2);
|
|
}
|
|
imageHelper.DrawPath(pen,lineSegments);
|
|
|
|
lineSegments.Clear();
|
|
for(int index=0;index<k.Length;index++)
|
|
{
|
|
if(0==index)continue;
|
|
Point p1=new Point(index-1,(int)k[index-1]);
|
|
Point p2=new Point(index,(int)k[index]);
|
|
lineSegments.Add(p1,p2);
|
|
}
|
|
imageHelper.DrawPath(pen,lineSegments);
|
|
|
|
lineSegments.Clear();
|
|
for(int index=0;index<l.Length;index++)
|
|
{
|
|
if(0==index)continue;
|
|
Point p1=new Point(index-1,(int)l[index-1]);
|
|
Point p2=new Point(index,(int)l[index]);
|
|
lineSegments.Add(p1,p2);
|
|
}
|
|
imageHelper.DrawPath(pen,lineSegments);
|
|
|
|
if(0.00!=noise)imageHelper.AddNoise(NoiseColor,noise);
|
|
|
|
if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile))
|
|
{
|
|
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Writing {0}",testCase.ToString()));
|
|
if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName);
|
|
if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName);
|
|
else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName);
|
|
}
|
|
else
|
|
{
|
|
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Streaming {0}",testCase.ToString()));
|
|
testCase.Streams.Add(imageHelper.SaveBlackAndWhiteJPG());
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// ProcessBollingerBandData item
|
|
/// </summary>
|
|
///<param name="testCase">Symbol</param>
|
|
private void ProcessBollingerBandData(TestCase testCase,float penWidth,double noise)
|
|
{
|
|
DateGenerator dateGenerator=new DateGenerator();
|
|
|
|
int daysInPeriod=dateGenerator.DaysBetweenActual(testCase.PurchaseDate,testCase.HistDate);
|
|
daysInPeriod+=60;
|
|
Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,daysInPeriod);
|
|
BollingerBands bollingerBands=BollingerBandGenerator.GenerateBollingerBands(prices); // we want to grab K, L, and Close
|
|
bollingerBands=new BollingerBands(bollingerBands.Where(x=>x.Date>=testCase.HistDate).ToList());
|
|
float[] k=new float[bollingerBands.Count];
|
|
float[] l=new float[bollingerBands.Count];
|
|
float[] close=new float[bollingerBands.Count];
|
|
|
|
// populate the arrays in reverse order so that we have the most historical date in the lowest index
|
|
for(int index=bollingerBands.Count-1;index>=0;index--)
|
|
{
|
|
BollingerBandElement bollingerBandElement=bollingerBands[index];
|
|
k[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.K)*1000.00f;
|
|
l[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.L)*1000.00f;
|
|
close[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.Close)*1000.00f;
|
|
}
|
|
Numerics.ZeroForNaNOrInfinity(ref k);
|
|
Numerics.ZeroForNaNOrInfinity(ref l);
|
|
Numerics.ZeroForNaNOrInfinity(ref close);
|
|
float maxY=Math.Max(Numerics.Max(ref l),Math.Max(Numerics.Max(ref close),Numerics.Max(ref k)));
|
|
float minY=Math.Min(Numerics.Min(ref l),Math.Min(Numerics.Min(ref close),Numerics.Min(ref k)))-5f;
|
|
float maxX=close.Length;
|
|
float minX=0.00f;
|
|
|
|
Pen pen=new Pen(DrawingBrush,penWidth);
|
|
ImageHelper imageHelper=new ImageHelper();
|
|
|
|
PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY);
|
|
imageHelper.CreateImage(Width,Height,pointMapping);
|
|
imageHelper.Fill(FillBrush);
|
|
|
|
LineSegments lineSegments=new LineSegments();
|
|
for(int index=0;index<close.Length && DrawPrice;index++)
|
|
{
|
|
if(0==index)continue;
|
|
Point p1=new Point(index-1,(int)close[index-1]);
|
|
Point p2=new Point(index,(int)close[index]);
|
|
lineSegments.Add(p1,p2);
|
|
}
|
|
imageHelper.DrawPath(pen,lineSegments);
|
|
|
|
lineSegments.Clear();
|
|
for(int index=0;index<k.Length;index++)
|
|
{
|
|
if(0==index)continue;
|
|
Point p1=new Point(index-1,(int)k[index-1]);
|
|
Point p2=new Point(index,(int)k[index]);
|
|
lineSegments.Add(p1,p2);
|
|
}
|
|
imageHelper.DrawPath(pen,lineSegments);
|
|
|
|
lineSegments.Clear();
|
|
for(int index=0;index<l.Length;index++)
|
|
{
|
|
if(0==index)continue;
|
|
Point p1=new Point(index-1,(int)l[index-1]);
|
|
Point p2=new Point(index,(int)l[index]);
|
|
lineSegments.Add(p1,p2);
|
|
}
|
|
imageHelper.DrawPath(pen,lineSegments);
|
|
|
|
if(0.00!=noise)imageHelper.AddNoise(NoiseColor,noise);
|
|
|
|
if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile))
|
|
{
|
|
if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName);
|
|
if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName);
|
|
else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName);
|
|
}
|
|
else
|
|
{
|
|
testCase.Streams.Add(imageHelper.SaveBlackAndWhiteJPG());
|
|
}
|
|
}
|
|
/// <summary>
|
|
/// ProcessPriceData item
|
|
/// </summary>
|
|
///<param name="testCase">TestCase</param>
|
|
private void ProcessPriceData(TestCase testCase,float penWidth,double noise)
|
|
{
|
|
Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,testCase.HistDate);
|
|
prices.Reverse(); // get the most historical date into the lowest index
|
|
float[] priceArray=prices.GetPrices();
|
|
for(int index=0;index<priceArray.Length;index++)
|
|
{
|
|
priceArray[index]=(float)Math.Log((double)priceArray[index])*1000.00f;
|
|
}
|
|
Numerics.ZeroForNaNOrInfinity(ref priceArray);
|
|
|
|
float maxY=Numerics.Max(ref priceArray);
|
|
float minY=Numerics.Min(ref priceArray)-5f;
|
|
float maxX=priceArray.Length;
|
|
float minX=0.00f;
|
|
|
|
Pen pen=new Pen(DrawingBrush,penWidth);
|
|
ImageHelper imageHelper=new ImageHelper();
|
|
|
|
PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY);
|
|
imageHelper.CreateImage(Width,Height,pointMapping);
|
|
imageHelper.Fill(FillBrush);
|
|
|
|
LineSegments lineSegments=new LineSegments();
|
|
for(int index=0;index<priceArray.Length;index++)
|
|
{
|
|
if(0==index)continue;
|
|
Point p1=new Point(index-1,(int)priceArray[index-1]);
|
|
Point p2=new Point(index,(int)priceArray[index]);
|
|
lineSegments.Add(p1,p2);
|
|
}
|
|
imageHelper.DrawPath(pen,lineSegments);
|
|
if(0.00!=noise)imageHelper.AddNoise(NoiseColor,noise);
|
|
|
|
if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile))
|
|
{
|
|
if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName);
|
|
if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName);
|
|
else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName);
|
|
}
|
|
else
|
|
{
|
|
testCase.Streams.Add(imageHelper.SaveBlackAndWhiteJPG());
|
|
}
|
|
}
|
|
}
|
|
}
|