using MarketData.Numerical;
using MarketData.DataAccess;
using MarketData.MarketDataModel;
using MarketData.Generator;
using MarketData.Utils;
using SkiaSharp;
namespace MarketData.CNNProcessing
{
public class DataProcessor
{
private String strFolderPath=default;
private readonly SKColor colorBlack = new SKColor(0,0,0);
private readonly SKColor colorWhite = new SKColor(255,255,255);
private readonly SKColor colorRed = new SKColor(255,0,0);
private readonly SKColor colorGreen = new SKColor(0,128,0); // This conforms to the way green is produced when generating CNN charts
public DataProcessor(int width=640,int height=480)
{
Width=width;
Height=height;
PenWidth=2f;
DrawingBrush = colorBlack;
DrawingBrushRed = colorRed;
DrawingBrushGreen = colorGreen;
FillBrush = colorWhite;
DrawPrice=true;
UseGrayScale=false;
NoiseColor=colorWhite;
strFolderPath=Directory.GetCurrentDirectory();
}
public bool Debug { get; set; } = false;
///
/// Width
///
///Gets/Sets the width of the output image
public int Width{get;set;}
///
/// Height
///
///Gets/Sets the height of the output image
public int Height{get;set;}
///
/// PenWidth
///
///Gets/Sets the width of the drawing pen
public float PenWidth{get;set;}
///
/// FillBrush
///
///Gets/Sets the background brush
public SKColor FillBrush { get; set;}
///
/// DrawingBrush
///
///Gets/Sets the drawing brush brush
public SKColor DrawingBrush{get;set;}
///
/// DrawingBrushRed
///
///Gets/Sets the drawing brush brush
public SKColor DrawingBrushRed{get;set;}
///
/// DrawingBrushGreen
///
///Gets/Sets the drawing brush brush
public SKColor DrawingBrushGreen{get;set;}
///
/// DrawBlack
///
///Sets up the canvas to do black drawing
public void DrawBlack()
{
DrawingBrush = colorBlack;
FillBrush = colorWhite;
}
///
/// IncludePrice
///
///Indicates whether we want to include price data in the graph
public bool DrawPrice{get; set;}
///
/// UseGrayScale
///
///Indicates whether we want to use grayscale.. default is B&W
public bool UseGrayScale{get; set;}
///
/// DrawWhite
///
///Sets up the canvas to do white drawing
public void DrawWhite()
{
DrawingBrush = colorBlack;
FillBrush = colorWhite;
}
///
/// MovingAverageArray
///
///Gets/Sets the moving average array
public int[] MovingAverageArray{get; set; }
///
/// PenWidthArray
///
///Gets/Sets thepen width array
public float[] PenWidthArray{get; set; }
///
/// NoiseArray
///
///Gets/Sets the noise array
public double[] NoiseArray{get; set; }
///
/// NoiseColor
///
///Gets/Sets the noise color
public SKColor NoiseColor{get; set; }
///
/// UseNoise
///
///The seed value
///The increment
///The size
public void UseNoise(double seed, double increment, int length, SKColor noiseColor)
{
NoiseColor=noiseColor;
NoiseArray=new double[length+1];
NoiseArray[0]=0.00;
for(int index=1;index
/// SetOutputFolderPath
///
///The test cases
public void SetOutputFolderPath(String strFolderPath)
{
this.strFolderPath=strFolderPath;
if(!this.strFolderPath.EndsWith(@"/"))this.strFolderPath=this.strFolderPath+@"/";
}
///
/// ClearFolderPath
///
///The test cases
public void ClearFolderPath()
{
if(String.IsNullOrEmpty(strFolderPath))throw new InvalidDataException($"{nameof(strFolderPath)} cannot be null");
if(!Directory.Exists(strFolderPath))
{
Directory.CreateDirectory(strFolderPath);
}
else
{
String[] pathFileNames = Directory.GetFiles(strFolderPath);
Console.WriteLine($"Deleting {pathFileNames.Length} files from {strFolderPath}");
foreach(String file in pathFileNames)
{
File.Delete(file);
}
}
}
public void ProcessData(TestCases testCases)
{
for(int index=0;index
/// ProcessData item
///
///The test cases
public void ProcessData(TestCase testCase,int index=0)
{
if(null==PenWidthArray)PenWidthArray=new float[]{PenWidth};
if(null==NoiseArray)NoiseArray=new double[]{0.00};
if(testCase.TypeGenerate.Equals(TestCase.GenerateType.BollingerBand))
{
testCase.PathFileNames.Add(CreateFileName(testCase.HomePath,testCase.Symbol,testCase.DayCount,0,0,0,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBand,testCase.PurchaseDate));
ProcessBollingerBandData(testCase,PenWidthArray[0],NoiseArray[0]);
}
else if(testCase.TypeGenerate.Equals(TestCase.GenerateType.BollingerBandWithVIX))
{
testCase.PathFileNames.Add(CreateFileName(testCase.HomePath,testCase.Symbol,testCase.DayCount,0,0,0,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBandWithVIX,testCase.PurchaseDate));
ProcessBollingerBandDataWithVolatility(testCase,PenWidthArray[0],NoiseArray[0]);
}
}
private String CreateFileName(String strFolderPath,String symbol,int dayCount,int index,int penIndex,int noiseIndex,TestCase.CaseType caseType,TestCase.GenerateType generateType,DateTime purchaseDate)
{
if(!strFolderPath.EndsWith("/"))strFolderPath+="/";
return String.Format("{0}{1}_{2}_{3}_{4}_{5}_{6}_{7}_{8}d.jpg",strFolderPath,symbol,index,penIndex,noiseIndex,caseType.ToString(),generateType.ToString(),Utility.DateToLong(purchaseDate),dayCount);
}
///
/// ProcessBollingerBandData item
///
///Symbol
private void ProcessBollingerBandData(TestCase testCase,float penWidth,double noise)
{
DateGenerator dateGenerator=new DateGenerator();
int daysInPeriod=dateGenerator.DaysBetweenActual(testCase.PurchaseDate,testCase.HistDate);
daysInPeriod+=60;
Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,daysInPeriod);
BollingerBands bollingerBands=BollingerBandGenerator.GenerateBollingerBands(prices); // we want to grab K, L, and Close
bollingerBands=new BollingerBands(bollingerBands.Where(x=>x.Date>=testCase.HistDate).ToList());
float[] k=new float[bollingerBands.Count];
float[] l=new float[bollingerBands.Count];
float[] close=new float[bollingerBands.Count];
// populate the arrays in reverse order so that we have the most historical date in the lowest index
for(int index=bollingerBands.Count-1;index>=0;index--)
{
BollingerBandElement bollingerBandElement=bollingerBands[index];
k[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.K)*1000.00f;
l[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.L)*1000.00f;
close[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.Close)*1000.00f;
}
Numerics.ZeroForNaNOrInfinity(ref k);
Numerics.ZeroForNaNOrInfinity(ref l);
Numerics.ZeroForNaNOrInfinity(ref close);
float maxY=Math.Max(Numerics.Max(ref l),Math.Max(Numerics.Max(ref close),Numerics.Max(ref k)));
float minY=Math.Min(Numerics.Min(ref l),Math.Min(Numerics.Min(ref close),Numerics.Min(ref k)))-5f;
float maxX=close.Length;
float minX=0.00f;
ImageHelper imageHelper=new ImageHelper();
PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY);
imageHelper.CreateImage(Width,Height,pointMapping);
imageHelper.Fill(FillBrush);
// draw price
if(DrawPrice)imageHelper.DrawSeries(DrawingBrushRed,penWidth,close);
// draw k
imageHelper.DrawSeries(DrawingBrushRed,penWidth,k);
// draw l
imageHelper.DrawSeries(DrawingBrushRed,penWidth,l);
if(0.00!=noise)imageHelper.AddNoise(NoiseColor,noise);
if(Debug || testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile))
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Writing {testCase.LastPathFileName}");
if(null==testCase.LastPathFileName)testCase.PathFileNames.Add($"{strFolderPath}{testCase.Symbol}.jpg");
if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName);
if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName);
else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName);
}
if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputStream))
{
testCase.Streams.Add(imageHelper.SaveBlackAndWhiteJPG());
}
}
///
/// ProcessBollingerBandDataWithVolatility item - Draws Price, K, L and Volatility
///
///Symbol
private void ProcessBollingerBandDataWithVolatility(TestCase testCase,float penWidth,double noise)
{
String symbolVolatility="^VIX";
DateGenerator dateGenerator=new DateGenerator();
int daysInPeriod=dateGenerator.DaysBetweenActual(testCase.PurchaseDate,testCase.HistDate);
daysInPeriod+=60;
Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,daysInPeriod);
Prices volatilityPrices=PricingDA.GetPrices(symbolVolatility,testCase.PurchaseDate,daysInPeriod);
BollingerBands bollingerBands=BollingerBandGenerator.GenerateBollingerBands(prices); // we want to grab K, L, and Close
bollingerBands=new BollingerBands(bollingerBands.Where(x=>x.Date>=testCase.HistDate).ToList());
float[] k=new float[bollingerBands.Count];
float[] l=new float[bollingerBands.Count];
float[] close=new float[bollingerBands.Count];
// Line up volatility dates with bollinger bands
DateTime minDate = bollingerBands.Min(x=>x.Date);
DateTime maxDate = bollingerBands.Max(x=>x.Date);
volatilityPrices = new Prices(volatilityPrices.Where(x=>x.Date<=maxDate && x.Date>=minDate).OrderBy(x=>x.Date).ToList()); // most historical date in lowest index
float[] v=volatilityPrices.GetPrices();
float minV=Numerics.Min(ref v); // get the minimum volatility value
double minP=bollingerBands.Min(x=>x.Close); // get minimum price
double factor=minP/minV; // determine scaling factor
for(int index=0;index=0;index--)
{
BollingerBandElement bollingerBandElement=bollingerBands[index];
k[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.K)*1000.00f; // put the data in log form
l[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.L)*1000.00f; // put the data in log form
close[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.Close)*1000.00f; // put the data in log form
}
Numerics.ZeroForNaNOrInfinity(ref k);
Numerics.ZeroForNaNOrInfinity(ref l);
Numerics.ZeroForNaNOrInfinity(ref close);
Numerics.ZeroForNaNOrInfinity(ref v);
float maxY=Math.Max(Math.Max(Numerics.Max(ref l),Math.Max(Numerics.Max(ref close),Numerics.Max(ref k))),Numerics.Max(ref v));
float minY=Math.Min(Math.Min(Numerics.Min(ref l),Math.Min(Numerics.Min(ref close),Numerics.Min(ref k))),Numerics.Min(ref v))-5f;
float maxX=close.Length;
float minX=0.00f;
ImageHelper imageHelper=new ImageHelper();
PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY);
imageHelper.CreateImage(Width,Height,pointMapping);
imageHelper.Fill(FillBrush);
// draw volatility
imageHelper.DrawSeries(DrawingBrushRed,penWidth,v);
// draw prices
if(DrawPrice)imageHelper.DrawSeries(DrawingBrush,penWidth,close);
// draw k
imageHelper.DrawSeries(DrawingBrushGreen,penWidth,k);
// draw l
imageHelper.DrawSeries(DrawingBrushGreen,penWidth,l);
if(0.00!=noise)imageHelper.AddNoise(NoiseColor,noise);
if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile))
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Writing {testCase.LastPathFileName}");
if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName);
if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName);
else imageHelper.Save(testCase.LastPathFileName);
}
else
{
testCase.Streams.Add(imageHelper.ToStream());
}
}
///
/// ProcessPriceData item
///
///TestCase
private void ProcessPriceData(TestCase testCase,float penWidth,double noise)
{
Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,testCase.HistDate);
prices.Reverse(); // get the most historical date into the lowest index
float[] priceArray=prices.GetPrices();
for(int index=0;index