using MarketData.Numerical; using MarketData.DataAccess; using MarketData.MarketDataModel; using MarketData.Generator; using MarketData.Utils; using SkiaSharp; namespace MarketData.CNNProcessing { public class DataProcessor { private String strFolderPath=default; private readonly SKColor colorBlack = new SKColor(0,0,0); private readonly SKColor colorWhite = new SKColor(255,255,255); private readonly SKColor colorRed = new SKColor(255,0,0); private readonly SKColor colorGreen = new SKColor(0,255,0); public DataProcessor(int width=640,int height=480) { Width=width; Height=height; PenWidth=2f; DrawingBrush = colorBlack; DrawingBrushRed = colorRed; DrawingBrushGreen = colorGreen; FillBrush = colorWhite; DrawPrice=true; UseGrayScale=false; NoiseColor=colorWhite; strFolderPath=Directory.GetCurrentDirectory(); } public bool Debug { get; set; } = false; /// /// Width /// ///Gets/Sets the width of the output image public int Width{get;set;} /// /// Height /// ///Gets/Sets the height of the output image public int Height{get;set;} /// /// PenWidth /// ///Gets/Sets the width of the drawing pen public float PenWidth{get;set;} /// /// FillBrush /// ///Gets/Sets the background brush public SKColor FillBrush { get; set;} /// /// DrawingBrush /// ///Gets/Sets the drawing brush brush public SKColor DrawingBrush{get;set;} /// /// DrawingBrushRed /// ///Gets/Sets the drawing brush brush public SKColor DrawingBrushRed{get;set;} /// /// DrawingBrushGreen /// ///Gets/Sets the drawing brush brush public SKColor DrawingBrushGreen{get;set;} /// /// DrawBlack /// ///Sets up the canvas to do black drawing public void DrawBlack() { DrawingBrush = colorBlack; FillBrush = colorWhite; } /// /// IncludePrice /// ///Indicates whether we want to include price data in the graph public bool DrawPrice{get; set;} /// /// UseGrayScale /// ///Indicates whether we want to use grayscale.. default is B&W public bool UseGrayScale{get; set;} /// /// DrawWhite /// ///Sets up the canvas to do white drawing public void DrawWhite() { DrawingBrush = colorBlack; FillBrush = colorWhite; } /// /// MovingAverageArray /// ///Gets/Sets the moving average array public int[] MovingAverageArray{get; set; } /// /// PenWidthArray /// ///Gets/Sets thepen width array public float[] PenWidthArray{get; set; } /// /// NoiseArray /// ///Gets/Sets the noise array public double[] NoiseArray{get; set; } /// /// NoiseColor /// ///Gets/Sets the noise color public SKColor NoiseColor{get; set; } /// /// UseNoise /// ///The seed value ///The increment ///The size public void UseNoise(double seed, double increment, int length, SKColor noiseColor) { NoiseColor=noiseColor; NoiseArray=new double[length+1]; NoiseArray[0]=0.00; for(int index=1;index /// SetOutputFolderPath /// ///The test cases public void SetOutputFolderPath(String strFolderPath) { this.strFolderPath=strFolderPath; if(!this.strFolderPath.EndsWith(@"/"))this.strFolderPath=this.strFolderPath+@"/"; } /// /// ClearFolderPath /// ///The test cases public void ClearFolderPath() { if(String.IsNullOrEmpty(strFolderPath))throw new InvalidDataException($"{nameof(strFolderPath)} cannot be null"); if(!Directory.Exists(strFolderPath)) { Directory.CreateDirectory(strFolderPath); } else { String[] pathFileNames = Directory.GetFiles(strFolderPath); Console.WriteLine($"Deleting {pathFileNames.Length} files from {strFolderPath}"); foreach(String file in pathFileNames) { File.Delete(file); } } } public void ProcessData(TestCases testCases) { for(int index=0;index /// ProcessData item /// ///The test cases public void ProcessData(TestCase testCase,int index=0) { if(null==PenWidthArray)PenWidthArray=new float[]{PenWidth}; if(null==NoiseArray)NoiseArray=new double[]{0.00}; if(testCase.TypeGenerate.Equals(TestCase.GenerateType.BollingerBand)) { testCase.PathFileNames.Add(CreateFileName(testCase.HomePath,testCase.Symbol,testCase.DayCount,0,0,0,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBand,testCase.PurchaseDate)); ProcessBollingerBandData(testCase,PenWidthArray[0],NoiseArray[0]); } else if(testCase.TypeGenerate.Equals(TestCase.GenerateType.BollingerBandWithVIX)) { testCase.PathFileNames.Add(CreateFileName(testCase.HomePath,testCase.Symbol,testCase.DayCount,0,0,0,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBandWithVIX,testCase.PurchaseDate)); ProcessBollingerBandDataWithVolatility(testCase,PenWidthArray[0],NoiseArray[0]); } } private String CreateFileName(String strFolderPath,String symbol,int dayCount,int index,int penIndex,int noiseIndex,TestCase.CaseType caseType,TestCase.GenerateType generateType,DateTime purchaseDate) { if(!strFolderPath.EndsWith("/"))strFolderPath+="/"; return String.Format("{0}{1}_{2}_{3}_{4}_{5}_{6}_{7}_{8}d.jpg",strFolderPath,symbol,index,penIndex,noiseIndex,caseType.ToString(),generateType.ToString(),Utility.DateToLong(purchaseDate),dayCount); } /// /// ProcessBollingerBandData item /// ///Symbol private void ProcessBollingerBandData(TestCase testCase,float penWidth,double noise) { DateGenerator dateGenerator=new DateGenerator(); int daysInPeriod=dateGenerator.DaysBetweenActual(testCase.PurchaseDate,testCase.HistDate); daysInPeriod+=60; Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,daysInPeriod); BollingerBands bollingerBands=BollingerBandGenerator.GenerateBollingerBands(prices); // we want to grab K, L, and Close bollingerBands=new BollingerBands(bollingerBands.Where(x=>x.Date>=testCase.HistDate).ToList()); float[] k=new float[bollingerBands.Count]; float[] l=new float[bollingerBands.Count]; float[] close=new float[bollingerBands.Count]; // populate the arrays in reverse order so that we have the most historical date in the lowest index for(int index=bollingerBands.Count-1;index>=0;index--) { BollingerBandElement bollingerBandElement=bollingerBands[index]; k[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.K)*1000.00f; l[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.L)*1000.00f; close[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.Close)*1000.00f; } Numerics.ZeroForNaNOrInfinity(ref k); Numerics.ZeroForNaNOrInfinity(ref l); Numerics.ZeroForNaNOrInfinity(ref close); float maxY=Math.Max(Numerics.Max(ref l),Math.Max(Numerics.Max(ref close),Numerics.Max(ref k))); float minY=Math.Min(Numerics.Min(ref l),Math.Min(Numerics.Min(ref close),Numerics.Min(ref k)))-5f; float maxX=close.Length; float minX=0.00f; ImageHelper imageHelper=new ImageHelper(); PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY); imageHelper.CreateImage(Width,Height,pointMapping); imageHelper.Fill(FillBrush); // draw price if(DrawPrice)imageHelper.DrawSeries(DrawingBrushRed,penWidth,close); // draw k imageHelper.DrawSeries(DrawingBrushRed,penWidth,k); // draw l imageHelper.DrawSeries(DrawingBrushRed,penWidth,l); if(0.00!=noise)imageHelper.AddNoise(NoiseColor,noise); if(Debug || testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile)) { MDTrace.WriteLine(LogLevel.DEBUG,$"Writing {testCase.LastPathFileName}"); if(null==testCase.LastPathFileName)testCase.PathFileNames.Add($"{strFolderPath}{testCase.Symbol}.jpg"); if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName); if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName); else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName); } if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputStream)) { testCase.Streams.Add(imageHelper.SaveBlackAndWhiteJPG()); } } /// /// ProcessBollingerBandDataWithVolatility item - Draws Price, K, L and Volatility /// ///Symbol private void ProcessBollingerBandDataWithVolatility(TestCase testCase,float penWidth,double noise) { String symbolVolatility="^VIX"; DateGenerator dateGenerator=new DateGenerator(); int daysInPeriod=dateGenerator.DaysBetweenActual(testCase.PurchaseDate,testCase.HistDate); daysInPeriod+=60; Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,daysInPeriod); Prices volatilityPrices=PricingDA.GetPrices(symbolVolatility,testCase.PurchaseDate,daysInPeriod); BollingerBands bollingerBands=BollingerBandGenerator.GenerateBollingerBands(prices); // we want to grab K, L, and Close bollingerBands=new BollingerBands(bollingerBands.Where(x=>x.Date>=testCase.HistDate).ToList()); float[] k=new float[bollingerBands.Count]; float[] l=new float[bollingerBands.Count]; float[] close=new float[bollingerBands.Count]; // Line up volatility dates with bollinger bands DateTime minDate = bollingerBands.Min(x=>x.Date); DateTime maxDate = bollingerBands.Max(x=>x.Date); volatilityPrices = new Prices(volatilityPrices.Where(x=>x.Date<=maxDate && x.Date>=minDate).OrderBy(x=>x.Date).ToList()); // most historical date in lowest index float[] v=volatilityPrices.GetPrices(); float minV=Numerics.Min(ref v); // get the minimum volatility value double minP=bollingerBands.Min(x=>x.Close); // get minimum price double factor=minP/minV; // determine scaling factor for(int index=0;index=0;index--) { BollingerBandElement bollingerBandElement=bollingerBands[index]; k[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.K)*1000.00f; // put the data in log form l[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.L)*1000.00f; // put the data in log form close[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.Close)*1000.00f; // put the data in log form } Numerics.ZeroForNaNOrInfinity(ref k); Numerics.ZeroForNaNOrInfinity(ref l); Numerics.ZeroForNaNOrInfinity(ref close); Numerics.ZeroForNaNOrInfinity(ref v); float maxY=Math.Max(Math.Max(Numerics.Max(ref l),Math.Max(Numerics.Max(ref close),Numerics.Max(ref k))),Numerics.Max(ref v)); float minY=Math.Min(Math.Min(Numerics.Min(ref l),Math.Min(Numerics.Min(ref close),Numerics.Min(ref k))),Numerics.Min(ref v))-5f; float maxX=close.Length; float minX=0.00f; ImageHelper imageHelper=new ImageHelper(); PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY); imageHelper.CreateImage(Width,Height,pointMapping); imageHelper.Fill(FillBrush); // draw volatility imageHelper.DrawSeries(DrawingBrushRed,penWidth,v); // draw prices if(DrawPrice)imageHelper.DrawSeries(DrawingBrush,penWidth,close); // draw k imageHelper.DrawSeries(DrawingBrushGreen,penWidth,k); // draw l imageHelper.DrawSeries(DrawingBrushGreen,penWidth,l); if(0.00!=noise)imageHelper.AddNoise(NoiseColor,noise); if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile)) { MDTrace.WriteLine(LogLevel.DEBUG,$"Writing {testCase.LastPathFileName}"); if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName); if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName); else imageHelper.Save(testCase.LastPathFileName); } else { testCase.Streams.Add(imageHelper.ToStream()); } } /// /// ProcessPriceData item /// ///TestCase private void ProcessPriceData(TestCase testCase,float penWidth,double noise) { Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,testCase.HistDate); prices.Reverse(); // get the most historical date into the lowest index float[] priceArray=prices.GetPrices(); for(int index=0;index