From ddff56545896c9ebe2ea35341d10e36cfb2a4c9d Mon Sep 17 00:00:00 2001 From: Sean Date: Thu, 24 Apr 2025 11:35:17 -0400 Subject: [PATCH] Merge MKDT_0002 --- .../CNNProcessing/BitmapExtensions.cs | 32 ++ .../MarketDataLib/CNNProcessing/CNNClient.cs | 129 +++++ .../CNNProcessing/DataProcessor.cs | 305 ++++++++++++ .../CNNProcessing/ImageHelper.cs | 450 ++++++++++++++++++ .../CNNProcessing/LineSegment.cs | 34 ++ .../CNNProcessing/PointMapping.cs | 47 ++ .../MarketDataLib/CNNProcessing/TestCase.cs | 61 +++ MarketData/MarketDataLib/MarketDataLib.csproj | 2 + MarketData/MarketDataLib/MarketDataLib.sln | 24 + 9 files changed, 1084 insertions(+) create mode 100755 MarketData/MarketDataLib/CNNProcessing/BitmapExtensions.cs create mode 100755 MarketData/MarketDataLib/CNNProcessing/CNNClient.cs create mode 100755 MarketData/MarketDataLib/CNNProcessing/DataProcessor.cs create mode 100644 MarketData/MarketDataLib/CNNProcessing/ImageHelper.cs create mode 100755 MarketData/MarketDataLib/CNNProcessing/LineSegment.cs create mode 100755 MarketData/MarketDataLib/CNNProcessing/PointMapping.cs create mode 100755 MarketData/MarketDataLib/CNNProcessing/TestCase.cs create mode 100644 MarketData/MarketDataLib/MarketDataLib.sln diff --git a/MarketData/MarketDataLib/CNNProcessing/BitmapExtensions.cs b/MarketData/MarketDataLib/CNNProcessing/BitmapExtensions.cs new file mode 100755 index 0000000..cdbeca2 --- /dev/null +++ b/MarketData/MarketDataLib/CNNProcessing/BitmapExtensions.cs @@ -0,0 +1,32 @@ +using System.IO; +//using System.Linq; +// using MarketData.Utils; +//using System.Drawing.Imaging; +//using System.Runtime.InteropServices; +//using System.Drawing; +using SkiaSharp; + +namespace MarketData.CNNProcessing +{ + public static class BitmapExtensions + { +// This call is used to generate the training, test, and validation bitmaps.. +// I noticed a large difference in the accuracy or the model when the quality parameter is 100. +// I therefore intentionally not using EncoderParameters to specify quality but instead taking the default quality. + public static void SaveJPG100(this SKBitmap bitmap, string filename) + { + if(File.Exists(filename))File.Delete(filename); + using FileStream stream = new FileStream(filename, FileMode.Create, FileAccess.Write); + using SKImage image = SKImage.FromBitmap(bitmap); + using SKData encodedImage = image.Encode(SKEncodedImageFormat.Jpeg,100); + encodedImage.SaveTo(stream); + } + + public static void SaveJPG100(this SKBitmap bitmap, Stream stream) + { + using SKImage image = SKImage.FromBitmap(bitmap); + using SKData encodedImage = image.Encode(SKEncodedImageFormat.Jpeg,100); + encodedImage.SaveTo(stream); + } + } +} diff --git a/MarketData/MarketDataLib/CNNProcessing/CNNClient.cs b/MarketData/MarketDataLib/CNNProcessing/CNNClient.cs new file mode 100755 index 0000000..5c90402 --- /dev/null +++ b/MarketData/MarketDataLib/CNNProcessing/CNNClient.cs @@ -0,0 +1,129 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; + +namespace MarketData.CNNProcessing +{ + public class CNNClient + { + public enum Model{resnet50,resnet50B,resnet50_20241024_270,inception,vgg16,lenet5,ping}; + private static readonly string Alive="Alive"; + private readonly HttpClient client = new HttpClient(); + private string baseUrl; + private static int REQUEST_TIMEOUT=15000; + + public CNNClient(String baseUrl) + { + client.Timeout=new TimeSpan(0,0,0,0,REQUEST_TIMEOUT); + if(!baseUrl.StartsWith("http://") && !baseUrl.StartsWith("HTTP://"))baseUrl="http://"+baseUrl; + this.baseUrl=baseUrl; + } + + private String GetModelUrl(CNNClient.Model model) + { + return baseUrl+"/predict_"+model.ToString(); + } + + public String Predict(CNNClient.Model model,Stream stream) + { + try + { + string responsePayload = Upload(GetModelUrl(model),stream).GetAwaiter().GetResult(); + return responsePayload; + } + catch(Exception exception) + { + return exception.ToString(); + } + } + +// This method is used to process an image through PIL. All images that are being used in training should ultimately be processed through +// PIL (Python Image Library) so that images can be normalized by python prior to training. We do this because we use PIL during the +// prediction process and we want all images used in training/validation and all images used for prediction to be touched by and processed +// by PIL + public Stream ProcessImage(Stream stream) + { + return UploadImage(baseUrl+"/process_image", stream); + } + + public bool Ping() + { + try + { + string responsePayload = Upload(baseUrl+"/ping").GetAwaiter().GetResult(); + if(null==responsePayload)return false; + return responsePayload.Equals(Alive); + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception encountered: {0}",exception.ToString())); + return false; + } + } + + private async Task Upload(String url,Stream stream) + { + try + { + using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, url)) + { + int streamEnd = Convert.ToInt32(stream.Length); + byte[] byteArray = new byte[streamEnd]; + stream.Read(byteArray, 0, streamEnd); + request.Content=new ByteArrayContent(byteArray); + using (HttpResponseMessage response = await client.SendAsync(request)) + { + return await response.Content.ReadAsStringAsync(); + } + } + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception encountered: {0}",exception.ToString())); + return null; + } + } + /// + /// This method uploads an image stream to the specified url and receives the processed image back. + /// The processed image simply processes the image through PIL (Python Image Library) + /// + private Stream UploadImage(String url,Stream stream) + { + try + { + using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, url)) + { + int streamEnd = Convert.ToInt32(stream.Length); + byte[] byteArray = new byte[streamEnd]; + stream.Read(byteArray, 0, streamEnd); + request.Content=new ByteArrayContent(byteArray); + HttpResponseMessage response = client.SendAsync(request).GetAwaiter().GetResult(); + return response.Content.ReadAsStreamAsync().GetAwaiter().GetResult(); + } + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception encountered: {0}",exception.ToString())); + return null; + } + } + + private async Task Upload(String url) + { + try + { + using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, url)) + { + HttpResponseMessage response = await client.SendAsync(request); + return await response.Content.ReadAsStringAsync(); + } + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception encountered: {0}",exception.ToString())); + return null; + } + } + } +} diff --git a/MarketData/MarketDataLib/CNNProcessing/DataProcessor.cs b/MarketData/MarketDataLib/CNNProcessing/DataProcessor.cs new file mode 100755 index 0000000..a8de462 --- /dev/null +++ b/MarketData/MarketDataLib/CNNProcessing/DataProcessor.cs @@ -0,0 +1,305 @@ +using MarketData.Numerical; +using MarketData.DataAccess; +using MarketData.MarketDataModel; +using MarketData.Generator; +using MarketData.Utils; +using SkiaSharp; + +namespace MarketData.CNNProcessing +{ + public class DataProcessor + { + private String strFolderPath=default; + private readonly SKColor colorBlack = new SKColor(0,0,0); + private readonly SKColor colorWhite = new SKColor(255,255,255); + + public DataProcessor(int width=640,int height=480) + { + Width=width; + Height=height; + PenWidth=2f; + DrawingBrush = colorBlack; + FillBrush = colorWhite; + DrawPrice=true; + UseGrayScale=false; + NoiseColor=colorWhite; + strFolderPath=Directory.GetCurrentDirectory(); + } + + public bool Debug { get; set; } = false; + + /// + /// Width + /// + ///Gets/Sets the width of the output image + public int Width{get;set;} + + /// + /// Height + /// + ///Gets/Sets the height of the output image + public int Height{get;set;} + + /// + /// PenWidth + /// + ///Gets/Sets the width of the drawing pen + public float PenWidth{get;set;} + + /// + /// FillBrush + /// + ///Gets/Sets the background brush + public SKColor FillBrush { get; set;} + + /// + /// DrawingBrush + /// + ///Gets/Sets the drawing brush brush + public SKColor DrawingBrush{get;set;} + + /// + /// DrawBlack + /// + ///Sets up the canvas to do black drawing + public void DrawBlack() + { + DrawingBrush = colorBlack; + FillBrush = colorWhite; + } + + /// + /// IncludePrice + /// + ///Indicates whether we want to include price data in the graph + public bool DrawPrice{get; set;} + + /// + /// UseGrayScale + /// + ///Indicates whether we want to use grayscale.. default is B&W + public bool UseGrayScale{get; set;} + + /// + /// DrawWhite + /// + ///Sets up the canvas to do white drawing + public void DrawWhite() + { + DrawingBrush = colorBlack; + FillBrush = colorWhite; + } + + /// + /// MovingAverageArray + /// + ///Gets/Sets the moving average array + public int[] MovingAverageArray{get; set; } + + /// + /// PenWidthArray + /// + ///Gets/Sets thepen width array + public float[] PenWidthArray{get; set; } + + /// + /// NoiseArray + /// + ///Gets/Sets the noise array + public double[] NoiseArray{get; set; } + + /// + /// NoiseColor + /// + ///Gets/Sets the noise color + public SKColor NoiseColor{get; set; } + + /// + /// UseNoise + /// + ///The seed value + ///The increment + ///The size + public void UseNoise(double seed, double increment, int length, SKColor noiseColor) + { + NoiseColor=noiseColor; + NoiseArray=new double[length+1]; + NoiseArray[0]=0.00; + for(int index=1;index + /// SetOutputFolderPath + /// + ///The test cases + public void SetOutputFolderPath(String strFolderPath) + { + this.strFolderPath=strFolderPath; + if(!this.strFolderPath.EndsWith(@"/"))this.strFolderPath=this.strFolderPath+@"/"; + } + + public void ProcessData(TestCases testCases) + { + for(int index=0;index + /// ProcessData item + /// + ///The test cases + public void ProcessData(TestCase testCase,int index=0) + { + if(null==PenWidthArray)PenWidthArray=new float[]{PenWidth}; + if(null==NoiseArray)NoiseArray=new double[]{0.00}; + + ProcessBollingerBandData(testCase,PenWidthArray[0],NoiseArray[0]); + } + + private String CreateFileName(String strFolderPath,String symbol,int dayCount,int index,int penIndex,int noiseIndex,TestCase.CaseType caseType,TestCase.GenerateType generateType,DateTime purchaseDate) + { + return String.Format("{0}{1}_{2}_{3}_{4}_{5}_{6}_{7}_{8}d.jpg",strFolderPath,symbol,index,penIndex,noiseIndex,caseType.ToString(),generateType.ToString(),Utility.DateToLong(purchaseDate),dayCount); + } + + /// + /// ProcessBollingerBandData item + /// + ///Symbol + private void ProcessBollingerBandData(TestCase testCase,float penWidth,double noise) + { + DateGenerator dateGenerator=new DateGenerator(); + + int daysInPeriod=dateGenerator.DaysBetweenActual(testCase.PurchaseDate,testCase.HistDate); + daysInPeriod+=60; + Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,daysInPeriod); + BollingerBands bollingerBands=BollingerBandGenerator.GenerateBollingerBands(prices); // we want to grab K, L, and Close + bollingerBands=new BollingerBands(bollingerBands.Where(x=>x.Date>=testCase.HistDate).ToList()); + float[] k=new float[bollingerBands.Count]; + float[] l=new float[bollingerBands.Count]; + float[] close=new float[bollingerBands.Count]; + +// populate the arrays in reverse order so that we have the most historical date in the lowest index + for(int index=bollingerBands.Count-1;index>=0;index--) + { + BollingerBandElement bollingerBandElement=bollingerBands[index]; + k[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.K)*1000.00f; + l[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.L)*1000.00f; + close[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.Close)*1000.00f; + } + Numerics.ZeroForNaNOrInfinity(ref k); + Numerics.ZeroForNaNOrInfinity(ref l); + Numerics.ZeroForNaNOrInfinity(ref close); + float maxY=Math.Max(Numerics.Max(ref l),Math.Max(Numerics.Max(ref close),Numerics.Max(ref k))); + float minY=Math.Min(Numerics.Min(ref l),Math.Min(Numerics.Min(ref close),Numerics.Min(ref k)))-5f; + float maxX=close.Length; + float minX=0.00f; + + ImageHelper imageHelper=new ImageHelper(); + + PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY); + imageHelper.CreateImage(Width,Height,pointMapping); + imageHelper.Fill(FillBrush); + + LineSegments lineSegments=new LineSegments(); + for(int index=0;index + /// ProcessPriceData item + /// + ///TestCase + private void ProcessPriceData(TestCase testCase,float penWidth,double noise) + { + Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,testCase.HistDate); + prices.Reverse(); // get the most historical date into the lowest index + float[] priceArray=prices.GetPrices(); + for(int index=0;index + /// Copy constructor + /// + /// + public ImageHelper(ImageHelper imageHelper) + { + this.bitmap=Copy(imageHelper.bitmap); + pointMapping = new PointMapping(Width,Height,Width,0,Height,0); + } + + public void Dispose() + { + DisposeAll(); + } + + private void DisposeAll() + { + if(null!=bitmap) + { + bitmap.Dispose(); + bitmap=null; + } + } + + private SKBitmap Copy(SKBitmap bitmap) + { + return bitmap.Copy(bitmap.ColorType); + } + + /// + /// Load image from file + /// + /// + /// + public bool LoadImage(string pathFileName) + { + FileStream stream = new FileStream(pathFileName, FileMode.Open); + return LoadImage(stream); + } + + /// + /// Load image from stream retaining the color space and pixel depth of the source image + /// + /// + /// + public bool LoadImage(Stream stream) + { + try + { + DisposeAll(); + using SKImage image=SKImage.FromEncodedData(stream); + SKImageInfo imageInfo = image.Info; + bitmap = new SKBitmap(image.Info); + if(!image.ReadPixels (imageInfo, bitmap.GetPixels (), imageInfo.RowBytes,0, 0)) + { + bitmap.Dispose (); + bitmap = null; + return false; + } + pointMapping = new PointMapping(Width,Height,Width,0,Height,0); + return true; + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); + return false; + } + } + + /// + /// Resize image to given width maintaining aspect ration for height. + /// If aspect ration is used for CNN then you will need to introduce padding. For instance 640,640 image and then paste the aspect + /// mainained image onto that. The CNN will then have to learn that black means nothing. Resizing is the recommended way to go. + /// + /// + /// + /// + public bool Resize(int newWidth) + { + double aspectRatio=(double)bitmap.Width/(double)bitmap.Height; + int newHeight=(int)(((double)newWidth)/aspectRatio); + if(0!=newHeight%2)newHeight++; + return Resize(newWidth,newHeight); + } + + /// + /// Resize image to given width and height. The documenation on CNN's indicates that resizing will work better than maintaining apsect + /// ration with padding + /// + public bool Resize(int newWidth, int newHeight) + { + try + { + Validate(); + SKImageInfo imageInfo = new SKImageInfo(newWidth, newHeight); + SKBitmap newBitmap = bitmap.Resize(imageInfo,SKSamplingOptions.Default); + bitmap.Dispose(); + bitmap = newBitmap; + pointMapping = new PointMapping(Width,Height,Width,0,Height,0); + return true; + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); + return false; + } + } + + /// + /// Create a new 32 bit bitmap with 8 bits per channel which includes 8 bits for the alpha channel where 0 = fully transparent and 255 = fully opaque + /// + /// + /// + /// + public void CreateImage(int width, int height,PointMapping pointMapping) + { + DisposeAll(); + this.pointMapping=pointMapping; + bitmap=new SKBitmap(width,height,SKColorType.Rgba8888, SKAlphaType.Premul); + Validate(); + } + + // This is the method that is currently being used in the CNNClient.Predict to send the image stream to Flask + public Stream SaveBlackAndWhiteJPG() + { + Validate(); + SKBitmap bwBitmap=ToBlackAndWhite(bitmap); + Stream memoryStream=new MemoryStream(); + BitmapExtensions.SaveJPG100(bwBitmap,memoryStream); + memoryStream.Position=0; + bwBitmap.Dispose(); + return memoryStream; + } + + // Convert bitmap to stream for use in CNNClient + public Stream ToStream() + { + Validate(); + Stream memoryStream=new MemoryStream(); + BitmapExtensions.SaveJPG100(bitmap,memoryStream); + memoryStream.Position=0; + return memoryStream; + } + + public bool SaveGrayScaleJPG(String pathFileName) + { + Validate(); + SKBitmap bwBitmap=ToGrayScale(bitmap); + BitmapExtensions.SaveJPG100(bwBitmap,pathFileName); + bwBitmap.Dispose(); + return true; + } + + public bool SaveBlackAndWhiteJPG(String pathFileName) + { + Validate(); + SKBitmap bwBitmap=ToBlackAndWhite(bitmap); + BitmapExtensions.SaveJPG100(bwBitmap,pathFileName); + bwBitmap.Dispose(); + return true; + } + + public void Save(String pathFileName) + { + Save(pathFileName,null); + } + + private void Save(String pathFileName,SKBitmap altBitmap=null) + { + Validate(); + if(null==altBitmap)BitmapExtensions.SaveJPG100(bitmap,pathFileName); + else BitmapExtensions.SaveJPG100(altBitmap,pathFileName); + } + + // Convert to 8 bits per pixel black and white + private SKBitmap ToBlackAndWhite(SKBitmap bitmap) + { + float n = 1.0f/3.0f; + SKColorFilter bwColorFilter = SKColorFilter.CreateColorMatrix(new float[] + { + n, n, n, 0, 0, + n, n, n, 0, 0, + n, n, n, 0, 0, + 0, 0, 0, 1, 0 + }); + + SKBitmap dstBitmap = new SKBitmap(bitmap.Width, bitmap.Height,SKColorType.Gray8, SKAlphaType.Premul); + using SKCanvas canvas = new SKCanvas(dstBitmap); + using SKPaint paint = new SKPaint(); + paint.ColorFilter = bwColorFilter; + canvas.DrawBitmap(bitmap, new SKPoint(0,0), paint); + return dstBitmap; + } + + public bool RotateRight() + { + Validate(); + SKBitmap rotated = Rotate(bitmap, 90); + bitmap.Dispose(); + bitmap=rotated; + pointMapping = new PointMapping(Width,Height,Width,0,Height,0); + return true; + } + + public bool RotateLeft() + { + Validate(); + SKBitmap rotated = Rotate(bitmap, -90); + bitmap.Dispose(); + bitmap=rotated; + pointMapping = new PointMapping(Width,Height,Width,0,Height,0); + return true; + } + + private SKBitmap Rotate(SKBitmap bitmap, double angle) + { + double radians = Math.PI * angle / 180; + float sine = (float)Math.Abs(Math.Sin(radians)); + float cosine = (float)Math.Abs(Math.Cos(radians)); + int originalWidth = bitmap.Width; + int originalHeight = bitmap.Height; + int rotatedWidth = (int)(cosine * originalWidth + sine * originalHeight); + int rotatedHeight = (int)(cosine * originalHeight + sine * originalWidth); + + SKBitmap rotatedBitmap = new SKBitmap(rotatedWidth, rotatedHeight); + + using (SKCanvas surface = new SKCanvas(rotatedBitmap)) + { + surface.Clear(); + surface.Translate(rotatedWidth / 2, rotatedHeight / 2); + surface.RotateDegrees((float)angle); + surface.Translate(-originalWidth / 2, -originalHeight / 2); + surface.DrawBitmap(bitmap, new SKPoint()); + } + return rotatedBitmap; + } + + public void ToGrayScale() + { + Validate(); + SKBitmap grayScaleBitmap = ToGrayScale(bitmap); + bitmap.Dispose(); + bitmap=grayScaleBitmap; + } + + // Convert to 8 bits per pixel black and white + private SKBitmap ToGrayScale(SKBitmap bitmap) + { + SKColorFilter grayScaleColorFilter = SKColorFilter.CreateColorMatrix(new float[] + { + 0.2126f, 0.7152f, 0.0722f, 0, 0, + 0.2126f, 0.7152f, 0.0722f, 0, 0, + 0.2126f, 0.7152f, 0.0722f, 0, 0, + 0, 0, 0, 1, 0 + }); + + SKBitmap dstBitmap = new SKBitmap(bitmap.Width, bitmap.Height,SKColorType.Gray8, SKAlphaType.Premul); + using SKCanvas canvas = new SKCanvas(dstBitmap); + using SKPaint paint = new SKPaint(); + paint.ColorFilter = grayScaleColorFilter; + canvas.DrawBitmap(bitmap, new SKPoint(0,0), paint); + return dstBitmap; + } + + public bool Blur(float sigmaX) + { + Validate(); + SKBitmap blurredBitmap = Blur(bitmap, sigmaX); + bitmap.Dispose(); + bitmap = blurredBitmap; + return true; + } + + private SKBitmap Blur(SKBitmap image, float sigmaX=5.0f, float sigmaY=5.0f) + { + SKImageFilter imageFilter = SKImageFilter.CreateBlur(sigmaX, sigmaY); + SKBitmap blurredBitmap = new SKBitmap(image.Width, image.Height); + using SKCanvas canvas = new SKCanvas(blurredBitmap); + using SKPaint paint = new SKPaint() + { + ImageFilter = imageFilter + }; + canvas.DrawBitmap(bitmap, new SKPoint(0,0), paint); + return blurredBitmap; + } + + /// + /// GetPixel - No point translation + /// + /// + /// + /// + public SKColor GetPixel(int x,int y) + { + Validate(); + return bitmap.GetPixel(x, y); + } + + /// + /// SetPixel - No point translation + /// + /// + /// + /// + public void SetPixel(int x, int y,SKColor color) + { + Validate(); + bitmap.SetPixel(x, y, color); + } + + public SKPoint TranslatePoint(SKPoint point) + { + return pointMapping.TranslatePoint(point); + } + + public void Validate() + { + if(null==bitmap)throw new InvalidDataException("The image has not been initialized"); + } + + public int Width + { + get + { + return bitmap.Width; + } + } + + public int Height + { + get + { + return bitmap.Height; + } + } + + public void Fill(SKColor color) + { + SKRectI rect = new SKRectI(0, 0, Width, Height); // x, y, width, height + bitmap.Erase(color, rect); + } + + /// + /// DrawPoint - With translation + /// + /// + /// + public void DrawPoint(SKColor color,SKPoint drawPoint) + { + Validate(); + using SKCanvas canvas = new SKCanvas(bitmap); + canvas.Clear(); + SKPoint txPoint = pointMapping.MapPoint(drawPoint); + canvas.DrawPoint(drawPoint, color); + } + + /// + /// DrawPoint - with given strokeWidth and translation + /// + /// + /// + /// + public void DrawPoint(SKColor color, float strokeWidth, SKPoint drawPoint) + { + Validate(); + SKPoint txPoint = pointMapping.MapPoint(drawPoint); + using SKCanvas canvas = new SKCanvas(bitmap); + using SKPaint paint = new SKPaint(); + paint.Style = SKPaintStyle.Stroke; + paint.StrokeWidth = strokeWidth; // Set the desired stroke width + paint.Color = SKColors.Black; + + SKPoint[] points = new SKPoint[] { txPoint}; + + canvas.DrawPoints(SKPointMode.Points, points, paint); + } + + /// + /// Add noise to the image + /// + /// + /// + public void AddNoise(SKColor color,double percentDecimal) + { + Random random=new Random(); + int amount=(int)(percentDecimal*(Width*Height)); + for(int index=0;index + { + public LineSegments() + { + } + public void Add(SKPoint p1,SKPoint p2) + { + Add(new LineSegment(p1,p2)); + } + } +} diff --git a/MarketData/MarketDataLib/CNNProcessing/PointMapping.cs b/MarketData/MarketDataLib/CNNProcessing/PointMapping.cs new file mode 100755 index 0000000..b9bebfe --- /dev/null +++ b/MarketData/MarketDataLib/CNNProcessing/PointMapping.cs @@ -0,0 +1,47 @@ +using SkiaSharp; + +namespace MarketData.CNNProcessing +{ + public class PointMapping + { + public PointMapping(double width,double height,double xDataExtent,double xDataExtentMin,double yDataExtent,double yDataExtentMin,double xMarginExtent=0.00,double yMarginExtent=0.00) + { + Width=width; + Height=height; + XMargin=xMarginExtent; + YMargin=yMarginExtent; + XDataExtent=xDataExtent; + XDataExtentMin=xDataExtentMin; + YDataExtent=yDataExtent; + YDataExtentMin=yDataExtentMin; + XScalingFactor=(Width-(XMargin*2.00))/(XDataExtent-XDataExtentMin); + YScalingFactor=(Height-(YMargin*2.00))/(YDataExtent-YDataExtentMin); + } +// MapPoint will both scale the given point and translate the given point from an upper left origin system to a bottom left origin system + public SKPoint MapPoint(SKPoint sourcePoint) + { + SKPoint mappedPoint=new SKPoint((int)((sourcePoint.X-XDataExtentMin)*XScalingFactor),(int)(Height-((sourcePoint.Y-YDataExtentMin)*YScalingFactor))); + mappedPoint.X+=(int)XMargin; // offset by the xMargin + mappedPoint.Y-=(int)YMargin; // offset by the yMargin + return mappedPoint; + } +// TranslatePoint will only translate the given point from an upper left origin system to a bottom left origin system + public SKPoint TranslatePoint(SKPoint sourcePoint) + { + SKPoint mappedPoint=new SKPoint((int)sourcePoint.X,(int)(Height-sourcePoint.Y-1)); + return mappedPoint; + } + public double Width{get;private set;} + public double Height{get;private set;} + public double XDataExtent{get;private set;} + public double XDataExtentMin{get;private set;} + public double XRange{get{return XDataExtent-XDataExtentMin;}} + public double YDataExtent{get;private set;} + public double YDataExtentMin{get;private set;} + public double YRange{get{return YDataExtent-YDataExtentMin;}} + public double XMargin{get;private set;} + public double YMargin{get;private set;} + public double XScalingFactor{get;private set;} + public double YScalingFactor{get;private set;} + } +} diff --git a/MarketData/MarketDataLib/CNNProcessing/TestCase.cs b/MarketData/MarketDataLib/CNNProcessing/TestCase.cs new file mode 100755 index 0000000..888942c --- /dev/null +++ b/MarketData/MarketDataLib/CNNProcessing/TestCase.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MarketData.Utils; +using System.IO; + +namespace MarketData.CNNProcessing +{ + public class TestCases : List + { + } + public class TestCase + { + public enum CaseType{Training,Test,Validation}; + public enum GenerateType{Price,BollingerBand}; + public enum OutputType{OutputFile,OutputStream} + private readonly List streams=new List(); + private readonly List pathFileNames=new List(); + private readonly List responses=new List(); + /// + /// ProcessData item + /// + ///Symbol + ///The start datee. + ///The purchasedate. + public TestCase(String symbol,DateTime purchaseDate, int dayCount,TestCase.CaseType caseType,TestCase.GenerateType generateType=TestCase.GenerateType.Price,OutputType outputType=OutputType.OutputFile) + { + DateGenerator dateGenerator=new DateGenerator(); + this.Symbol=symbol; + this.HistDate=dateGenerator.GenerateHistoricalDate(purchaseDate,dayCount); + this.PurchaseDate=dateGenerator.GetNextBusinessDay(purchaseDate); + this.TypeCase=caseType; + TypeGenerate=generateType; + TypeOutput=outputType; + DayCount=dayCount; + } + + public String Symbol{get;set;} + public DateTime PurchaseDate{get;set;} + public DateTime HistDate{get;set;} + public CaseType TypeCase{get;set;} + public int DayCount{get;private set;} + public GenerateType TypeGenerate{get;set;} + public OutputType TypeOutput{get;set;} + public List Streams{get{return streams;}} // This will get set automatically depending on OutputType + public List PathFileNames{get{return pathFileNames;}} // This will get set automaticall depending on OutputType + public List Responses{get{return responses;}} + public Stream LastStream{get{return Streams.Count>0?Streams[Streams.Count-1]:null;}} + public String LastPathFileName{get{return PathFileNames.Count>0?PathFileNames[PathFileNames.Count-1]:null;}} + public String LastResponse{get{return Responses.Count>0?Responses[Responses.Count-1]:null;}} + public override String ToString() + { + StringBuilder sb=new StringBuilder(); + if(OutputType.OutputFile.Equals(TypeOutput))return LastPathFileName; + sb.Append(String.Format("{0}:P:{1}->H:{2}",Symbol,PurchaseDate.ToShortDateString(),HistDate.ToShortDateString())); + return sb.ToString(); + } + } +} diff --git a/MarketData/MarketDataLib/MarketDataLib.csproj b/MarketData/MarketDataLib/MarketDataLib.csproj index e4ff123..f9c09b2 100755 --- a/MarketData/MarketDataLib/MarketDataLib.csproj +++ b/MarketData/MarketDataLib/MarketDataLib.csproj @@ -18,5 +18,7 @@ + + diff --git a/MarketData/MarketDataLib/MarketDataLib.sln b/MarketData/MarketDataLib/MarketDataLib.sln new file mode 100644 index 0000000..3583e07 --- /dev/null +++ b/MarketData/MarketDataLib/MarketDataLib.sln @@ -0,0 +1,24 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.2.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MarketDataLib", "MarketDataLib.csproj", "{863C2A9E-7B6F-7477-E7E0-47D306D6C5D7}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {863C2A9E-7B6F-7477-E7E0-47D306D6C5D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {863C2A9E-7B6F-7477-E7E0-47D306D6C5D7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {863C2A9E-7B6F-7477-E7E0-47D306D6C5D7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {863C2A9E-7B6F-7477-E7E0-47D306D6C5D7}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {0D40B150-3460-44B3-AF3E-48674E77F642} + EndGlobalSection +EndGlobal