Merge MKDT_0002

MarketData/MarketDataLib/CNNProcessing/BitmapExtensions.cs

@@ -0,0 +1,32 @@
+using System.IO;
+//using System.Linq;
+// using MarketData.Utils;
+//using System.Drawing.Imaging;
+//using System.Runtime.InteropServices;
+//using System.Drawing;
+using SkiaSharp;
+
+namespace MarketData.CNNProcessing
+{
+	public static class BitmapExtensions
+	{
+// This call is used to generate the training, test, and validation bitmaps..  
+// I noticed a large difference in the accuracy or the model when the quality parameter is 100.
+// I therefore intentionally not using EncoderParameters to specify quality but instead taking the default quality.
+			public static void SaveJPG100(this SKBitmap bitmap, string filename)
+			{       
+			  if(File.Exists(filename))File.Delete(filename);
+				using FileStream stream = new FileStream(filename, FileMode.Create, FileAccess.Write);
+				using SKImage image = SKImage.FromBitmap(bitmap); 
+				using SKData encodedImage = image.Encode(SKEncodedImageFormat.Jpeg,100);
+				encodedImage.SaveTo(stream);		
+			}
+
+			public static void SaveJPG100(this SKBitmap bitmap, Stream stream)
+			{
+				using SKImage image = SKImage.FromBitmap(bitmap); 
+				using SKData encodedImage = image.Encode(SKEncodedImageFormat.Jpeg,100);
+				encodedImage.SaveTo(stream);			
+			}
+	}
+}

MarketData/MarketDataLib/CNNProcessing/CNNClient.cs

@@ -0,0 +1,129 @@
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Threading.Tasks;
+
+namespace MarketData.CNNProcessing
+{
+  public class CNNClient
+	{
+	  public enum Model{resnet50,resnet50B,resnet50_20241024_270,inception,vgg16,lenet5,ping};
+		private static readonly string Alive="Alive";
+    private readonly HttpClient client = new HttpClient();
+		private string baseUrl;
+    private static int REQUEST_TIMEOUT=15000;
+
+		public CNNClient(String baseUrl)
+		{
+      client.Timeout=new TimeSpan(0,0,0,0,REQUEST_TIMEOUT);
+      if(!baseUrl.StartsWith("http://") && !baseUrl.StartsWith("HTTP://"))baseUrl="http://"+baseUrl;
+		  this.baseUrl=baseUrl;
+		}
+
+		private String GetModelUrl(CNNClient.Model model)
+		{
+		  return baseUrl+"/predict_"+model.ToString();
+		}
+
+    public String Predict(CNNClient.Model model,Stream stream)
+    {
+		  try
+			{
+        string responsePayload = Upload(GetModelUrl(model),stream).GetAwaiter().GetResult();
+				return responsePayload;
+			}
+			catch(Exception exception)
+			{
+			  return exception.ToString();
+			}
+    }
+
+// This method is used to process an image through PIL.  All images that are being used in training should ultimately be processed through
+// PIL (Python Image Library) so that images can be normalized by python prior to training.  We do this because we use PIL during the 
+// prediction process and we want all images used in training/validation and all images used for prediction to be touched by and processed
+// by PIL
+    public Stream ProcessImage(Stream stream)
+    {
+      return UploadImage(baseUrl+"/process_image", stream);
+    }
+
+    public bool Ping()
+    {
+		  try
+			{
+        string responsePayload = Upload(baseUrl+"/ping").GetAwaiter().GetResult();
+        if(null==responsePayload)return false;
+				return responsePayload.Equals(Alive);
+			}
+			catch(Exception exception)
+			{
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception encountered: {0}",exception.ToString()));
+			  return false;
+			}
+    }
+
+    private async Task<String> Upload(String url,Stream stream)
+    {
+      try
+      {
+        using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, url))
+        {
+				  int streamEnd = Convert.ToInt32(stream.Length);
+				  byte[] byteArray = new byte[streamEnd];
+				  stream.Read(byteArray, 0, streamEnd);
+          request.Content=new ByteArrayContent(byteArray);
+				  using (HttpResponseMessage response = await client.SendAsync(request))
+          {
+            return await response.Content.ReadAsStringAsync();
+          }
+        }
+      }
+      catch(Exception exception)
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception encountered: {0}",exception.ToString()));
+        return null;
+      }
+    }
+    /// <summary>
+    /// This method uploads an image stream to the specified url and receives the processed image back.
+    /// The processed image simply processes the image through PIL (Python Image Library)
+    /// </summary>
+    private Stream UploadImage(String url,Stream stream)
+    {
+      try
+      {
+        using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, url))
+        {
+				  int streamEnd = Convert.ToInt32(stream.Length);
+				  byte[] byteArray = new byte[streamEnd];
+				  stream.Read(byteArray, 0, streamEnd);
+          request.Content=new ByteArrayContent(byteArray);
+          HttpResponseMessage response = client.SendAsync(request).GetAwaiter().GetResult();
+          return response.Content.ReadAsStreamAsync().GetAwaiter().GetResult();
+        }
+      }
+      catch(Exception exception)
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception encountered: {0}",exception.ToString()));
+        return null;
+      }
+    }
+
+    private async Task<String> Upload(String url)
+    {
+      try
+      {
+        using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, url))
+        {
+				  HttpResponseMessage response = await client.SendAsync(request);
+          return await response.Content.ReadAsStringAsync();
+        }
+      }
+      catch(Exception exception)
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception encountered: {0}",exception.ToString()));
+        return null;
+      }
+    }
+	}
+}

MarketData/MarketDataLib/CNNProcessing/DataProcessor.cs

@@ -0,0 +1,305 @@
+using MarketData.Numerical;
+using MarketData.DataAccess;
+using MarketData.MarketDataModel;
+using MarketData.Generator;
+using MarketData.Utils;
+using SkiaSharp;
+
+namespace MarketData.CNNProcessing
+{
+  public class DataProcessor
+	{
+	  private String strFolderPath=default;
+    private readonly SKColor colorBlack = new SKColor(0,0,0);
+		private readonly SKColor colorWhite = new SKColor(255,255,255);
+
+	  public DataProcessor(int width=640,int height=480)
+		{
+		  Width=width;
+			Height=height;
+			PenWidth=2f;
+			DrawingBrush = colorBlack;
+			FillBrush = colorWhite;
+			DrawPrice=true;
+			UseGrayScale=false;
+      NoiseColor=colorWhite;
+			strFolderPath=Directory.GetCurrentDirectory();
+		}
+
+    public bool Debug { get; set; } = false;
+
+		/// <summary>
+		/// Width
+		/// </summary>
+		///<param name="value">Gets/Sets the width of the output image</param>
+		public int Width{get;set;}
+
+		/// <summary>
+		/// Height
+		/// </summary>
+		///<param name="value">Gets/Sets the height of the output image</param>
+		public int Height{get;set;}
+
+		/// <summary>
+		/// PenWidth
+		/// </summary>
+		///<param name="value">Gets/Sets the width of the drawing pen</param>
+		public float PenWidth{get;set;}
+
+		/// <summary>
+		/// FillBrush
+		/// </summary>
+		///<param name="value">Gets/Sets the background brush</param>
+    public SKColor FillBrush { get; set;}
+
+		/// <summary>
+		/// DrawingBrush
+		/// </summary>
+		///<param name="value">Gets/Sets the drawing brush brush</param>
+		public SKColor DrawingBrush{get;set;}
+
+		/// <summary>
+		/// DrawBlack
+		/// </summary>
+		///<param name="value">Sets up the canvas to do black drawing</param>
+		public void DrawBlack()
+		{
+			DrawingBrush = colorBlack;
+			FillBrush = colorWhite;			
+		}
+
+		/// <summary>
+		/// IncludePrice
+		/// </summary>
+		///<param name="value">Indicates whether we want to include price data in the graph</param>
+		public bool DrawPrice{get; set;}
+
+		/// <summary>
+		/// UseGrayScale
+		/// </summary>
+		///<param name="value">Indicates whether we want to use grayscale.. default is B&W</param>
+		public bool UseGrayScale{get; set;}
+
+		/// <summary>
+		/// DrawWhite
+		/// </summary>
+		///<param name="value">Sets up the canvas to do white drawing</param>
+		public void DrawWhite()
+		{
+			DrawingBrush = colorBlack;
+			FillBrush = colorWhite;			
+		}
+
+		/// <summary>
+		/// MovingAverageArray
+		/// </summary>
+		///<param name="MovingAverageArray">Gets/Sets the moving average array</param>
+		public int[] MovingAverageArray{get; set; }
+
+		/// <summary>
+		/// PenWidthArray
+		/// </summary>
+		///<param name="PenWidthArray">Gets/Sets thepen width array</param>
+		public float[] PenWidthArray{get; set; }
+
+		/// <summary>
+		/// NoiseArray
+		/// </summary>
+		///<param name="NoiseArray">Gets/Sets the noise array</param>
+		public double[] NoiseArray{get; set; }
+
+		/// <summary>
+		/// NoiseColor
+		/// </summary>
+		///<param name="NoiseColor">Gets/Sets the noise color</param>
+		public SKColor NoiseColor{get; set; }
+
+		/// <summary>
+		/// UseNoise
+		/// </summary>
+		///<param name="seed">The seed value</param>
+		///<param name="increment">The increment</param>
+		///<param name="length">The size</param>
+		public void UseNoise(double seed, double increment, int length, SKColor noiseColor)
+    {
+      NoiseColor=noiseColor;
+      NoiseArray=new double[length+1];
+      NoiseArray[0]=0.00;
+      for(int index=1;index<length;index++)
+      {
+        NoiseArray[index]=seed;
+        seed+=increment;
+      }
+    }
+
+		/// <summary>
+		/// SetOutputFolderPath
+		/// </summary>
+		///<param name="testCases">The test cases</param>
+		public void SetOutputFolderPath(String strFolderPath)
+		{
+		  this.strFolderPath=strFolderPath;
+			if(!this.strFolderPath.EndsWith(@"/"))this.strFolderPath=this.strFolderPath+@"/";
+		}
+
+		public void ProcessData(TestCases testCases)
+		{
+			for(int index=0;index<testCases.Count;index++)
+			{
+				TestCase testCase=testCases[index];
+				ProcessData(testCase,index);
+			}
+		}
+
+		/// <summary>
+		/// ProcessData item
+		/// </summary>
+		///<param name="testCases">The test cases</param>
+		public void ProcessData(TestCase testCase,int index=0)
+		{
+			if(null==PenWidthArray)PenWidthArray=new float[]{PenWidth};
+      if(null==NoiseArray)NoiseArray=new double[]{0.00};
+
+			ProcessBollingerBandData(testCase,PenWidthArray[0],NoiseArray[0]);
+		}
+
+    private String CreateFileName(String strFolderPath,String symbol,int dayCount,int index,int penIndex,int noiseIndex,TestCase.CaseType caseType,TestCase.GenerateType generateType,DateTime purchaseDate)
+    {
+      return String.Format("{0}{1}_{2}_{3}_{4}_{5}_{6}_{7}_{8}d.jpg",strFolderPath,symbol,index,penIndex,noiseIndex,caseType.ToString(),generateType.ToString(),Utility.DateToLong(purchaseDate),dayCount);
+    }
+
+		/// <summary>
+		/// ProcessBollingerBandData item
+		/// </summary>
+		///<param name="testCase">Symbol</param>
+		private void ProcessBollingerBandData(TestCase testCase,float penWidth,double noise)
+		{
+		  DateGenerator dateGenerator=new DateGenerator();
+
+			int daysInPeriod=dateGenerator.DaysBetweenActual(testCase.PurchaseDate,testCase.HistDate);
+			daysInPeriod+=60;
+		  Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,daysInPeriod);
+			BollingerBands bollingerBands=BollingerBandGenerator.GenerateBollingerBands(prices);	 // we want to grab K, L, and Close
+			bollingerBands=new BollingerBands(bollingerBands.Where(x=>x.Date>=testCase.HistDate).ToList());
+			float[] k=new float[bollingerBands.Count];
+			float[] l=new float[bollingerBands.Count];
+			float[] close=new float[bollingerBands.Count];
+
+// populate the arrays in reverse order so that we have the most historical date in the lowest index
+			for(int index=bollingerBands.Count-1;index>=0;index--)
+			{
+				BollingerBandElement bollingerBandElement=bollingerBands[index];
+				k[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.K)*1000.00f;
+				l[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.L)*1000.00f;
+				close[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.Close)*1000.00f;
+			}
+			Numerics.ZeroForNaNOrInfinity(ref k);
+			Numerics.ZeroForNaNOrInfinity(ref l);
+			Numerics.ZeroForNaNOrInfinity(ref close);
+			float maxY=Math.Max(Numerics.Max(ref l),Math.Max(Numerics.Max(ref close),Numerics.Max(ref k)));
+			float minY=Math.Min(Numerics.Min(ref l),Math.Min(Numerics.Min(ref close),Numerics.Min(ref k)))-5f;
+			float maxX=close.Length;
+			float minX=0.00f;
+
+      ImageHelper imageHelper=new ImageHelper();
+
+      PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY);
+      imageHelper.CreateImage(Width,Height,pointMapping);
+      imageHelper.Fill(FillBrush);
+
+			LineSegments lineSegments=new LineSegments();
+			for(int index=0;index<close.Length && DrawPrice;index++)
+			{
+			  if(0==index)continue;
+				SKPoint p1=new SKPoint(index-1,(int)close[index-1]);
+				SKPoint p2=new SKPoint(index,(int)close[index]);
+	      lineSegments.Add(p1,p2);
+			}
+			imageHelper.DrawPath(DrawingBrush,penWidth,lineSegments);
+
+			lineSegments.Clear();
+			for(int index=0;index<k.Length;index++)
+			{
+			  if(0==index)continue;
+				SKPoint p1=new SKPoint(index-1,(int)k[index-1]);
+				SKPoint p2=new SKPoint(index,(int)k[index]);
+				lineSegments.Add(p1,p2);
+			}
+			imageHelper.DrawPath(DrawingBrush,penWidth,lineSegments);
+
+			lineSegments.Clear();
+			for(int index=0;index<l.Length;index++)
+			{
+			  if(0==index)continue;
+				SKPoint p1=new SKPoint(index-1,(int)l[index-1]);
+				SKPoint p2=new SKPoint(index,(int)l[index]);
+				lineSegments.Add(p1,p2);
+			}
+			imageHelper.DrawPath(DrawingBrush,penWidth,lineSegments);
+
+      if(0.00!=noise)imageHelper.AddNoise(NoiseColor,noise);
+
+			if(Debug || testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile))
+			{
+			  if(null==testCase.LastPathFileName)testCase.PathFileNames.Add($"{strFolderPath}{testCase.Symbol}.jpg");
+				if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName);
+				if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName);
+				else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName);
+			}
+
+			if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputStream))
+			{
+			  testCase.Streams.Add(imageHelper.SaveBlackAndWhiteJPG());
+			}
+		}
+
+		/// <summary>
+		/// ProcessPriceData item
+		/// </summary>
+		///<param name="testCase">TestCase</param>
+		private void ProcessPriceData(TestCase testCase,float penWidth,double noise)
+		{
+		  Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,testCase.HistDate);
+			prices.Reverse();			// get the most historical date into the lowest index
+			float[] priceArray=prices.GetPrices();
+			for(int index=0;index<priceArray.Length;index++)
+			{
+			  priceArray[index]=(float)Math.Log((double)priceArray[index])*1000.00f;
+			}
+			Numerics.ZeroForNaNOrInfinity(ref priceArray);
+
+			float maxY=Numerics.Max(ref priceArray);
+			float minY=Numerics.Min(ref priceArray)-5f;
+			float maxX=priceArray.Length;
+			float minX=0.00f;
+      
+      ImageHelper imageHelper=new ImageHelper();
+
+      PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY);
+      imageHelper.CreateImage(Width,Height,pointMapping);
+      imageHelper.Fill(FillBrush);
+
+			LineSegments lineSegments=new LineSegments();
+			for(int index=0;index<priceArray.Length;index++)
+			{
+			  if(0==index)continue;
+				SKPoint p1=new SKPoint(index-1,(int)priceArray[index-1]);
+				SKPoint p2=new SKPoint(index,(int)priceArray[index]);
+				lineSegments.Add(p1,p2);
+			}
+			imageHelper.DrawPath(DrawingBrush,penWidth,lineSegments);
+      if(0.00!=noise)imageHelper.AddNoise(NoiseColor,noise);
+			
+			if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile))
+			{
+				if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName);
+				if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName);
+				else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName);
+			}
+			else
+			{
+			  testCase.Streams.Add(imageHelper.SaveBlackAndWhiteJPG());
+			}
+		}
+	}
+}

MarketData/MarketDataLib/CNNProcessing/ImageHelper.cs

@@ -0,0 +1,450 @@
+using SkiaSharp;
+
+namespace MarketData.CNNProcessing
+{
+  public class ImageHelper : IDisposable
+  {
+      private SKBitmap bitmap = null;
+      private PointMapping pointMapping;
+
+      public ImageHelper()
+      {
+      }
+
+      /// <summary>
+      /// Copy constructor
+      /// </summary>
+      /// <param name="imageHelper"></param>
+      public ImageHelper(ImageHelper imageHelper)
+      {
+        this.bitmap=Copy(imageHelper.bitmap);
+        pointMapping = new PointMapping(Width,Height,Width,0,Height,0);
+      }    
+
+      public void Dispose()
+      {
+        DisposeAll();
+      }
+
+      private void DisposeAll()
+      {
+        if(null!=bitmap)
+        {
+          bitmap.Dispose();
+          bitmap=null;
+        }
+      }
+
+      private SKBitmap Copy(SKBitmap bitmap)
+      {
+        return bitmap.Copy(bitmap.ColorType);
+      }      
+
+      /// <summary>
+      /// Load image from file
+      /// </summary>
+      /// <param name="pathFileName"></param>
+      /// <returns></returns>
+      public bool LoadImage(string pathFileName) 
+      {
+        FileStream stream = new FileStream(pathFileName, FileMode.Open);
+        return LoadImage(stream);
+      } 
+
+      /// <summary>
+      /// Load image from stream retaining the color space and pixel depth of the source image
+      /// </summary>
+      /// <param name="stream"></param>
+      /// <returns></returns>
+      public bool LoadImage(Stream stream)
+      {
+        try
+        {
+          DisposeAll();
+          using SKImage image=SKImage.FromEncodedData(stream);
+          SKImageInfo imageInfo = image.Info;
+          bitmap = new SKBitmap(image.Info);
+          if(!image.ReadPixels (imageInfo, bitmap.GetPixels (), imageInfo.RowBytes,0, 0))
+          {
+            bitmap.Dispose ();
+            bitmap = null;
+            return false;
+          }          
+          pointMapping = new PointMapping(Width,Height,Width,0,Height,0);
+          return true;
+        }
+        catch(Exception exception)
+        {
+          MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
+          return false;
+        }
+      } 
+
+      /// <summary>
+      /// Resize image to given width maintaining aspect ration for height.
+      /// If aspect ration is used for CNN then you will need to introduce padding.  For instance 640,640 image and then paste the aspect 
+      /// mainained image onto that.  The CNN will then have to learn that black means nothing.  Resizing is the recommended way to go.
+      /// </summary>    
+      /// <param name="newWidth"></param>
+      /// <returns></returns>
+      /// 
+      public bool Resize(int newWidth)
+      {
+        double aspectRatio=(double)bitmap.Width/(double)bitmap.Height;
+        int newHeight=(int)(((double)newWidth)/aspectRatio);
+        if(0!=newHeight%2)newHeight++;
+        return Resize(newWidth,newHeight);
+      } 
+
+      /// <summary>
+      /// Resize image to given width and height.  The documenation on CNN's indicates that resizing will work better than maintaining apsect 
+      /// ration with padding
+      /// </summary>  
+      public bool Resize(int newWidth, int newHeight)
+      {
+        try
+        {
+          Validate();
+          SKImageInfo imageInfo = new SKImageInfo(newWidth, newHeight);
+          SKBitmap newBitmap = bitmap.Resize(imageInfo,SKSamplingOptions.Default);
+          bitmap.Dispose();
+          bitmap = newBitmap;
+          pointMapping = new PointMapping(Width,Height,Width,0,Height,0);          
+          return true;
+        }
+        catch(Exception exception)
+        {
+          MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
+          return false;
+        }
+      }  
+
+      /// <summary>
+      /// Create a new 32 bit bitmap with 8 bits per channel which includes 8 bits for the alpha channel where 0 = fully transparent and 255 = fully opaque
+      /// </summary>
+      /// <param name="width"></param>
+      /// <param name="height"></param>
+      /// <param name="pointMapping"></param>
+      public void CreateImage(int width, int height,PointMapping pointMapping)
+      {
+        DisposeAll();
+        this.pointMapping=pointMapping;
+        bitmap=new SKBitmap(width,height,SKColorType.Rgba8888, SKAlphaType.Premul);
+        Validate();
+      }   
+
+      // This is the method that is currently being used in the CNNClient.Predict to send the image stream to Flask
+      public Stream SaveBlackAndWhiteJPG()
+      {
+        Validate();
+        SKBitmap bwBitmap=ToBlackAndWhite(bitmap);
+        Stream memoryStream=new MemoryStream();
+        BitmapExtensions.SaveJPG100(bwBitmap,memoryStream);
+        memoryStream.Position=0;
+        bwBitmap.Dispose();
+        return memoryStream;
+      } 
+
+      // Convert bitmap to stream for use in CNNClient
+      public Stream ToStream()
+      {
+        Validate();
+        Stream memoryStream=new MemoryStream();
+        BitmapExtensions.SaveJPG100(bitmap,memoryStream);
+        memoryStream.Position=0;
+        return memoryStream;
+      }  
+
+      public bool SaveGrayScaleJPG(String pathFileName)
+      {
+        Validate();
+        SKBitmap bwBitmap=ToGrayScale(bitmap);
+        BitmapExtensions.SaveJPG100(bwBitmap,pathFileName);
+        bwBitmap.Dispose();
+        return true;
+      }                  
+
+      public bool SaveBlackAndWhiteJPG(String pathFileName)
+      {
+        Validate();
+        SKBitmap bwBitmap=ToBlackAndWhite(bitmap);
+        BitmapExtensions.SaveJPG100(bwBitmap,pathFileName);
+        bwBitmap.Dispose();
+        return true;
+      }    
+
+      public void Save(String pathFileName)
+      {
+        Save(pathFileName,null);
+      }
+
+      private void Save(String pathFileName,SKBitmap altBitmap=null)
+      {
+       Validate();
+       if(null==altBitmap)BitmapExtensions.SaveJPG100(bitmap,pathFileName);
+       else BitmapExtensions.SaveJPG100(altBitmap,pathFileName);
+      }           
+
+    // Convert to 8 bits per pixel black and white
+      private SKBitmap ToBlackAndWhite(SKBitmap bitmap)
+      {
+        float n = 1.0f/3.0f;
+        SKColorFilter bwColorFilter = SKColorFilter.CreateColorMatrix(new float[]
+                      {
+                          n, n, n, 0, 0,
+                          n, n, n, 0, 0,
+                          n, n, n, 0, 0,
+                          0, 0, 0, 1, 0
+                      });   
+
+        SKBitmap dstBitmap = new SKBitmap(bitmap.Width, bitmap.Height,SKColorType.Gray8, SKAlphaType.Premul);
+        using SKCanvas canvas = new SKCanvas(dstBitmap);
+        using SKPaint paint = new SKPaint();
+        paint.ColorFilter = bwColorFilter;
+        canvas.DrawBitmap(bitmap, new SKPoint(0,0), paint);
+        return dstBitmap;
+      }
+
+      public bool RotateRight()
+      {
+        Validate();
+        SKBitmap rotated = Rotate(bitmap, 90);
+        bitmap.Dispose();
+        bitmap=rotated;
+        pointMapping = new PointMapping(Width,Height,Width,0,Height,0);         
+        return true;
+      }
+
+      public bool RotateLeft()
+      {
+        Validate();
+        SKBitmap rotated = Rotate(bitmap, -90);
+        bitmap.Dispose();
+        bitmap=rotated;
+        pointMapping = new PointMapping(Width,Height,Width,0,Height,0);         
+        return true;
+      }
+
+      private SKBitmap Rotate(SKBitmap bitmap, double angle)
+      {
+        double radians = Math.PI * angle / 180;
+        float sine = (float)Math.Abs(Math.Sin(radians));
+        float cosine = (float)Math.Abs(Math.Cos(radians));
+        int originalWidth = bitmap.Width;
+        int originalHeight = bitmap.Height;
+        int rotatedWidth = (int)(cosine * originalWidth + sine * originalHeight);
+        int rotatedHeight = (int)(cosine * originalHeight + sine * originalWidth);
+
+        SKBitmap rotatedBitmap = new SKBitmap(rotatedWidth, rotatedHeight);
+
+        using (SKCanvas surface = new SKCanvas(rotatedBitmap))
+        {
+          surface.Clear();
+          surface.Translate(rotatedWidth / 2, rotatedHeight / 2);
+          surface.RotateDegrees((float)angle);
+          surface.Translate(-originalWidth / 2, -originalHeight / 2);
+          surface.DrawBitmap(bitmap, new SKPoint());
+        }
+        return rotatedBitmap;
+      } 
+
+      public void ToGrayScale()
+      {
+        Validate();
+        SKBitmap grayScaleBitmap = ToGrayScale(bitmap);
+        bitmap.Dispose();
+        bitmap=grayScaleBitmap;
+      }
+
+    // Convert to 8 bits per pixel black and white
+      private SKBitmap ToGrayScale(SKBitmap bitmap)
+      {
+        SKColorFilter grayScaleColorFilter = SKColorFilter.CreateColorMatrix(new float[]
+                      {
+                          0.2126f, 0.7152f, 0.0722f, 0, 0,
+                          0.2126f, 0.7152f, 0.0722f, 0, 0,
+                          0.2126f, 0.7152f, 0.0722f, 0, 0,
+                          0,     0,     0,     1, 0
+                      });
+
+        SKBitmap dstBitmap = new SKBitmap(bitmap.Width, bitmap.Height,SKColorType.Gray8, SKAlphaType.Premul);
+        using SKCanvas canvas = new SKCanvas(dstBitmap);
+        using SKPaint paint = new SKPaint();
+        paint.ColorFilter = grayScaleColorFilter;
+        canvas.DrawBitmap(bitmap, new SKPoint(0,0), paint);
+        return dstBitmap;
+      } 
+
+      public bool Blur(float sigmaX)
+      {
+        Validate();
+        SKBitmap blurredBitmap = Blur(bitmap, sigmaX);
+        bitmap.Dispose();
+        bitmap = blurredBitmap;
+        return true;
+      }
+
+      private SKBitmap Blur(SKBitmap image, float sigmaX=5.0f, float sigmaY=5.0f)
+      {
+        SKImageFilter imageFilter = SKImageFilter.CreateBlur(sigmaX, sigmaY);
+        SKBitmap blurredBitmap = new SKBitmap(image.Width, image.Height);
+        using SKCanvas canvas = new SKCanvas(blurredBitmap);
+        using SKPaint paint = new SKPaint()
+        {
+          ImageFilter = imageFilter
+        };
+        canvas.DrawBitmap(bitmap, new SKPoint(0,0), paint);
+        return blurredBitmap;
+      }
+
+      /// <summary>
+      /// GetPixel - No point translation
+      /// </summary>
+      /// <param name="x"></param>
+      /// <param name="y"></param>
+      /// <returns></returns>
+      public SKColor GetPixel(int x,int y)
+      {
+        Validate();
+        return bitmap.GetPixel(x, y);
+      }
+
+      /// <summary>
+      /// SetPixel - No point translation
+      /// </summary>
+      /// <param name="x"></param>
+      /// <param name="y"></param>
+      /// <param name="color"></param>
+      public void SetPixel(int x, int y,SKColor color)
+      {
+        Validate();
+        bitmap.SetPixel(x, y, color);
+      }
+
+      public SKPoint TranslatePoint(SKPoint point)
+      {
+        return pointMapping.TranslatePoint(point);
+      }
+
+      public void Validate()
+      {
+        if(null==bitmap)throw new InvalidDataException("The image has not been initialized");
+      }  
+
+      public int Width
+      {
+        get 
+        {
+          return bitmap.Width;
+        }
+      }
+
+      public int Height
+      {
+        get
+        {
+          return bitmap.Height;
+        }
+      } 
+
+      public void Fill(SKColor color)
+      {
+        SKRectI rect = new SKRectI(0, 0, Width, Height); // x, y, width, height
+        bitmap.Erase(color, rect);
+      }     
+
+      /// <summary>
+      /// DrawPoint - With translation
+      /// </summary>
+      /// <param name="color"></param>
+      /// <param name="drawPoint"></param>
+      public void DrawPoint(SKColor color,SKPoint drawPoint)
+      {
+        Validate();
+        using SKCanvas canvas = new SKCanvas(bitmap);
+        canvas.Clear();
+        SKPoint txPoint = pointMapping.MapPoint(drawPoint);
+        canvas.DrawPoint(drawPoint, color);
+      }
+
+      /// <summary>
+      /// DrawPoint - with given strokeWidth and translation 
+      /// </summary>
+      /// <param name="color"></param>
+      /// <param name="strokeWidth"></param>
+      /// <param name="drawPoint"></param>
+      public void DrawPoint(SKColor color, float strokeWidth, SKPoint drawPoint)
+      {
+        Validate();
+        SKPoint txPoint = pointMapping.MapPoint(drawPoint);
+        using SKCanvas canvas = new SKCanvas(bitmap);
+        using SKPaint paint = new SKPaint();
+        paint.Style = SKPaintStyle.Stroke;
+        paint.StrokeWidth = strokeWidth; // Set the desired stroke width
+        paint.Color = SKColors.Black;
+
+        SKPoint[] points = new SKPoint[] { txPoint};
+
+        canvas.DrawPoints(SKPointMode.Points, points, paint);       
+      }
+
+      /// <summary>
+      /// Add noise to the image
+      /// </summary>
+      /// <param name="color"></param>
+      /// <param name="percentDecimal"></param>
+      public void AddNoise(SKColor color,double percentDecimal)
+      {
+        Random random=new Random();
+        int amount=(int)(percentDecimal*(Width*Height));
+        for(int index=0;index<amount;index++)
+        {
+          int x=random.Next(Width-1);
+          int y=random.Next(Height-1);
+          SetPixel(x, y,color);
+        }
+      } 
+
+      public void DrawLine(SKColor color, float strokeWidth, SKPoint srcPoint,SKPoint dstPoint)
+      {
+        Validate();
+        SKPoint txSrcPoint=pointMapping.MapPoint(srcPoint);
+        SKPoint txDstPoint=pointMapping.MapPoint(dstPoint);
+        using SKCanvas canvas = new SKCanvas(bitmap);
+        using SKPaint paint = new SKPaint();
+        paint.Style = SKPaintStyle.Stroke;
+        paint.StrokeWidth = strokeWidth; // Set the desired stroke width
+        paint.Color = SKColors.Black;
+        canvas.DrawLine(txSrcPoint, txDstPoint, paint);
+      } 
+
+      public void DrawPath(SKColor color, float strokeWidth, LineSegments lineSegments)
+      {
+        Validate();
+
+        using SKCanvas canvas = new SKCanvas(bitmap);
+        using SKPaint paint = new SKPaint();
+        paint.Style = SKPaintStyle.Stroke;
+        paint.StrokeWidth = strokeWidth; // Set the desired stroke width
+        paint.Color = SKColors.Black;
+        foreach(LineSegment lineSegment in lineSegments)
+        {
+          SKPoint txSrcPoint=pointMapping.MapPoint(lineSegment.P1);
+          SKPoint txDstPoint=pointMapping.MapPoint(lineSegment.P2);
+          canvas.DrawLine(txSrcPoint, txDstPoint, paint); 
+        }
+      }                  
+
+      public bool DrawCircle(SKColor color, SKPoint center, float radius=1.00f)
+      {
+        Validate();
+        SKPoint txPointCenter=pointMapping.MapPoint(center);
+        using SKPaint paint = new SKPaint();
+        paint.Color = color;
+        paint.Style = SKPaintStyle.Fill;
+        using SKCanvas canvas = new SKCanvas(bitmap);
+        canvas.DrawCircle(txPointCenter, radius, paint);
+        return true;
+      }
+   }
+}

MarketData/MarketDataLib/CNNProcessing/LineSegment.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using SkiaSharp;
+
+namespace MarketData.CNNProcessing
+{
+  public class LineSegment
+	{
+	  public LineSegment()
+		{
+		}
+		public LineSegment(SKPoint p1,SKPoint p2)
+		{
+		  P1=p1;
+			P2=p2;
+		}
+	  public SKPoint P1{get;set;}
+		public SKPoint P2{get;set;}
+	}
+
+	public class LineSegments : List<LineSegment>
+	{
+	  public LineSegments()
+		{
+		}
+		public void Add(SKPoint p1,SKPoint p2)
+		{
+		  Add(new LineSegment(p1,p2));
+		}
+	}
+}

MarketData/MarketDataLib/CNNProcessing/PointMapping.cs

@@ -0,0 +1,47 @@
+using SkiaSharp;
+
+namespace MarketData.CNNProcessing
+{
+  public class PointMapping
+  {
+    public PointMapping(double width,double height,double xDataExtent,double xDataExtentMin,double yDataExtent,double yDataExtentMin,double xMarginExtent=0.00,double yMarginExtent=0.00)
+    {
+      Width=width;
+      Height=height;
+      XMargin=xMarginExtent; 
+      YMargin=yMarginExtent;
+      XDataExtent=xDataExtent;
+      XDataExtentMin=xDataExtentMin;
+      YDataExtent=yDataExtent;
+      YDataExtentMin=yDataExtentMin;
+      XScalingFactor=(Width-(XMargin*2.00))/(XDataExtent-XDataExtentMin);
+      YScalingFactor=(Height-(YMargin*2.00))/(YDataExtent-YDataExtentMin);
+    }
+// MapPoint will both scale the given point and translate the given point from an upper left origin system to a bottom left origin system
+    public SKPoint MapPoint(SKPoint sourcePoint)
+    {
+      SKPoint mappedPoint=new SKPoint((int)((sourcePoint.X-XDataExtentMin)*XScalingFactor),(int)(Height-((sourcePoint.Y-YDataExtentMin)*YScalingFactor)));
+      mappedPoint.X+=(int)XMargin;   // offset by the xMargin
+      mappedPoint.Y-=(int)YMargin;   // offset by the yMargin
+      return mappedPoint;
+    }
+// TranslatePoint will only translate the given point from an upper left origin system to a bottom left origin system
+    public SKPoint TranslatePoint(SKPoint sourcePoint)
+    {
+      SKPoint mappedPoint=new SKPoint((int)sourcePoint.X,(int)(Height-sourcePoint.Y-1));
+      return mappedPoint;
+    }
+    public double Width{get;private set;}
+    public double Height{get;private set;}
+    public double XDataExtent{get;private set;}
+    public double XDataExtentMin{get;private set;}
+    public double XRange{get{return XDataExtent-XDataExtentMin;}}
+    public double YDataExtent{get;private set;}
+    public double YDataExtentMin{get;private set;}
+    public double YRange{get{return YDataExtent-YDataExtentMin;}}
+    public double XMargin{get;private set;}
+    public double YMargin{get;private set;}
+    public double XScalingFactor{get;private set;}
+    public double YScalingFactor{get;private set;}
+  }
+}

MarketData/MarketDataLib/CNNProcessing/TestCase.cs

@@ -0,0 +1,61 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MarketData.Utils;
+using System.IO;
+
+namespace MarketData.CNNProcessing
+{
+  public class TestCases : List<TestCase>
+	{
+	}
+	public class TestCase
+	{
+    public enum CaseType{Training,Test,Validation};
+		public enum GenerateType{Price,BollingerBand};
+		public enum OutputType{OutputFile,OutputStream}
+		private readonly List<Stream> streams=new List<Stream>();
+		private readonly List<String> pathFileNames=new List<String>();
+		private readonly List<String> responses=new List<String>();
+		/// <summary>
+		/// ProcessData item
+		/// </summary>
+		///<param name="symbol">Symbol</param>
+		///<param name="startDate">The start datee.</param>
+		///<param name="purchaseDate">The purchasedate.</param>
+	  public TestCase(String symbol,DateTime purchaseDate, int dayCount,TestCase.CaseType caseType,TestCase.GenerateType generateType=TestCase.GenerateType.Price,OutputType outputType=OutputType.OutputFile)
+		{
+		  DateGenerator dateGenerator=new DateGenerator();
+		  this.Symbol=symbol;
+			this.HistDate=dateGenerator.GenerateHistoricalDate(purchaseDate,dayCount);
+			this.PurchaseDate=dateGenerator.GetNextBusinessDay(purchaseDate);
+			this.TypeCase=caseType;
+			TypeGenerate=generateType;
+			TypeOutput=outputType;
+      DayCount=dayCount;
+		}
+
+		public String Symbol{get;set;}
+		public DateTime PurchaseDate{get;set;}
+		public DateTime HistDate{get;set;}
+		public CaseType TypeCase{get;set;}
+    public int DayCount{get;private set;}
+		public GenerateType TypeGenerate{get;set;}
+		public OutputType TypeOutput{get;set;}
+		public List<Stream> Streams{get{return streams;}}						         // This will get set automatically depending on OutputType
+		public List<String> PathFileNames{get{return pathFileNames;}}   		 // This will get set automaticall depending on OutputType
+		public List<String> Responses{get{return responses;}}
+		public Stream LastStream{get{return Streams.Count>0?Streams[Streams.Count-1]:null;}}
+		public String LastPathFileName{get{return PathFileNames.Count>0?PathFileNames[PathFileNames.Count-1]:null;}}
+		public String LastResponse{get{return Responses.Count>0?Responses[Responses.Count-1]:null;}}
+		public override String ToString()
+		{
+		  StringBuilder sb=new StringBuilder();
+			if(OutputType.OutputFile.Equals(TypeOutput))return LastPathFileName;
+			sb.Append(String.Format("{0}:P:{1}->H:{2}",Symbol,PurchaseDate.ToShortDateString(),HistDate.ToShortDateString()));
+			return sb.ToString();
+		}
+	}
+}

MarketData/MarketDataLib/MarketDataLib.csproj

@@ -18,5 +18,7 @@
     <PackageReference Include="MySql.Data" Version="9.2.0" />
     <PackageReference Include="NewtonSoft.Json" Version="13.0.3" />
     <PackageReference Include="System.Drawing.Common" Version="9.0.3" />
+    <PackageReference Include="SkiaSharp" Version="3.116.1" />
+    <PackageReference Include="SkiaSharp.NativeAssets.Linux.NoDependencies" Version="3.116.1" />    
   </ItemGroup>
 </Project>

MarketData/MarketDataLib/MarketDataLib.sln

@@ -0,0 +1,24 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.2.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MarketDataLib", "MarketDataLib.csproj", "{863C2A9E-7B6F-7477-E7E0-47D306D6C5D7}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{863C2A9E-7B6F-7477-E7E0-47D306D6C5D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{863C2A9E-7B6F-7477-E7E0-47D306D6C5D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{863C2A9E-7B6F-7477-E7E0-47D306D6C5D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{863C2A9E-7B6F-7477-E7E0-47D306D6C5D7}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {0D40B150-3460-44B3-AF3E-48674E77F642}
+	EndGlobalSection
+EndGlobal