From 3e0f78327861d7dd98c5202c9333261712fec9d7 Mon Sep 17 00:00:00 2001 From: Sean Date: Wed, 24 Apr 2024 13:35:31 -0400 Subject: [PATCH] Fix Headlines entry UTF8 issue. Fix issue with ImageHelper GrayScale. --- MarketDataLib/CNNProcessing/ImageHelper.cs | 92 ++++++++++------------ MarketDataLib/Helper/MarketDataHelper.cs | 3 + MarketDataLib/Utility/Utility.cs | 6 ++ 3 files changed, 51 insertions(+), 50 deletions(-) diff --git a/MarketDataLib/CNNProcessing/ImageHelper.cs b/MarketDataLib/CNNProcessing/ImageHelper.cs index 2d0b93f..927ef3a 100644 --- a/MarketDataLib/CNNProcessing/ImageHelper.cs +++ b/MarketDataLib/CNNProcessing/ImageHelper.cs @@ -121,38 +121,6 @@ namespace MarketData.CNNProcessing } } - /// - /// Resize the image to the specified width and height. - /// - /// The image to resize. - /// The width to resize to. - /// The height to resize to. - /// The resized image. - //public static Bitmap ResizeImage(Image image, int width, int height) - //{ - // var destRect = new Rectangle(0, 0, width, height); - // var destImage = new Bitmap(width, height); - - // destImage.SetResolution(image.HorizontalResolution, image.VerticalResolution); - - // using (var graphics = Graphics.FromImage(destImage)) - // { - // graphics.CompositingMode = CompositingMode.SourceCopy; - // graphics.CompositingQuality = CompositingQuality.HighQuality; - // graphics.InterpolationMode = InterpolationMode.HighQualityBicubic; - // graphics.SmoothingMode = SmoothingMode.HighQuality; - // graphics.PixelOffsetMode = PixelOffsetMode.HighQuality; - - // using (var wrapMode = new ImageAttributes()) - // { - // wrapMode.SetWrapMode(WrapMode.TileFlipXY); - // graphics.DrawImage(image, destRect, 0, 0, image.Width,image.Height, GraphicsUnit.Pixel, wrapMode); - // } - // } - - // return destImage; - //} - public void CreateImage(int width, int height,PointMapping pointMapping) { this.width=width; @@ -172,13 +140,16 @@ namespace MarketData.CNNProcessing bwBitmap.Dispose(); return memoryStream; } + public void SaveGrayScaleJPG(String pathFileName) { Validate(); - Bitmap bwBitmap=ToBlackAndWhite(bitmap); + Bitmap bwBitmap=ToGrayScale(bitmap); Save(pathFileName,bwBitmap); bwBitmap.Dispose(); } + + // This is the method that is currently being used in the CNNProcessor to generate images public void SaveBlackAndWhiteJPG(String pathFileName) { Validate(); @@ -196,6 +167,7 @@ namespace MarketData.CNNProcessing if(null==altBitmap)BitmapExtensions.SaveJPG100(bitmap,pathFileName); else BitmapExtensions.SaveJPG100(altBitmap,pathFileName); } + // Convert to 8 bits per pixel black and white private Bitmap ToBlackAndWhite(Bitmap bmp) { @@ -217,26 +189,46 @@ namespace MarketData.CNNProcessing result.UnlockBits(data); return result; } -// Convert to 8 bits per pixel gray scale + + // Convert to 8 bits per pixel gray scale + // private Bitmap ToGrayScale(Bitmap bmp) + //{ + // var result = new Bitmap(bmp.Width, bmp.Height, System.Drawing.Imaging.PixelFormat.Format8bppIndexed); + // BitmapData data = result.LockBits(new Rectangle(0, 0, result.Width, result.Height), ImageLockMode.WriteOnly, PixelFormat.Format8bppIndexed); + // byte[] bytes = new byte[data.Height * data.Stride]; + // Marshal.Copy(data.Scan0, bytes, 0, bytes.Length); + // for (int y = 0; y < bmp.Height; y++) + // { + // for (int x = 0; x < bmp.Width; x++) + // { + // Color color = bmp.GetPixel(x, y); + // byte rgb = (byte)((color.R + color.G + color.B) / 3); + // bytes[y * data.Stride + x] = rgb; + // } + // } + // Marshal.Copy(bytes, 0, data.Scan0, bytes.Length); + // result.UnlockBits(data); + // return result; + // } + + private Bitmap ToGrayScale(Bitmap bmp) - { - var result = new Bitmap(bmp.Width, bmp.Height, System.Drawing.Imaging.PixelFormat.Format8bppIndexed); - BitmapData data = result.LockBits(new Rectangle(0, 0, result.Width, result.Height), ImageLockMode.WriteOnly, PixelFormat.Format8bppIndexed); - byte[] bytes = new byte[data.Height * data.Stride]; - Marshal.Copy(data.Scan0, bytes, 0, bytes.Length); - for (int y = 0; y < bmp.Height; y++) - { - for (int x = 0; x < bmp.Width; x++) - { - Color color = bmp.GetPixel(x, y); - byte rgb = (byte)((color.R + color.G + color.B) / 3); - bytes[y * data.Stride + x] = rgb; - } + { + Bitmap bitmap = new Bitmap(bmp.Width, bmp.Height); + + for (int i = 0; i < bmp.Width; i++) + { + for (int x = 0; x < bmp.Height; x++) + { + Color oc = bmp.GetPixel(i, x); + int grayScale = (int)((oc.R * 0.3) + (oc.G * 0.59) + (oc.B * 0.11)); + Color nc = Color.FromArgb(oc.A, grayScale, grayScale, grayScale); + bitmap.SetPixel(i, x, nc); } - Marshal.Copy(bytes, 0, data.Scan0, bytes.Length); - result.UnlockBits(data); - return result; + } + return bitmap; } + public Color GetPixel(int x,int y) { return bitmap.GetPixel(x,y); diff --git a/MarketDataLib/Helper/MarketDataHelper.cs b/MarketDataLib/Helper/MarketDataHelper.cs index 4d46c20..aff9e26 100644 --- a/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketDataLib/Helper/MarketDataHelper.cs @@ -1712,6 +1712,8 @@ namespace MarketData.Helper headline.Entry=headline.Entry.Replace("\\"," "); headline.Entry=headline.Entry.Trim(); headline.Entry=Uri.UnescapeDataString(headline.Entry); + headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry)); + headline.Entry=Utility.RemoveHtml(headline.Entry); headline.Source = "Seeking Alpha"; headlines.Add(headline); } @@ -1781,6 +1783,7 @@ namespace MarketData.Helper else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value); headline.Entry=headline.Entry.Replace("\\"," "); headline.Entry=headline.Entry.Trim(); + headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry)); headline.Entry=Uri.UnescapeDataString(headline.Entry); headline.Source = "Seeking Alpha"; headlines.Add(headline); diff --git a/MarketDataLib/Utility/Utility.cs b/MarketDataLib/Utility/Utility.cs index 5d9d039..08406d5 100644 --- a/MarketDataLib/Utility/Utility.cs +++ b/MarketDataLib/Utility/Utility.cs @@ -115,6 +115,11 @@ namespace MarketData.Utils { return (new WindowsPrincipal(WindowsIdentity.GetCurrent())).IsInRole(WindowsBuiltInRole.Administrator); } + public static String ToUTF8(String str) + { + if(null==str)return str; + return Encoding.UTF8.GetString(Encoding.Default.GetBytes(str)); + } public static String RemoveHtml(String strItem) { if(null==strItem)return null; @@ -133,6 +138,7 @@ namespace MarketData.Utils strItem=strItem.Replace("”",@""""); strItem=strItem.Replace("”",@""""); strItem=strItem.Replace("–","-"); + strItem=strItem.Replace("'","'"); return strItem; } public static String RemoveDivs(String strItem)