From 5343f070c6afe863db21832496b5069ae7d8c9f7 Mon Sep 17 00:00:00 2001 From: Sean Date: Wed, 1 Oct 2025 20:51:03 -0400 Subject: [PATCH 1/4] Merge MKDT_0002 --- .../MarketDataLib/Helper/MarketDataHelper.cs | 155 ++++++++++++++---- .../Integration/HttpNetRequest.cs | 84 ++++++++-- 2 files changed, 194 insertions(+), 45 deletions(-) diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs index 7f6b385..cac0b56 100755 --- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs @@ -5240,23 +5240,29 @@ namespace MarketData.Helper return prices; } + /// + /// The main BigChart Feed which is no longer working + /// + /// + /// + /// public static Price GetPriceAsOf(String symbol, DateTime asOf) { - HttpNetResponse httpNetResponse=null; + HttpNetResponse httpNetResponse = null; try { String strRequest; StringBuilder sb = null; if (null == symbol) return null; - CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol); - if(null!=companyProfile && companyProfile.FreezePricing) + CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); + if (null != companyProfile && companyProfile.FreezePricing) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); return null; } - String requestSymbol=symbol; - if(requestSymbol.StartsWith("^"))requestSymbol=requestSymbol.Substring(1); + String requestSymbol = symbol; + if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1); sb = new StringBuilder(); sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb="); sb.Append(requestSymbol); @@ -5265,14 +5271,13 @@ namespace MarketData.Helper sb.Append("%2F"); sb.Append(asOf.Day.ToString()); sb.Append("%2F"); - sb.Append((asOf.Year-2000).ToString()); + sb.Append((asOf.Year - 2000).ToString()); sb.Append("&x=38&y=25"); strRequest = sb.ToString(); - - httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest); - if(!httpNetResponse.Success) + httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest); + if (!httpNetResponse.Success) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); @@ -5283,51 +5288,137 @@ namespace MarketData.Helper if (null == tables || 0 == tables.Count) return null; HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); if (rows.Count < 7) return null; - Price price=new Price(); - price.Source=Price.PriceSource.BigCharts; + Price price = new Price(); + price.Source = Price.PriceSource.BigCharts; price.Symbol = symbol.ToUpper(); - price.Date=FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText); - price.Close=FeedParser.ParseValueFromMarketWatch("Closing Price:",rows[2].InnerText); + price.Date = FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText); + price.Close = FeedParser.ParseValueFromMarketWatch("Closing Price:", rows[2].InnerText); price.AdjClose = price.Close; price.Open = FeedParser.ParseValueFromMarketWatch("Open:", rows[3].InnerText); price.High = FeedParser.ParseValueFromMarketWatch("High:", rows[4].InnerText); price.Low = FeedParser.ParseValueFromMarketWatch("Low:", rows[5].InnerText); price.Volume = FeedParser.ParseLongValueFromMarketWatch("Volume:", rows[6].InnerText); - if(!(price.Date.Date.Equals(asOf.Date.Date))) + if (!(price.Date.Date.Equals(asOf.Date.Date))) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}",symbol,price.Date.ToShortDateString(),asOf.ToShortDateString())); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}", symbol, price.Date.ToShortDateString(), asOf.ToShortDateString())); return null; } return price; } - catch(Exception) + catch (Exception) { return null; } finally { - if(null!=httpNetResponse)httpNetResponse.Dispose(); + if (null != httpNetResponse) httpNetResponse.Dispose(); } } -//******************************************************************************************************************************************************************************************************** -// ******************************************************************************** H I S T O R I C A L P R I C I N G Y A H O O ********************************************************************* -//******************************************************************************************************************************************************************************************************** + - public static Price GetDailyPrice(String symbol,DateTime pricingDate) + + public static Price GetPriceAsOfV2(String symbol, DateTime asOf) { - if(null==symbol)return null; - CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol); - if(null!=companyProfile && companyProfile.FreezePricing) + HttpNetResponse httpNetResponse = null; + try + { + String strRequest; + StringBuilder sb = null; + + if (null == symbol) return null; + CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); + if (null != companyProfile && companyProfile.FreezePricing) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); + return null; + } + String requestSymbol = symbol; + if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1); + sb = new StringBuilder(); + // sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb="); + // sb.Append(requestSymbol); + // sb.Append("&closeDate="); + // sb.Append(asOf.Month.ToString()); + // sb.Append("%2F"); + // sb.Append(asOf.Day.ToString()); + // sb.Append("%2F"); + // sb.Append((asOf.Year - 2000).ToString()); + // sb.Append("&x=38&y=25");0000000000000000000000000000000000000000 + // strRequest = sb.ToString(); + + + + + //https://www.marketwatch.com/investing/stock/TSCDY/download-data + sb.Append("https://www.marketwatch.com/investing/stock/").Append(symbol); + sb.Append("/downloaddatapartial?startdate="); + sb.Append(asOf.Month.ToString()).Append("/").Append(asOf.Day.ToString()).Append("/").Append(asOf.Year.ToString()); + sb.Append("%2000:00:00&enddate="); + sb.Append(asOf.Month.ToString()).Append("/").Append(asOf.Day.ToString()).Append("/").Append(asOf.Year.ToString()); + sb.Append("%2023:59:59&daterange=d30&frequency=p1d&csvdownload=true&downloadpartial=false&newdates=false"); + + + strRequest = sb.ToString(); + + httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest); + if (!httpNetResponse.Success) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); + return null; + } + byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); + MemoryStream memoryStream = new MemoryStream(streamBytes); + HtmlDocument htmlDocument = new HtmlDocument(); + htmlDocument.Load(memoryStream); + HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"historicalquote fatbottomed\"]"); + if (null == tables || 0 == tables.Count) return null; + HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); + if (rows.Count < 7) return null; + Price price = new Price(); + price.Source = Price.PriceSource.BigCharts; + price.Symbol = symbol.ToUpper(); + price.Date = FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText); + price.Close = FeedParser.ParseValueFromMarketWatch("Closing Price:", rows[2].InnerText); + price.AdjClose = price.Close; + price.Open = FeedParser.ParseValueFromMarketWatch("Open:", rows[3].InnerText); + price.High = FeedParser.ParseValueFromMarketWatch("High:", rows[4].InnerText); + price.Low = FeedParser.ParseValueFromMarketWatch("Low:", rows[5].InnerText); + price.Volume = FeedParser.ParseLongValueFromMarketWatch("Volume:", rows[6].InnerText); + if (!(price.Date.Date.Equals(asOf.Date.Date))) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}", symbol, price.Date.ToShortDateString(), asOf.ToShortDateString())); + return null; + } + return price; + } + catch (Exception) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol)); return null; } - Prices prices=GetDailyPrices(symbol,pricingDate,pricingDate); - if(null==prices||0==prices.Count)return null; - Price price=prices.FirstOrDefault(); - if(!price.Date.Date.Equals(pricingDate.Date)) + finally { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetDailyPrice: The pricing date returned for '{0}' was different than the date requested. Date requested '{1}', date returned '{2}'",symbol,pricingDate.ToShortDateString(),price.Date.Date.ToShortDateString())); + if (null != httpNetResponse) httpNetResponse.Dispose(); + } + } +//******************************************************************************************************************************************************************************************************** + // ******************************************************************************** H I S T O R I C A L P R I C I N G Y A H O O ********************************************************************* + //******************************************************************************************************************************************************************************************************** + + public static Price GetDailyPrice(String symbol, DateTime pricingDate) + { + if (null == symbol) return null; + CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); + if (null != companyProfile && companyProfile.FreezePricing) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); + return null; + } + Prices prices = GetDailyPrices(symbol, pricingDate, pricingDate); + if (null == prices || 0 == prices.Count) return null; + Price price = prices.FirstOrDefault(); + if (!price.Date.Date.Equals(pricingDate.Date)) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetDailyPrice: The pricing date returned for '{0}' was different than the date requested. Date requested '{1}', date returned '{2}'", symbol, pricingDate.ToShortDateString(), price.Date.Date.ToShortDateString())); return null; } return price; diff --git a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs index 278a1c8..de7185e 100755 --- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs +++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs @@ -406,24 +406,24 @@ namespace MarketData.Integration } } - public static HttpNetResponse GetRequestCSV(String strRequest,String referer) + public static HttpNetResponse GetRequestNoEncodingV2(String strRequest,WebProxy webProxy=null) { HttpWebResponse webResponse=null; try { - MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestCSV[ENTER]{0}",strRequest)); + ServicePointManager.Expect100Continue = true; + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",strRequest)); int charCount = 0; byte[] buffer = new byte[8192]; StringBuilder sb = new StringBuilder(); HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; webRequest.Timeout = REQUEST_TIMEOUT; - webRequest.Accept="text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + webRequest.Accept = "text/html, text/csv"; webRequest.ContentType="text/csv; charset=utf-8"; - webRequest.Headers.Add("Accept-Encoding: identity"); // "gzip, deflate" - webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); - webRequest.Headers.Add("Upgrade-Insecure-Requests: 1"); - webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0"; - webRequest.Referer=referer; + webRequest.Headers.Add("Accept-Encoding: None"); + webRequest.Headers.Add("Accept-Language: en-US"); + webRequest.UserAgent = "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36"; webRequest.KeepAlive = true; try{webResponse = (HttpWebResponse)webRequest.GetResponse();} catch(WebException webException) @@ -439,7 +439,7 @@ namespace MarketData.Integration while (true) { charCount = responseStream.Read(buffer, 0, buffer.Length); - if (0 == charCount) break; + if (0 >= charCount) break; sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); } return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true); @@ -459,7 +459,64 @@ namespace MarketData.Integration webResponse.Close(); webResponse.Dispose(); } - MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestCSV[LEAVE]"); + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]"); + } + } + + public static HttpNetResponse GetRequestCSV(String strRequest, String referer) + { + HttpWebResponse webResponse = null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestCSV[ENTER]{0}", strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + webRequest.Timeout = REQUEST_TIMEOUT; + webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + webRequest.ContentType = "text/csv; charset=utf-8"; + webRequest.Headers.Add("Accept-Encoding: identity"); // "gzip, deflate" + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Upgrade-Insecure-Requests: 1"); + webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0"; + webRequest.Referer = referer; + webRequest.KeepAlive = true; + try { webResponse = (HttpWebResponse)webRequest.GetResponse(); } + catch (WebException webException) + { + if (IsMovedException(webException)) + { + webRequest = Redirect(webRequest, webException); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + } + else throw; + } + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + return new HttpNetResponse(sb.ToString(), strRequest, webResponse, true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); + } + finally + { + if (null != webResponse) + { + webResponse.Close(); + webResponse.Dispose(); + } + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestCSV[LEAVE]"); } } @@ -475,9 +532,10 @@ namespace MarketData.Integration HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); webRequest.Timeout = REQUEST_TIMEOUT; webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); - webRequest.Headers.Add("Accept-Encoding: None"); - webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0"; - webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; + webRequest.ContentType = "text/csv; charset=utf-8"; + webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br, zstd"); + webRequest.UserAgent = "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36"; + webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"; webRequest.KeepAlive = true; try{webResponse = (HttpWebResponse)webRequest.GetResponse();} catch(WebException webException) From e92a8e230551fb9a2b3c70c6f614ecaddb22d499 Mon Sep 17 00:00:00 2001 From: Sean Date: Wed, 1 Oct 2025 23:13:03 -0400 Subject: [PATCH 2/4] Commit Latest --- .../MarketDataLib/Helper/MarketDataHelper.cs | 82 ++++++++++++++- .../Integration/HttpNetRequest.cs | 99 +++++++++++-------- 2 files changed, 138 insertions(+), 43 deletions(-) diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs index cac0b56..96924cd 100755 --- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs @@ -5343,11 +5343,11 @@ namespace MarketData.Helper // sb.Append(asOf.Day.ToString()); // sb.Append("%2F"); // sb.Append((asOf.Year - 2000).ToString()); - // sb.Append("&x=38&y=25");0000000000000000000000000000000000000000 + // sb.Append("&x=38&y=25"); // strRequest = sb.ToString(); - + String requestUri = "https://www.marketwatch.com/investing/stock"; //https://www.marketwatch.com/investing/stock/TSCDY/download-data sb.Append("https://www.marketwatch.com/investing/stock/").Append(symbol); @@ -5360,7 +5360,11 @@ namespace MarketData.Helper strRequest = sb.ToString(); - httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest); + strRequest = "https://www.marketwatch.com/investing/stock/TSCDY/download-data"; + + CookieCollection cookieCollection = GetCookieCollection("www.marketwatch.com"); + + httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest,cookieCollection,new Uri("https://www.marketwatch.com/investing/stock")); if (!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); @@ -5400,6 +5404,78 @@ namespace MarketData.Helper if (null != httpNetResponse) httpNetResponse.Dispose(); } } + + private static CookieCollection GetCookieCollection(String cookieDomain) + { + String[] cookies = { + "refresh=off;", + "letsGetMikey=enabled;", + "datadome=fzsk~7E54ba21Kkt75NL7RdtwUZrtrG64Uvv9RuT_QixsRLt1Pg90R~N1RaukSzqDP0YqHWcvX166YjBBnpdSaX7vh~Wafia2pGVi77ERaKSz0~ZyRLxznkzn38oztAD;", + "mw_loc=%7B%22Region%22%3A%22FL%22%2C%22Country%22%3A%22US%22%2C%22Continent%22%3A%22NA%22%2C%22ApplicablePrivacy%22%3A0%7D;", + "gdprApplies=false;", + "ab_uuid=e7c9a8ca-7cd9-48f8-818f-7ccaa17ffc4b;", + "fullcss-quote=quote-1e61c76db6.min.css;", + "icons-loaded=true;", + "recentqsmkii=AmericanDepositoryReceiptStock-US-TSCDY|Stock-US-AAPL|Index-US-DJIA;", + "connect.sid=s%3A3XNkudEKQqYfNnQGrDguqwsGtYWTbtWy.3MJrxWzt5ZdjlLen9TChXjvHqpeUW%2FcORbyHcuXO9LQ;", + "_pubcid=92731ab0-a67f-4ca8-8a43-69a724febae8;", + "_pubcid_cst=DCwOLBEsaQ%3D%3D;", + "_lr_env_src_ats=false;", + "_sp_su=true;", + "usnatUUID=99ec3682-498e-4f9e-b269-c046af6d9640;", + "_ncg_domain_id_=ddcf2e8a-9f26-4e50-a630-6f20878eab2a.1.1759284504.1790820504;", + "utag_main=v_id:01999d87268e004df9958911c73005050003f00d00b10$_sn:2$_ss:0$_st:1759349260753$vapi_domain:marketwatch.com$ses_id:1759346895885%3Bexp-session$_pn:10%3Bexp-session$_prevpage:MW_Company%20Download%20Data%3Bexp-1759351060755;", + "AMCV_CB68E4BA55144CAA0A4C98A5%40AdobeOrg=1585540135%7CMCIDTS%7C20363%7CMCMID%7C73618612147539605398398216823576272907%7CMCAID%7CNONE%7CMCOPTOUT-1759354661s%7CNONE%7CvVersion%7C4.4.0;", + "consentUUID=db30bab1-8191-4fdd-9a50-4d1ad172586a;", + "ajs_anonymous_id=9222ad8b-156b-4f06-a002-81c83466cad5;", + "_fbp=fb.1.1759284504405.1136234062;", + "_meta_facebookTag_sync=1759284504405;", + "_meta_googleAdsSegments_library_loaded=1759284504406;", + "AMCVS_CB68E4BA55144CAA0A4C98A5%40AdobeOrg=1;", + "s_ppv=MW_Company%2520Download%2520Data%2C7%2C7%2C150;", + "s_tp=2100;", + "s_cc=true;", + "_dj_id.cff7=.1759284504.2.1759347461.1759284504.bb5714a9-80f8-4f28-9055-84dd571c8508.17332b11-d3d6-4cf3-b12a-0372b78f56c6.97da5143-591d-4be2-a4b7-cf7fd004273e.1759346896124.10;", + "_gcl_au=1.1.870637042.1759284505;", + "_meta_cross_domain_id=8166c525-59e1-4deb-8730-5b7346db4b2d;", + "_meta_cross_domain_recheck=1759284504614;", + "_ga_K2H7B9JRSS=GS2.1.s1759346896$o2$g1$t1759347460$j19$l0$h1325198532;", + "_ga=GA1.1.391315045.1759284505;", + "_dj_sp_id=02b318f5-3f5d-4b6b-9b50-e046bb67bfde;", + "_pctx=%7Bu%7DN4IgrgzgpgThIC4B2YA2qA05owMoBcBDfSREQpAeyRCwgEt8oBJAEzIE4AmHgZi4CsvAIwB2DqIAMADkHTRvEAF8gA;", + "_pcid=%7B%22browserId%22%3A%22mg7ckjvkk2y3p3w6%22%7D;", + "__pat=-14400000;", + "__pvi=eyJpZCI6InYtbWc4ZHB0bjB2a21oOTlraCIsImRvbWFpbiI6Ii5tYXJrZXR3YXRjaC5jb20iLCJ0aW1lIjoxNzU5MzQ3NDYxMjgyfQ%3D%3D;", + "xbc=%7Bkpcd%7DChBtZzdja2p2a2syeTNwM3c2Ego2eWtRMjdJRXB1GjxsRGZ0c1NJZXk5SXB5eEE5U0VyTVl2b2oxVmhIS0tuT3ZMdzV5anJtMWgxZWNwNUhNR3Y3VHdJbTdCaDggAA;", + "_pcus=eyJ1c2VyU2VnbWVudHMiOnsiQ09NUE9TRVIxWCI6eyJzZWdtZW50cyI6WyJMVHM6ODhjNWM5YTg1YjJmNWU4MGViYTZiOWViOWVjYjVmOTM3NDljNzM2OTpub19zY29yZSIsIkxUcmV0dXJuOmY3N2FkMmI5NDZmZTBlOWI1NDRlOWZlMzZiYTk1NGViNmI0ZGNlYWE6bm9fc2NvcmUiLCJDU2NvcmU6YzdiZDdjNTlhZTQ5ZmEwODI0NTc1MjAwOGMzODlhMmIyZDY0MGYyNTpub19zY29yZSJdfX19;", + "_awl=2.1759347461.5-51cc96805a6d7c5c688e4ef811495a68-6763652d75732d6561737431-1;", + "LANG=en_US;", + "LANG_CHANGED=en_US;", + "fullcss-error=section-ee8713325f.min.css;", + "refresh=off;", + "letsGetMikey=enabled;", + "usr_prof_v2=eyJpYyI6Mn0%3D;", + "fullcss-section=section-ee8713325f.min.css;", + "_lr_retry_request=true;", + "__tbc=%7Bkpcd%7DChBtZzdja2p2a2syeTNwM3c2Ego2eWtRMjdJRXB1GjxsRGZ0c1NJZXk5SXB5eEE5U0VyTVl2b2oxVmhIS0tuT3ZMdzV5anJtMWgxZWNwNUhNR3Y3VHdJbTdCaDggAA" + }; + CookieCollection cookieCollection = new CookieCollection(); + for (int index = 0; index < cookies.Count(); index++) + { + String strCookie = cookies[index]; + if (strCookie.EndsWith(";")) strCookie = strCookie.Substring(0, strCookie.Length - 1); + String[] pairs = strCookie.Split('='); + cookieCollection.Add(new Cookie() + { + Name = pairs[0], + Value = pairs[1], + Domain = cookieDomain + }); + + } + return cookieCollection; + } + //******************************************************************************************************************************************************************************************************** // ******************************************************************************** H I S T O R I C A L P R I C I N G Y A H O O ********************************************************************* //******************************************************************************************************************************************************************************************************** diff --git a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs index de7185e..2577830 100755 --- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs +++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs @@ -406,7 +406,7 @@ namespace MarketData.Integration } } - public static HttpNetResponse GetRequestNoEncodingV2(String strRequest,WebProxy webProxy=null) + public static HttpNetResponse GetRequestNoEncodingV2(String strRequest,CookieCollection cookieCollection,Uri uri,WebProxy webProxy=null) { HttpWebResponse webResponse=null; try @@ -424,17 +424,35 @@ namespace MarketData.Integration webRequest.Headers.Add("Accept-Encoding: None"); webRequest.Headers.Add("Accept-Language: en-US"); webRequest.UserAgent = "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36"; - webRequest.KeepAlive = true; - try{webResponse = (HttpWebResponse)webRequest.GetResponse();} - catch(WebException webException) + webRequest.Headers.Add("sec-fetch-dest", "document"); + webRequest.Headers.Add("sec-fetch-mode", "navigate"); + webRequest.Headers.Add("sec-fetch-site","none"); + webRequest.Headers.Add("sec-fetch-user","?1"); + webRequest.Headers.Add("upgrade-insecure-requests","1"); + webRequest.Headers.Add("priority","u=0, i"); + webRequest.AllowAutoRedirect = true; + +// webRequest.KeepAlive = true; + if (null != cookieCollection) { - if(IsMovedException(webException)) + CookieContainer cookieContainer = new CookieContainer(); + foreach (Cookie cookie in cookieCollection) { - webRequest=Redirect(webRequest, webException); - webResponse=(HttpWebResponse)webRequest.GetResponse(); + cookieContainer.Add(uri,cookie); } - else throw; + webRequest.CookieContainer = cookieContainer; } + + try { webResponse = (HttpWebResponse)webRequest.GetResponse(); } + catch (WebException webException) + { + if (IsMovedException(webException)) + { + webRequest = Redirect(webRequest, webException); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + } + else throw; + } Stream responseStream = webResponse.GetResponseStream(); while (true) { @@ -574,62 +592,63 @@ namespace MarketData.Integration MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV2[LEAVE]"); } } + // I am using this code specifically on the seeking alpha web site. it seems as though seeking alpha has implemeted some user agent based bot prevention mechanism on their website // to prevent scrapers. What I do here is to choose from a set of commonly used user agents and then randomly choose a user agent to code into the request. retry logic up to 3 times // and handling of 404 (not found) - public static HttpNetResponse GetRequestNoEncodingV3(String strRequest,String referer=null) + public static HttpNetResponse GetRequestNoEncodingV3(String strRequest, String referer = null) { - HttpWebResponse webResponse=null; - WebException lastWebException=null; - + HttpWebResponse webResponse = null; + WebException lastWebException = null; + try { - MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3[ENTER]{0}",strRequest)); - Random random=new Random(); - String[] userAgents= + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV3[ENTER]{0}", strRequest)); + Random random = new Random(); + String[] userAgents = { "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537." }; - int MAX_RETRIES=5; - int TIMEOUT_BETWEEN_RETRIES=1000; + int MAX_RETRIES = 5; + int TIMEOUT_BETWEEN_RETRIES = 1000; int charCount = 0; byte[] buffer = new byte[8192]; StringBuilder sb = new StringBuilder(); - for(int count=0;count Date: Thu, 2 Oct 2025 13:45:38 -0400 Subject: [PATCH 3/4] Commit Latest --- .../MarketDataLib/Helper/MarketDataHelper.cs | 15 ++++++++------- .../MarketDataLib/Integration/HttpNetRequest.cs | 3 ++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs index 96924cd..3b904ae 100755 --- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs @@ -5364,7 +5364,8 @@ namespace MarketData.Helper CookieCollection cookieCollection = GetCookieCollection("www.marketwatch.com"); - httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest,cookieCollection,new Uri("https://www.marketwatch.com/investing/stock")); +// httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest,cookieCollection,new Uri("https://www.marketwatch.com/investing/stock")); + httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest,cookieCollection,new Uri("https://www.marketwatch.com/investing")); if (!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); @@ -5452,12 +5453,12 @@ namespace MarketData.Helper "LANG=en_US;", "LANG_CHANGED=en_US;", "fullcss-error=section-ee8713325f.min.css;", - "refresh=off;", - "letsGetMikey=enabled;", - "usr_prof_v2=eyJpYyI6Mn0%3D;", - "fullcss-section=section-ee8713325f.min.css;", - "_lr_retry_request=true;", - "__tbc=%7Bkpcd%7DChBtZzdja2p2a2syeTNwM3c2Ego2eWtRMjdJRXB1GjxsRGZ0c1NJZXk5SXB5eEE5U0VyTVl2b2oxVmhIS0tuT3ZMdzV5anJtMWgxZWNwNUhNR3Y3VHdJbTdCaDggAA" + // "refresh=off;", + // "letsGetMikey=enabled;", + // "usr_prof_v2=eyJpYyI6Mn0%3D;", + // "fullcss-section=section-ee8713325f.min.css;", + // "_lr_retry_request=true;", + // "__tbc=%7Bkpcd%7DChBtZzdja2p2a2syeTNwM3c2Ego2eWtRMjdJRXB1GjxsRGZ0c1NJZXk5SXB5eEE5U0VyTVl2b2oxVmhIS0tuT3ZMdzV5anJtMWgxZWNwNUhNR3Y3VHdJbTdCaDggAA" }; CookieCollection cookieCollection = new CookieCollection(); for (int index = 0; index < cookies.Count(); index++) diff --git a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs index 2577830..c2f4197 100755 --- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs +++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs @@ -421,7 +421,7 @@ namespace MarketData.Integration webRequest.Timeout = REQUEST_TIMEOUT; webRequest.Accept = "text/html, text/csv"; webRequest.ContentType="text/csv; charset=utf-8"; - webRequest.Headers.Add("Accept-Encoding: None"); + webRequest.Headers.Add("Accept-Encoding: identity"); webRequest.Headers.Add("Accept-Language: en-US"); webRequest.UserAgent = "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36"; webRequest.Headers.Add("sec-fetch-dest", "document"); @@ -430,6 +430,7 @@ namespace MarketData.Integration webRequest.Headers.Add("sec-fetch-user","?1"); webRequest.Headers.Add("upgrade-insecure-requests","1"); webRequest.Headers.Add("priority","u=0, i"); + webRequest.Headers.Add("Referrer-Policy", "strict-origin-when-cross-origin"); webRequest.AllowAutoRedirect = true; // webRequest.KeepAlive = true; From 1b149411a4240e5731d286f6cd89b3c4c52d07c8 Mon Sep 17 00:00:00 2001 From: Sean Date: Fri, 10 Oct 2025 23:23:40 -0400 Subject: [PATCH 4/4] Fix for BigCharts Feed --- .../MarketDataLib/Helper/MarketDataHelper.cs | 127 ++++++------------ .../Integration/HttpNetRequest.cs | 99 +++++++++----- 2 files changed, 102 insertions(+), 124 deletions(-) diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs index 3b904ae..4536f2d 100755 --- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs @@ -5314,9 +5314,15 @@ namespace MarketData.Helper if (null != httpNetResponse) httpNetResponse.Dispose(); } } - - + /// + /// This is a modified version of the above query. It uses a cookie collection that contains the datadome cookie. + /// If this query starts to fail then you should try the query in developer tools in the chromium browser and capture the + /// datadome value and update it here in the GetCookieCollection() below. + /// + /// + /// + /// public static Price GetPriceAsOfV2(String symbol, DateTime asOf) { HttpNetResponse httpNetResponse = null; @@ -5335,36 +5341,21 @@ namespace MarketData.Helper String requestSymbol = symbol; if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1); sb = new StringBuilder(); - // sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb="); - // sb.Append(requestSymbol); - // sb.Append("&closeDate="); - // sb.Append(asOf.Month.ToString()); - // sb.Append("%2F"); - // sb.Append(asOf.Day.ToString()); - // sb.Append("%2F"); - // sb.Append((asOf.Year - 2000).ToString()); - // sb.Append("&x=38&y=25"); - // strRequest = sb.ToString(); - - - String requestUri = "https://www.marketwatch.com/investing/stock"; - - //https://www.marketwatch.com/investing/stock/TSCDY/download-data - sb.Append("https://www.marketwatch.com/investing/stock/").Append(symbol); - sb.Append("/downloaddatapartial?startdate="); - sb.Append(asOf.Month.ToString()).Append("/").Append(asOf.Day.ToString()).Append("/").Append(asOf.Year.ToString()); - sb.Append("%2000:00:00&enddate="); - sb.Append(asOf.Month.ToString()).Append("/").Append(asOf.Day.ToString()).Append("/").Append(asOf.Year.ToString()); - sb.Append("%2023:59:59&daterange=d30&frequency=p1d&csvdownload=true&downloadpartial=false&newdates=false"); - - + sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb="); + sb.Append(requestSymbol); + sb.Append("&closeDate="); + sb.Append(asOf.Month.ToString()); + sb.Append("%2F"); + sb.Append(asOf.Day.ToString()); + sb.Append("%2F"); + sb.Append((asOf.Year - 2000).ToString()); + sb.Append("&x=38&y=25"); strRequest = sb.ToString(); - strRequest = "https://www.marketwatch.com/investing/stock/TSCDY/download-data"; + MDTrace.WriteLine(LogLevel.DEBUG,$"{strRequest}"); CookieCollection cookieCollection = GetCookieCollection("www.marketwatch.com"); -// httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest,cookieCollection,new Uri("https://www.marketwatch.com/investing/stock")); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest,cookieCollection,new Uri("https://www.marketwatch.com/investing")); if (!httpNetResponse.Success) { @@ -5375,20 +5366,29 @@ namespace MarketData.Helper MemoryStream memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); - HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"historicalquote fatbottomed\"]"); + + HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"table table--overflow align--center\"]"); + if (null == tables || 0 == tables.Count) return null; HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); - if (rows.Count < 7) return null; + if (rows.Count < 2) return null; + HtmlNodeCollection data = rows[1].SelectNodes(".//td"); + if (data.Count != 6) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the correct number of data elements (expected 6 data elements).", symbol)); + return null; + } Price price = new Price(); price.Source = Price.PriceSource.BigCharts; price.Symbol = symbol.ToUpper(); - price.Date = FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText); - price.Close = FeedParser.ParseValueFromMarketWatch("Closing Price:", rows[2].InnerText); - price.AdjClose = price.Close; - price.Open = FeedParser.ParseValueFromMarketWatch("Open:", rows[3].InnerText); - price.High = FeedParser.ParseValueFromMarketWatch("High:", rows[4].InnerText); - price.Low = FeedParser.ParseValueFromMarketWatch("Low:", rows[5].InnerText); - price.Volume = FeedParser.ParseLongValueFromMarketWatch("Volume:", rows[6].InnerText); + String[] dateElements = data[0].InnerText.Trim().Split(" "); + + price.Date = Utility.ParseDate(dateElements[0].Replace("\n",null)); + price.Open = Utility.ParseCurrency(data[1].InnerText.Trim().Replace("\n",null)); + price.High = Utility.ParseCurrency(data[2].InnerText.Trim().Replace("\n",null)); + price.Low = Utility.ParseCurrency(data[3].InnerText.Trim().Replace("\n",null)); + price.Close = price.AdjClose = Utility.ParseCurrency(data[4].InnerText.Trim().Replace("\n",null)); + price.Volume = FeedParser.ParseValueLong(data[5].InnerText.Trim()); if (!(price.Date.Date.Equals(asOf.Date.Date))) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}", symbol, price.Date.ToShortDateString(), asOf.ToShortDateString())); @@ -5396,8 +5396,9 @@ namespace MarketData.Helper } return price; } - catch (Exception) + catch (Exception exception) { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception : {0}", exception.ToString())); return null; } finally @@ -5409,56 +5410,7 @@ namespace MarketData.Helper private static CookieCollection GetCookieCollection(String cookieDomain) { String[] cookies = { - "refresh=off;", - "letsGetMikey=enabled;", - "datadome=fzsk~7E54ba21Kkt75NL7RdtwUZrtrG64Uvv9RuT_QixsRLt1Pg90R~N1RaukSzqDP0YqHWcvX166YjBBnpdSaX7vh~Wafia2pGVi77ERaKSz0~ZyRLxznkzn38oztAD;", - "mw_loc=%7B%22Region%22%3A%22FL%22%2C%22Country%22%3A%22US%22%2C%22Continent%22%3A%22NA%22%2C%22ApplicablePrivacy%22%3A0%7D;", - "gdprApplies=false;", - "ab_uuid=e7c9a8ca-7cd9-48f8-818f-7ccaa17ffc4b;", - "fullcss-quote=quote-1e61c76db6.min.css;", - "icons-loaded=true;", - "recentqsmkii=AmericanDepositoryReceiptStock-US-TSCDY|Stock-US-AAPL|Index-US-DJIA;", - "connect.sid=s%3A3XNkudEKQqYfNnQGrDguqwsGtYWTbtWy.3MJrxWzt5ZdjlLen9TChXjvHqpeUW%2FcORbyHcuXO9LQ;", - "_pubcid=92731ab0-a67f-4ca8-8a43-69a724febae8;", - "_pubcid_cst=DCwOLBEsaQ%3D%3D;", - "_lr_env_src_ats=false;", - "_sp_su=true;", - "usnatUUID=99ec3682-498e-4f9e-b269-c046af6d9640;", - "_ncg_domain_id_=ddcf2e8a-9f26-4e50-a630-6f20878eab2a.1.1759284504.1790820504;", - "utag_main=v_id:01999d87268e004df9958911c73005050003f00d00b10$_sn:2$_ss:0$_st:1759349260753$vapi_domain:marketwatch.com$ses_id:1759346895885%3Bexp-session$_pn:10%3Bexp-session$_prevpage:MW_Company%20Download%20Data%3Bexp-1759351060755;", - "AMCV_CB68E4BA55144CAA0A4C98A5%40AdobeOrg=1585540135%7CMCIDTS%7C20363%7CMCMID%7C73618612147539605398398216823576272907%7CMCAID%7CNONE%7CMCOPTOUT-1759354661s%7CNONE%7CvVersion%7C4.4.0;", - "consentUUID=db30bab1-8191-4fdd-9a50-4d1ad172586a;", - "ajs_anonymous_id=9222ad8b-156b-4f06-a002-81c83466cad5;", - "_fbp=fb.1.1759284504405.1136234062;", - "_meta_facebookTag_sync=1759284504405;", - "_meta_googleAdsSegments_library_loaded=1759284504406;", - "AMCVS_CB68E4BA55144CAA0A4C98A5%40AdobeOrg=1;", - "s_ppv=MW_Company%2520Download%2520Data%2C7%2C7%2C150;", - "s_tp=2100;", - "s_cc=true;", - "_dj_id.cff7=.1759284504.2.1759347461.1759284504.bb5714a9-80f8-4f28-9055-84dd571c8508.17332b11-d3d6-4cf3-b12a-0372b78f56c6.97da5143-591d-4be2-a4b7-cf7fd004273e.1759346896124.10;", - "_gcl_au=1.1.870637042.1759284505;", - "_meta_cross_domain_id=8166c525-59e1-4deb-8730-5b7346db4b2d;", - "_meta_cross_domain_recheck=1759284504614;", - "_ga_K2H7B9JRSS=GS2.1.s1759346896$o2$g1$t1759347460$j19$l0$h1325198532;", - "_ga=GA1.1.391315045.1759284505;", - "_dj_sp_id=02b318f5-3f5d-4b6b-9b50-e046bb67bfde;", - "_pctx=%7Bu%7DN4IgrgzgpgThIC4B2YA2qA05owMoBcBDfSREQpAeyRCwgEt8oBJAEzIE4AmHgZi4CsvAIwB2DqIAMADkHTRvEAF8gA;", - "_pcid=%7B%22browserId%22%3A%22mg7ckjvkk2y3p3w6%22%7D;", - "__pat=-14400000;", - "__pvi=eyJpZCI6InYtbWc4ZHB0bjB2a21oOTlraCIsImRvbWFpbiI6Ii5tYXJrZXR3YXRjaC5jb20iLCJ0aW1lIjoxNzU5MzQ3NDYxMjgyfQ%3D%3D;", - "xbc=%7Bkpcd%7DChBtZzdja2p2a2syeTNwM3c2Ego2eWtRMjdJRXB1GjxsRGZ0c1NJZXk5SXB5eEE5U0VyTVl2b2oxVmhIS0tuT3ZMdzV5anJtMWgxZWNwNUhNR3Y3VHdJbTdCaDggAA;", - "_pcus=eyJ1c2VyU2VnbWVudHMiOnsiQ09NUE9TRVIxWCI6eyJzZWdtZW50cyI6WyJMVHM6ODhjNWM5YTg1YjJmNWU4MGViYTZiOWViOWVjYjVmOTM3NDljNzM2OTpub19zY29yZSIsIkxUcmV0dXJuOmY3N2FkMmI5NDZmZTBlOWI1NDRlOWZlMzZiYTk1NGViNmI0ZGNlYWE6bm9fc2NvcmUiLCJDU2NvcmU6YzdiZDdjNTlhZTQ5ZmEwODI0NTc1MjAwOGMzODlhMmIyZDY0MGYyNTpub19zY29yZSJdfX19;", - "_awl=2.1759347461.5-51cc96805a6d7c5c688e4ef811495a68-6763652d75732d6561737431-1;", - "LANG=en_US;", - "LANG_CHANGED=en_US;", - "fullcss-error=section-ee8713325f.min.css;", - // "refresh=off;", - // "letsGetMikey=enabled;", - // "usr_prof_v2=eyJpYyI6Mn0%3D;", - // "fullcss-section=section-ee8713325f.min.css;", - // "_lr_retry_request=true;", - // "__tbc=%7Bkpcd%7DChBtZzdja2p2a2syeTNwM3c2Ego2eWtRMjdJRXB1GjxsRGZ0c1NJZXk5SXB5eEE5U0VyTVl2b2oxVmhIS0tuT3ZMdzV5anJtMWgxZWNwNUhNR3Y3VHdJbTdCaDggAA" + "datadome=VLSWQ1PWL4SV2rljkJEV7GUOK3T11Sm73RP17IwDEtRzfWHykYwy7ZoeSPrjwmbwvGRAenazmLVTpyCB6Yqlw6vUL6Wbqq4zML5DGuHLxNU4LgAQ7Ko9tztglqjIGLZE", }; CookieCollection cookieCollection = new CookieCollection(); for (int index = 0; index < cookies.Count(); index++) @@ -5472,7 +5424,6 @@ namespace MarketData.Helper Value = pairs[1], Domain = cookieDomain }); - } return cookieCollection; } diff --git a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs index c2f4197..74d9ad5 100755 --- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs +++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs @@ -406,79 +406,106 @@ namespace MarketData.Integration } } - public static HttpNetResponse GetRequestNoEncodingV2(String strRequest,CookieCollection cookieCollection,Uri uri,WebProxy webProxy=null) + /// + /// This one is being used in the MarketWatch price fetch + /// + /// + /// + /// + /// + /// + public static HttpNetResponse GetRequestNoEncodingV2(String strRequest, CookieCollection cookieCollection, Uri uri, WebProxy webProxy = null) { - HttpWebResponse webResponse=null; + HttpWebResponse webResponse = null; try { - ServicePointManager.Expect100Continue = true; - MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",strRequest)); + ServicePointManager.Expect100Continue = true; + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV2[ENTER]{0}", strRequest)); int charCount = 0; byte[] buffer = new byte[8192]; StringBuilder sb = new StringBuilder(); HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); - if(null!=webProxy)webRequest.Proxy=webProxy; + if (null != webProxy) webRequest.Proxy = webProxy; webRequest.Timeout = REQUEST_TIMEOUT; - webRequest.Accept = "text/html, text/csv"; - webRequest.ContentType="text/csv; charset=utf-8"; - webRequest.Headers.Add("Accept-Encoding: identity"); - webRequest.Headers.Add("Accept-Language: en-US"); + webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"; + webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br, zstd"); + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.9"); webRequest.UserAgent = "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36"; - webRequest.Headers.Add("sec-fetch-dest", "document"); - webRequest.Headers.Add("sec-fetch-mode", "navigate"); - webRequest.Headers.Add("sec-fetch-site","none"); - webRequest.Headers.Add("sec-fetch-user","?1"); - webRequest.Headers.Add("upgrade-insecure-requests","1"); - webRequest.Headers.Add("priority","u=0, i"); + webRequest.Headers.Add("sec-fetch-dest", "document"); + webRequest.Headers.Add("sec-fetch-mode", "navigate"); + webRequest.Headers.Add("sec-fetch-site", "none"); + webRequest.Headers.Add("sec-fetch-user", "?1"); + webRequest.Headers.Add("upgrade-insecure-requests", "1"); + + webRequest.Headers.Add("priority", "u=0, i"); + webRequest.Headers.Add("sec-ch-device-memory", "8"); + webRequest.Headers.Add("Referrer-Policy", "strict-origin-when-cross-origin"); webRequest.AllowAutoRedirect = true; -// webRequest.KeepAlive = true; if (null != cookieCollection) { CookieContainer cookieContainer = new CookieContainer(); foreach (Cookie cookie in cookieCollection) { - cookieContainer.Add(uri,cookie); + cookieContainer.Add(uri, cookie); } webRequest.CookieContainer = cookieContainer; } try { webResponse = (HttpWebResponse)webRequest.GetResponse(); } - catch (WebException webException) - { - if (IsMovedException(webException)) - { - webRequest = Redirect(webRequest, webException); - webResponse = (HttpWebResponse)webRequest.GetResponse(); - } - else throw; - } - Stream responseStream = webResponse.GetResponseStream(); - while (true) + catch (WebException webException) { - charCount = responseStream.Read(buffer, 0, buffer.Length); - if (0 >= charCount) break; - sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + if (IsMovedException(webException)) + { + webRequest = Redirect(webRequest, webException); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + } + else throw; } - return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true); + Stream responseStream = webResponse.GetResponseStream(); + + if (webResponse.ContentEncoding.ToLower().Contains("gzip")) + { + responseStream = new GZipStream(responseStream, CompressionMode.Decompress); + StreamReader reader = new StreamReader(responseStream, Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else if (webResponse.ContentEncoding.ToLower().Contains("deflate")) + { + responseStream = new DeflateStream(responseStream, CompressionMode.Decompress); + StreamReader reader = new StreamReader(responseStream, Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else + { + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + } + return new HttpNetResponse(sb.ToString(), strRequest, webResponse, true); } catch (WebException webException) { - return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message); + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); } catch (Exception exception) { - return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); } finally { - if(null!=webResponse) + if (null != webResponse) { webResponse.Close(); webResponse.Dispose(); } - MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]"); + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncoding[LEAVE]"); } }