Fix for BigCharts Feed

2025-10-10 23:23:40 -04:00
parent ee7bd07eba
commit 1b149411a4
2 changed files with 102 additions and 124 deletions
--- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs
+++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs
@@ -5314,9 +5314,15 @@ namespace MarketData.Helper
        if (null != httpNetResponse) httpNetResponse.Dispose();
      }
    }
-    
-

+    /// <summary>
+    /// This is a modified version of the above query.  It uses a cookie collection that contains the datadome cookie.
+    /// If this query starts to fail then you should try the query in developer tools in the chromium browser and capture the
+    /// datadome value and update it here in the GetCookieCollection() below.
+    /// </summary>
+    /// <param name="symbol"></param>
+    /// <param name="asOf"></param>
+    /// <returns></returns>
    public static Price GetPriceAsOfV2(String symbol, DateTime asOf)
    {
      HttpNetResponse httpNetResponse = null;
@@ -5335,36 +5341,21 @@ namespace MarketData.Helper
        String requestSymbol = symbol;
        if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1);
        sb = new StringBuilder();
-        // sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb=");
-        // sb.Append(requestSymbol);
-        // sb.Append("&closeDate=");
-        // sb.Append(asOf.Month.ToString());
-        // sb.Append("%2F");
-        // sb.Append(asOf.Day.ToString());
-        // sb.Append("%2F");
-        // sb.Append((asOf.Year - 2000).ToString());
-        // sb.Append("&x=38&y=25");
-        // strRequest = sb.ToString();
-
-
-        String requestUri = "https://www.marketwatch.com/investing/stock";
-
-        //https://www.marketwatch.com/investing/stock/TSCDY/download-data
-        sb.Append("https://www.marketwatch.com/investing/stock/").Append(symbol);
-        sb.Append("/downloaddatapartial?startdate=");
-        sb.Append(asOf.Month.ToString()).Append("/").Append(asOf.Day.ToString()).Append("/").Append(asOf.Year.ToString());
-        sb.Append("%2000:00:00&enddate=");
-        sb.Append(asOf.Month.ToString()).Append("/").Append(asOf.Day.ToString()).Append("/").Append(asOf.Year.ToString());
-        sb.Append("%2023:59:59&daterange=d30&frequency=p1d&csvdownload=true&downloadpartial=false&newdates=false");
-
-
+        sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb=");
+        sb.Append(requestSymbol);
+        sb.Append("&closeDate=");
+        sb.Append(asOf.Month.ToString());
+        sb.Append("%2F");
+        sb.Append(asOf.Day.ToString());
+        sb.Append("%2F");
+        sb.Append((asOf.Year - 2000).ToString());
+        sb.Append("&x=38&y=25");
        strRequest = sb.ToString();

-        strRequest = "https://www.marketwatch.com/investing/stock/TSCDY/download-data";
+        MDTrace.WriteLine(LogLevel.DEBUG,$"{strRequest}");

        CookieCollection cookieCollection = GetCookieCollection("www.marketwatch.com");

-//        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest,cookieCollection,new Uri("https://www.marketwatch.com/investing/stock"));
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest,cookieCollection,new Uri("https://www.marketwatch.com/investing"));        
        if (!httpNetResponse.Success)
        {
@@ -5375,20 +5366,29 @@ namespace MarketData.Helper
        MemoryStream memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
-        HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"historicalquote fatbottomed\"]");
+
+        HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"table table--overflow align--center\"]");
+
        if (null == tables || 0 == tables.Count) return null;
        HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
-        if (rows.Count < 7) return null;
+        if (rows.Count < 2) return null;
+        HtmlNodeCollection data = rows[1].SelectNodes(".//td");
+        if (data.Count != 6)
+        {
+          MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the correct number of data elements (expected 6 data elements).", symbol));
+          return null;
+        }
        Price price = new Price();
        price.Source = Price.PriceSource.BigCharts;
        price.Symbol = symbol.ToUpper();
-        price.Date = FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText);
-        price.Close = FeedParser.ParseValueFromMarketWatch("Closing Price:", rows[2].InnerText);
-        price.AdjClose = price.Close;
-        price.Open = FeedParser.ParseValueFromMarketWatch("Open:", rows[3].InnerText);
-        price.High = FeedParser.ParseValueFromMarketWatch("High:", rows[4].InnerText);
-        price.Low = FeedParser.ParseValueFromMarketWatch("Low:", rows[5].InnerText);
-        price.Volume = FeedParser.ParseLongValueFromMarketWatch("Volume:", rows[6].InnerText);
+        String[] dateElements = data[0].InnerText.Trim().Split(" ");
+
+        price.Date = Utility.ParseDate(dateElements[0].Replace("\n",null));
+        price.Open = Utility.ParseCurrency(data[1].InnerText.Trim().Replace("\n",null));
+        price.High = Utility.ParseCurrency(data[2].InnerText.Trim().Replace("\n",null));
+        price.Low = Utility.ParseCurrency(data[3].InnerText.Trim().Replace("\n",null));
+        price.Close = price.AdjClose = Utility.ParseCurrency(data[4].InnerText.Trim().Replace("\n",null));
+        price.Volume = FeedParser.ParseValueLong(data[5].InnerText.Trim());
        if (!(price.Date.Date.Equals(asOf.Date.Date)))
        {
          MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the requested date.  Requested date {1} Retrieved date {2}", symbol, price.Date.ToShortDateString(), asOf.ToShortDateString()));
@@ -5396,8 +5396,9 @@ namespace MarketData.Helper
        }
        return price;
      }
-      catch (Exception)
+      catch (Exception exception)
      {
+        MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception : {0}", exception.ToString()));
        return null;
      }
      finally
@@ -5409,56 +5410,7 @@ namespace MarketData.Helper
    private static CookieCollection GetCookieCollection(String cookieDomain)
    {
      String[] cookies = {
-      "refresh=off;",
-      "letsGetMikey=enabled;",
-      "datadome=fzsk~7E54ba21Kkt75NL7RdtwUZrtrG64Uvv9RuT_QixsRLt1Pg90R~N1RaukSzqDP0YqHWcvX166YjBBnpdSaX7vh~Wafia2pGVi77ERaKSz0~ZyRLxznkzn38oztAD;",
-      "mw_loc=%7B%22Region%22%3A%22FL%22%2C%22Country%22%3A%22US%22%2C%22Continent%22%3A%22NA%22%2C%22ApplicablePrivacy%22%3A0%7D;",
-      "gdprApplies=false;",
-      "ab_uuid=e7c9a8ca-7cd9-48f8-818f-7ccaa17ffc4b;",
-      "fullcss-quote=quote-1e61c76db6.min.css;",
-      "icons-loaded=true;",
-      "recentqsmkii=AmericanDepositoryReceiptStock-US-TSCDY|Stock-US-AAPL|Index-US-DJIA;",
-      "connect.sid=s%3A3XNkudEKQqYfNnQGrDguqwsGtYWTbtWy.3MJrxWzt5ZdjlLen9TChXjvHqpeUW%2FcORbyHcuXO9LQ;",
-      "_pubcid=92731ab0-a67f-4ca8-8a43-69a724febae8;",
-      "_pubcid_cst=DCwOLBEsaQ%3D%3D;",
-      "_lr_env_src_ats=false;",
-      "_sp_su=true;",
-      "usnatUUID=99ec3682-498e-4f9e-b269-c046af6d9640;",
-      "_ncg_domain_id_=ddcf2e8a-9f26-4e50-a630-6f20878eab2a.1.1759284504.1790820504;",
-      "utag_main=v_id:01999d87268e004df9958911c73005050003f00d00b10$_sn:2$_ss:0$_st:1759349260753$vapi_domain:marketwatch.com$ses_id:1759346895885%3Bexp-session$_pn:10%3Bexp-session$_prevpage:MW_Company%20Download%20Data%3Bexp-1759351060755;",
-      "AMCV_CB68E4BA55144CAA0A4C98A5%40AdobeOrg=1585540135%7CMCIDTS%7C20363%7CMCMID%7C73618612147539605398398216823576272907%7CMCAID%7CNONE%7CMCOPTOUT-1759354661s%7CNONE%7CvVersion%7C4.4.0;",
-      "consentUUID=db30bab1-8191-4fdd-9a50-4d1ad172586a;",
-      "ajs_anonymous_id=9222ad8b-156b-4f06-a002-81c83466cad5;",
-      "_fbp=fb.1.1759284504405.1136234062;",
-      "_meta_facebookTag_sync=1759284504405;",
-      "_meta_googleAdsSegments_library_loaded=1759284504406;",
-      "AMCVS_CB68E4BA55144CAA0A4C98A5%40AdobeOrg=1;",
-      "s_ppv=MW_Company%2520Download%2520Data%2C7%2C7%2C150;",
-      "s_tp=2100;",
-      "s_cc=true;",
-      "_dj_id.cff7=.1759284504.2.1759347461.1759284504.bb5714a9-80f8-4f28-9055-84dd571c8508.17332b11-d3d6-4cf3-b12a-0372b78f56c6.97da5143-591d-4be2-a4b7-cf7fd004273e.1759346896124.10;",
-      "_gcl_au=1.1.870637042.1759284505;",
-      "_meta_cross_domain_id=8166c525-59e1-4deb-8730-5b7346db4b2d;",
-      "_meta_cross_domain_recheck=1759284504614;",
-      "_ga_K2H7B9JRSS=GS2.1.s1759346896$o2$g1$t1759347460$j19$l0$h1325198532;",
-      "_ga=GA1.1.391315045.1759284505;",
-      "_dj_sp_id=02b318f5-3f5d-4b6b-9b50-e046bb67bfde;",
-      "_pctx=%7Bu%7DN4IgrgzgpgThIC4B2YA2qA05owMoBcBDfSREQpAeyRCwgEt8oBJAEzIE4AmHgZi4CsvAIwB2DqIAMADkHTRvEAF8gA;",
-      "_pcid=%7B%22browserId%22%3A%22mg7ckjvkk2y3p3w6%22%7D;",
-      "__pat=-14400000;",
-      "__pvi=eyJpZCI6InYtbWc4ZHB0bjB2a21oOTlraCIsImRvbWFpbiI6Ii5tYXJrZXR3YXRjaC5jb20iLCJ0aW1lIjoxNzU5MzQ3NDYxMjgyfQ%3D%3D;",
-      "xbc=%7Bkpcd%7DChBtZzdja2p2a2syeTNwM3c2Ego2eWtRMjdJRXB1GjxsRGZ0c1NJZXk5SXB5eEE5U0VyTVl2b2oxVmhIS0tuT3ZMdzV5anJtMWgxZWNwNUhNR3Y3VHdJbTdCaDggAA;",
-      "_pcus=eyJ1c2VyU2VnbWVudHMiOnsiQ09NUE9TRVIxWCI6eyJzZWdtZW50cyI6WyJMVHM6ODhjNWM5YTg1YjJmNWU4MGViYTZiOWViOWVjYjVmOTM3NDljNzM2OTpub19zY29yZSIsIkxUcmV0dXJuOmY3N2FkMmI5NDZmZTBlOWI1NDRlOWZlMzZiYTk1NGViNmI0ZGNlYWE6bm9fc2NvcmUiLCJDU2NvcmU6YzdiZDdjNTlhZTQ5ZmEwODI0NTc1MjAwOGMzODlhMmIyZDY0MGYyNTpub19zY29yZSJdfX19;",
-      "_awl=2.1759347461.5-51cc96805a6d7c5c688e4ef811495a68-6763652d75732d6561737431-1;",
-      "LANG=en_US;",
-      "LANG_CHANGED=en_US;",
-      "fullcss-error=section-ee8713325f.min.css;",
-      // "refresh=off;",
-      // "letsGetMikey=enabled;",
-      // "usr_prof_v2=eyJpYyI6Mn0%3D;",
-      // "fullcss-section=section-ee8713325f.min.css;",
-      // "_lr_retry_request=true;",
-      // "__tbc=%7Bkpcd%7DChBtZzdja2p2a2syeTNwM3c2Ego2eWtRMjdJRXB1GjxsRGZ0c1NJZXk5SXB5eEE5U0VyTVl2b2oxVmhIS0tuT3ZMdzV5anJtMWgxZWNwNUhNR3Y3VHdJbTdCaDggAA"
+      "datadome=VLSWQ1PWL4SV2rljkJEV7GUOK3T11Sm73RP17IwDEtRzfWHykYwy7ZoeSPrjwmbwvGRAenazmLVTpyCB6Yqlw6vUL6Wbqq4zML5DGuHLxNU4LgAQ7Ko9tztglqjIGLZE",
      };
      CookieCollection cookieCollection = new CookieCollection();
      for (int index = 0; index < cookies.Count(); index++)
@@ -5472,7 +5424,6 @@ namespace MarketData.Helper
          Value = pairs[1],
          Domain = cookieDomain
        });
-
      }
      return cookieCollection;
    }