Merge MKDT_0004

MarketData/MarketDataLib/Integration/HttpNetRequest.cs

@@ -1523,6 +1523,81 @@ namespace MarketData.Integration
       }
     }
 
+    /// <summary>
+    /// GetRequestV6 - Uses HttpClient to more closely match web browser.
+    /// </summary>
+    /// <param name="url"></param>
+    /// <param name="timeoutMS"></param>
+    /// <param name="proxy"></param>
+    /// <param name="cookieHeader"></param>
+    /// <returns></returns>
+    public static HttpNetResponse GetRequestV6(string url,int timeoutMS,WebProxy proxy = null,string cookieHeader = null)
+    {
+      HttpResponseMessage response = null;
+      try
+      {
+        MDTrace.WriteLine(LogLevel.VERBOSE, $"GetRequestV6[ENTER]{url}");
+        var handler = new HttpClientHandler()
+        {
+          AllowAutoRedirect = true,
+          AutomaticDecompression =
+              DecompressionMethods.GZip |
+              DecompressionMethods.Deflate |
+              DecompressionMethods.Brotli,
+          UseCookies = true,
+          CookieContainer = new CookieContainer()
+        };
+        if (proxy != null)
+        {
+          handler.Proxy = proxy;
+          handler.UseProxy = true;
+        }
+        using (HttpClient client = new HttpClient(handler))
+        {
+          client.Timeout = TimeSpan.FromMilliseconds(timeoutMS);
+          // Core headers
+          client.DefaultRequestHeaders.TryAddWithoutValidation(
+              "User-Agent",
+              "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0");
+          client.DefaultRequestHeaders.TryAddWithoutValidation(
+              "Accept",
+              "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
+          client.DefaultRequestHeaders.TryAddWithoutValidation(
+              "Accept-Language",
+              "en-US,en;q=0.9");
+          client.DefaultRequestHeaders.TryAddWithoutValidation(
+              "Accept-Encoding",
+              "gzip, deflate, br, zstd");
+
+          client.DefaultRequestHeaders.Connection.Add("keep-alive");
+          // Browser-like headers
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests", "1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest", "document");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode", "navigate");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site", "none");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User", "?1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC", "1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Priority", "u=0, i");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("TE", "trailers");
+          if (!string.IsNullOrEmpty(cookieHeader))
+          {
+            client.DefaultRequestHeaders.TryAddWithoutValidation("Cookie", cookieHeader);
+          }
+          response = client.GetAsync(url).GetAwaiter().GetResult();
+          string html = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+          return new HttpNetResponse(html,url,null,handler.CookieContainer.GetCookies(new Uri(url)),response.IsSuccessStatusCode);
+        }
+      }
+      catch (Exception ex)
+      {
+        return new HttpNetResponse(null,url,false,ex.Message);
+      }
+      finally
+      {
+        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestV6[LEAVE]");
+      }
+    }
+
     private static HttpNetResponse ProcessWebResponse(String strRequest,HttpWebResponse webResponse)
     {
       try