Fix Feeds for Bureau of Labor Statistics and EDGAR.

2026-03-17 17:51:06 -04:00
parent 86baa9de07
commit 0a4cfa4744
2 changed files with 135 additions and 7 deletions
--- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs
+++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs
@@ -1598,6 +1598,134 @@ namespace MarketData.Integration
      }
    }

+    /// <summary>
+    /// GetRequestBOLS - This request is used for Bureau of Labor Statistics
+    /// </summary>
+    /// <param name="strRequest"></param>
+    /// <param name="host"></param>
+    /// <param name="webRequestTimeoutMS"></param>
+    /// <param name="webProxy"></param>
+    /// <param name="useRandomUserAgent"></param>
+    /// <param name="cookieCollection"></param>
+    /// <returns></returns>
+    public static HttpNetResponse GetRequestBOLS(string strRequest, string host, int webRequestTimeoutMS, WebProxy webProxy = null, bool useRandomUserAgent = false, CookieCollection cookieCollection = null)
+    {
+      HttpResponseMessage response = null;
+      try
+      {
+        MDTrace.WriteLine(LogLevel.VERBOSE,$"GetRequestBOLS[ENTER]{strRequest}");
+        HttpClientHandler handler = new HttpClientHandler()
+        {
+            AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli,
+            UseCookies = true,
+            CookieContainer = new CookieContainer(),
+            AllowAutoRedirect = true
+        };
+
+        if (webProxy != null)
+        {
+            handler.Proxy = webProxy;
+            handler.UseProxy = true;
+        }
+
+        if (cookieCollection != null)
+        {
+            handler.CookieContainer.Add(cookieCollection);
+        }
+
+        using (HttpClient client = new HttpClient(handler))
+        {
+          client.Timeout = TimeSpan.FromMilliseconds(webRequestTimeoutMS);
+
+          Uri uri = new Uri(strRequest);
+
+          // Required headers
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip, deflate, br, zstd");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Language", "en-US,en;q=0.9");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Connection", "keep-alive");
+          client.DefaultRequestHeaders.Host = host;
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Priority", "u=0, i");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest", "document");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode", "navigate");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site", "none");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User", "?1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC", "1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests","1");
+
+          // User agent
+          if (useRandomUserAgent)client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent",UserAgent.GetInstance().GetUserAgent());
+          else client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0");
+
+          response = client.GetAsync(strRequest).GetAwaiter().GetResult();
+          string html = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+          return new HttpNetResponse(html, strRequest, null, handler.CookieContainer.GetCookies(uri), response.IsSuccessStatusCode);
+        } // using
+      }
+      catch (Exception ex)
+      {
+          return new HttpNetResponse(null, strRequest, false, ex.Message);
+      }
+      finally
+      {
+          MDTrace.WriteLine(LogLevel.DEBUG,"GetRequestGDP[LEAVE]");
+      }
+    }
+
+    /// <summary>
+    /// This one is used to fetch data from GetRequestSECEDGAR like CIK.
+    /// </summary>
+    /// <param name="strRequest"></param>
+    /// <param name="webRequestTimeoutMS"></param>
+    /// <param name="webProxy"></param>
+    /// <returns></returns>
+    public static HttpNetResponse GetRequestSECEDGAR(string strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
+    {
+      HttpResponseMessage response=null;
+      try
+      {
+        MDTrace.WriteLine(LogLevel.VERBOSE,$"GetRequestSECEDGAR[ENTER]{strRequest}");
+
+        var handler=new HttpClientHandler()
+        {
+          AllowAutoRedirect=true,
+          AutomaticDecompression=DecompressionMethods.GZip|DecompressionMethods.Deflate|DecompressionMethods.Brotli,
+          UseCookies=true,
+          CookieContainer=new CookieContainer()
+        };
+
+        if(null!=webProxy){handler.Proxy=webProxy;handler.UseProxy=true;}
+
+        using(HttpClient client=new HttpClient(handler))
+        {
+          client.Timeout=TimeSpan.FromMilliseconds(webRequestTimeoutMS);
+          Uri uri=new Uri(strRequest);
+
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding","gzip, deflate, br, zstd");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Language","en-US,en;q=0.9");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Connection","keep-alive");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Priority","u=0, i");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest","document");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode","navigate");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site","none");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User","?1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC","1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("TE","trailers");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests","1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0");
+
+          response=client.GetAsync(strRequest).GetAwaiter().GetResult();
+          string html=response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+
+          return new HttpNetResponse(html,strRequest,null,handler.CookieContainer.GetCookies(uri),response.IsSuccessStatusCode);
+        }
+      }
+      catch(WebException webException){return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);}
+      catch(Exception exception){return new HttpNetResponse(null,strRequest,false,exception.Message);}
+      finally{MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestSECEDGAR[LEAVE]");}
+    }        
+
    private static HttpNetResponse ProcessWebResponse(String strRequest,HttpWebResponse webResponse)
    {
      try