Fix Feeds for Bureau of Labor Statistics and EDGAR.

2026-03-17 17:51:06 -04:00
parent 86baa9de07
commit 0a4cfa4744
2 changed files with 135 additions and 7 deletions
--- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs
+++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs
@@ -906,7 +906,7 @@ namespace MarketData.Helper
        sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?CIK=").Append(symbol.ToUpper()).Append("&Find=Search&owner=exclude&action=getcompany");
        strRequest = sb.ToString();
        WebProxy webProxy=HttpNetRequest.GetProxy("GetCIK");
-        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
+        httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -976,7 +976,7 @@ namespace MarketData.Helper
        sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
        strRequest = sb.ToString();
        WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions");
-        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
+        httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1143,7 +1143,7 @@ namespace MarketData.Helper
        sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
        strRequest = sb.ToString();
        WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilings");
-        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
+        httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilings] Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1202,7 +1202,7 @@ namespace MarketData.Helper
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]Retrieving documents for {0} at {1}",symbol,secFilingDocumentUrl));
        WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments");
        if(null == webProxy)MDTrace.WriteLine(LogLevel.DEBUG,"[GetSECFilingDocuments] Not using web proxy.");
-        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy);
+        httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy);        
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]Request Symbol:{0} Request:{1} failed with status {2}",symbol,httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1246,7 +1246,7 @@ namespace MarketData.Helper
            String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Symbol:{0} Requesting:{1}",symbol,secFilingDocumentXmlUrl));
            try{Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS);}catch(Exception){;}
-            httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy);
+            httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy);
            if(!httpNetResponse.Success)
            {
              MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Symbol:{0} Request:{1} failed with status {2}",symbol,httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1986,7 +1986,7 @@ namespace MarketData.Helper
      }
    }
 // ***************************************************************************************************************************************************************************
-// ************************************ C O N S U M E R  P R I C E  I N D E X  -  B U R E A U  O F  L A B O R  S T I S T I C S ************************************************
+// ************************************ C O N S U M E R  P R I C E  I N D E X  -  B U R E A U  O F  L A B O R  S T A T I S T I C S ************************************************
 // ***************************************************************************************************************************************************************************

 		public static PriceIndices GetConsumerPriceIndices()
@@ -2007,7 +2007,7 @@ namespace MarketData.Helper
        WebProxy webProxy=HttpNetRequest.GetProxy("GetConsumerPriceIndices");
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetConsumerPriceIndices: {0}", strRequest));
-        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"www.bls.gov", 300000,webProxy);
+        httpNetResponse=HttpNetRequest.GetRequestBOLS(strRequest,"www.bls.gov", 300000,webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
--- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs
+++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs
@@ -1598,6 +1598,134 @@ namespace MarketData.Integration
      }
    }

+    /// <summary>
+    /// GetRequestBOLS - This request is used for Bureau of Labor Statistics
+    /// </summary>
+    /// <param name="strRequest"></param>
+    /// <param name="host"></param>
+    /// <param name="webRequestTimeoutMS"></param>
+    /// <param name="webProxy"></param>
+    /// <param name="useRandomUserAgent"></param>
+    /// <param name="cookieCollection"></param>
+    /// <returns></returns>
+    public static HttpNetResponse GetRequestBOLS(string strRequest, string host, int webRequestTimeoutMS, WebProxy webProxy = null, bool useRandomUserAgent = false, CookieCollection cookieCollection = null)
+    {
+      HttpResponseMessage response = null;
+      try
+      {
+        MDTrace.WriteLine(LogLevel.VERBOSE,$"GetRequestBOLS[ENTER]{strRequest}");
+        HttpClientHandler handler = new HttpClientHandler()
+        {
+            AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli,
+            UseCookies = true,
+            CookieContainer = new CookieContainer(),
+            AllowAutoRedirect = true
+        };
+
+        if (webProxy != null)
+        {
+            handler.Proxy = webProxy;
+            handler.UseProxy = true;
+        }
+
+        if (cookieCollection != null)
+        {
+            handler.CookieContainer.Add(cookieCollection);
+        }
+
+        using (HttpClient client = new HttpClient(handler))
+        {
+          client.Timeout = TimeSpan.FromMilliseconds(webRequestTimeoutMS);
+
+          Uri uri = new Uri(strRequest);
+
+          // Required headers
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip, deflate, br, zstd");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Language", "en-US,en;q=0.9");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Connection", "keep-alive");
+          client.DefaultRequestHeaders.Host = host;
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Priority", "u=0, i");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest", "document");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode", "navigate");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site", "none");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User", "?1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC", "1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests","1");
+
+          // User agent
+          if (useRandomUserAgent)client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent",UserAgent.GetInstance().GetUserAgent());
+          else client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0");
+
+          response = client.GetAsync(strRequest).GetAwaiter().GetResult();
+          string html = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+          return new HttpNetResponse(html, strRequest, null, handler.CookieContainer.GetCookies(uri), response.IsSuccessStatusCode);
+        } // using
+      }
+      catch (Exception ex)
+      {
+          return new HttpNetResponse(null, strRequest, false, ex.Message);
+      }
+      finally
+      {
+          MDTrace.WriteLine(LogLevel.DEBUG,"GetRequestGDP[LEAVE]");
+      }
+    }
+
+    /// <summary>
+    /// This one is used to fetch data from GetRequestSECEDGAR like CIK.
+    /// </summary>
+    /// <param name="strRequest"></param>
+    /// <param name="webRequestTimeoutMS"></param>
+    /// <param name="webProxy"></param>
+    /// <returns></returns>
+    public static HttpNetResponse GetRequestSECEDGAR(string strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
+    {
+      HttpResponseMessage response=null;
+      try
+      {
+        MDTrace.WriteLine(LogLevel.VERBOSE,$"GetRequestSECEDGAR[ENTER]{strRequest}");
+
+        var handler=new HttpClientHandler()
+        {
+          AllowAutoRedirect=true,
+          AutomaticDecompression=DecompressionMethods.GZip|DecompressionMethods.Deflate|DecompressionMethods.Brotli,
+          UseCookies=true,
+          CookieContainer=new CookieContainer()
+        };
+
+        if(null!=webProxy){handler.Proxy=webProxy;handler.UseProxy=true;}
+
+        using(HttpClient client=new HttpClient(handler))
+        {
+          client.Timeout=TimeSpan.FromMilliseconds(webRequestTimeoutMS);
+          Uri uri=new Uri(strRequest);
+
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding","gzip, deflate, br, zstd");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Language","en-US,en;q=0.9");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Connection","keep-alive");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Priority","u=0, i");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest","document");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode","navigate");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site","none");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User","?1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC","1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("TE","trailers");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests","1");
+          client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0");
+
+          response=client.GetAsync(strRequest).GetAwaiter().GetResult();
+          string html=response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+
+          return new HttpNetResponse(html,strRequest,null,handler.CookieContainer.GetCookies(uri),response.IsSuccessStatusCode);
+        }
+      }
+      catch(WebException webException){return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);}
+      catch(Exception exception){return new HttpNetResponse(null,strRequest,false,exception.Message);}
+      finally{MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestSECEDGAR[LEAVE]");}
+    }        
+
    private static HttpNetResponse ProcessWebResponse(String strRequest,HttpWebResponse webResponse)
    {
      try