diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs
index 6104012..728f115 100755
--- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs
+++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs
@@ -906,7 +906,7 @@ namespace MarketData.Helper
sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?CIK=").Append(symbol.ToUpper()).Append("&Find=Search&owner=exclude&action=getcompany");
strRequest = sb.ToString();
WebProxy webProxy=HttpNetRequest.GetProxy("GetCIK");
- httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
+ httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -976,7 +976,7 @@ namespace MarketData.Helper
sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
strRequest = sb.ToString();
WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions");
- httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
+ httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1143,7 +1143,7 @@ namespace MarketData.Helper
sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
strRequest = sb.ToString();
WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilings");
- httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
+ httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilings] Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1202,7 +1202,7 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]Retrieving documents for {0} at {1}",symbol,secFilingDocumentUrl));
WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments");
if(null == webProxy)MDTrace.WriteLine(LogLevel.DEBUG,"[GetSECFilingDocuments] Not using web proxy.");
- httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy);
+ httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]Request Symbol:{0} Request:{1} failed with status {2}",symbol,httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1246,7 +1246,7 @@ namespace MarketData.Helper
String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Symbol:{0} Requesting:{1}",symbol,secFilingDocumentXmlUrl));
try{Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS);}catch(Exception){;}
- httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy);
+ httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Symbol:{0} Request:{1} failed with status {2}",symbol,httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1986,7 +1986,7 @@ namespace MarketData.Helper
}
}
// ***************************************************************************************************************************************************************************
-// ************************************ C O N S U M E R P R I C E I N D E X - B U R E A U O F L A B O R S T I S T I C S ************************************************
+// ************************************ C O N S U M E R P R I C E I N D E X - B U R E A U O F L A B O R S T A T I S T I C S ************************************************
// ***************************************************************************************************************************************************************************
public static PriceIndices GetConsumerPriceIndices()
@@ -2007,7 +2007,7 @@ namespace MarketData.Helper
WebProxy webProxy=HttpNetRequest.GetProxy("GetConsumerPriceIndices");
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetConsumerPriceIndices: {0}", strRequest));
- httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"www.bls.gov", 300000,webProxy);
+ httpNetResponse=HttpNetRequest.GetRequestBOLS(strRequest,"www.bls.gov", 300000,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
diff --git a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs
index fb4d32d..8c0b747 100755
--- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs
+++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs
@@ -1598,6 +1598,134 @@ namespace MarketData.Integration
}
}
+ ///
+ /// GetRequestBOLS - This request is used for Bureau of Labor Statistics
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ public static HttpNetResponse GetRequestBOLS(string strRequest, string host, int webRequestTimeoutMS, WebProxy webProxy = null, bool useRandomUserAgent = false, CookieCollection cookieCollection = null)
+ {
+ HttpResponseMessage response = null;
+ try
+ {
+ MDTrace.WriteLine(LogLevel.VERBOSE,$"GetRequestBOLS[ENTER]{strRequest}");
+ HttpClientHandler handler = new HttpClientHandler()
+ {
+ AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli,
+ UseCookies = true,
+ CookieContainer = new CookieContainer(),
+ AllowAutoRedirect = true
+ };
+
+ if (webProxy != null)
+ {
+ handler.Proxy = webProxy;
+ handler.UseProxy = true;
+ }
+
+ if (cookieCollection != null)
+ {
+ handler.CookieContainer.Add(cookieCollection);
+ }
+
+ using (HttpClient client = new HttpClient(handler))
+ {
+ client.Timeout = TimeSpan.FromMilliseconds(webRequestTimeoutMS);
+
+ Uri uri = new Uri(strRequest);
+
+ // Required headers
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip, deflate, br, zstd");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Language", "en-US,en;q=0.9");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Connection", "keep-alive");
+ client.DefaultRequestHeaders.Host = host;
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Priority", "u=0, i");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest", "document");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode", "navigate");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site", "none");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User", "?1");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC", "1");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests","1");
+
+ // User agent
+ if (useRandomUserAgent)client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent",UserAgent.GetInstance().GetUserAgent());
+ else client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0");
+
+ response = client.GetAsync(strRequest).GetAwaiter().GetResult();
+ string html = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+ return new HttpNetResponse(html, strRequest, null, handler.CookieContainer.GetCookies(uri), response.IsSuccessStatusCode);
+ } // using
+ }
+ catch (Exception ex)
+ {
+ return new HttpNetResponse(null, strRequest, false, ex.Message);
+ }
+ finally
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,"GetRequestGDP[LEAVE]");
+ }
+ }
+
+ ///
+ /// This one is used to fetch data from GetRequestSECEDGAR like CIK.
+ ///
+ ///
+ ///
+ ///
+ ///
+ public static HttpNetResponse GetRequestSECEDGAR(string strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
+ {
+ HttpResponseMessage response=null;
+ try
+ {
+ MDTrace.WriteLine(LogLevel.VERBOSE,$"GetRequestSECEDGAR[ENTER]{strRequest}");
+
+ var handler=new HttpClientHandler()
+ {
+ AllowAutoRedirect=true,
+ AutomaticDecompression=DecompressionMethods.GZip|DecompressionMethods.Deflate|DecompressionMethods.Brotli,
+ UseCookies=true,
+ CookieContainer=new CookieContainer()
+ };
+
+ if(null!=webProxy){handler.Proxy=webProxy;handler.UseProxy=true;}
+
+ using(HttpClient client=new HttpClient(handler))
+ {
+ client.Timeout=TimeSpan.FromMilliseconds(webRequestTimeoutMS);
+ Uri uri=new Uri(strRequest);
+
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding","gzip, deflate, br, zstd");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Language","en-US,en;q=0.9");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Connection","keep-alive");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Priority","u=0, i");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest","document");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode","navigate");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site","none");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User","?1");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC","1");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("TE","trailers");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests","1");
+ client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0");
+
+ response=client.GetAsync(strRequest).GetAwaiter().GetResult();
+ string html=response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
+
+ return new HttpNetResponse(html,strRequest,null,handler.CookieContainer.GetCookies(uri),response.IsSuccessStatusCode);
+ }
+ }
+ catch(WebException webException){return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);}
+ catch(Exception exception){return new HttpNetResponse(null,strRequest,false,exception.Message);}
+ finally{MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestSECEDGAR[LEAVE]");}
+ }
+
private static HttpNetResponse ProcessWebResponse(String strRequest,HttpWebResponse webResponse)
{
try