diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs index 6104012..728f115 100755 --- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs @@ -906,7 +906,7 @@ namespace MarketData.Helper sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?CIK=").Append(symbol.ToUpper()).Append("&Find=Search&owner=exclude&action=getcompany"); strRequest = sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy("GetCIK"); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy); + httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(strRequest,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); @@ -976,7 +976,7 @@ namespace MarketData.Helper sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString()); strRequest = sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions"); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy); + httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(strRequest,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); @@ -1143,7 +1143,7 @@ namespace MarketData.Helper sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString()); strRequest = sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilings"); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy); + httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(strRequest,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilings] Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); @@ -1202,7 +1202,7 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]Retrieving documents for {0} at {1}",symbol,secFilingDocumentUrl)); WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments"); if(null == webProxy)MDTrace.WriteLine(LogLevel.DEBUG,"[GetSECFilingDocuments] Not using web proxy."); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy); + httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]Request Symbol:{0} Request:{1} failed with status {2}",symbol,httpNetResponse.Request,httpNetResponse.StatusCode)); @@ -1246,7 +1246,7 @@ namespace MarketData.Helper String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown"); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Symbol:{0} Requesting:{1}",symbol,secFilingDocumentXmlUrl)); try{Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS);}catch(Exception){;} - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy); + httpNetResponse=HttpNetRequest.GetRequestSECEDGAR(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Symbol:{0} Request:{1} failed with status {2}",symbol,httpNetResponse.Request,httpNetResponse.StatusCode)); @@ -1986,7 +1986,7 @@ namespace MarketData.Helper } } // *************************************************************************************************************************************************************************** -// ************************************ C O N S U M E R P R I C E I N D E X - B U R E A U O F L A B O R S T I S T I C S ************************************************ +// ************************************ C O N S U M E R P R I C E I N D E X - B U R E A U O F L A B O R S T A T I S T I C S ************************************************ // *************************************************************************************************************************************************************************** public static PriceIndices GetConsumerPriceIndices() @@ -2007,7 +2007,7 @@ namespace MarketData.Helper WebProxy webProxy=HttpNetRequest.GetProxy("GetConsumerPriceIndices"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetConsumerPriceIndices: {0}", strRequest)); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"www.bls.gov", 300000,webProxy); + httpNetResponse=HttpNetRequest.GetRequestBOLS(strRequest,"www.bls.gov", 300000,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); diff --git a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs index fb4d32d..8c0b747 100755 --- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs +++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs @@ -1598,6 +1598,134 @@ namespace MarketData.Integration } } + /// + /// GetRequestBOLS - This request is used for Bureau of Labor Statistics + /// + /// + /// + /// + /// + /// + /// + /// + public static HttpNetResponse GetRequestBOLS(string strRequest, string host, int webRequestTimeoutMS, WebProxy webProxy = null, bool useRandomUserAgent = false, CookieCollection cookieCollection = null) + { + HttpResponseMessage response = null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE,$"GetRequestBOLS[ENTER]{strRequest}"); + HttpClientHandler handler = new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli, + UseCookies = true, + CookieContainer = new CookieContainer(), + AllowAutoRedirect = true + }; + + if (webProxy != null) + { + handler.Proxy = webProxy; + handler.UseProxy = true; + } + + if (cookieCollection != null) + { + handler.CookieContainer.Add(cookieCollection); + } + + using (HttpClient client = new HttpClient(handler)) + { + client.Timeout = TimeSpan.FromMilliseconds(webRequestTimeoutMS); + + Uri uri = new Uri(strRequest); + + // Required headers + client.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip, deflate, br, zstd"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Language", "en-US,en;q=0.9"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Connection", "keep-alive"); + client.DefaultRequestHeaders.Host = host; + client.DefaultRequestHeaders.TryAddWithoutValidation("Priority", "u=0, i"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest", "document"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode", "navigate"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site", "none"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User", "?1"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC", "1"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests","1"); + + // User agent + if (useRandomUserAgent)client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent",UserAgent.GetInstance().GetUserAgent()); + else client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0"); + + response = client.GetAsync(strRequest).GetAwaiter().GetResult(); + string html = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + return new HttpNetResponse(html, strRequest, null, handler.CookieContainer.GetCookies(uri), response.IsSuccessStatusCode); + } // using + } + catch (Exception ex) + { + return new HttpNetResponse(null, strRequest, false, ex.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.DEBUG,"GetRequestGDP[LEAVE]"); + } + } + + /// + /// This one is used to fetch data from GetRequestSECEDGAR like CIK. + /// + /// + /// + /// + /// + public static HttpNetResponse GetRequestSECEDGAR(string strRequest,int webRequestTimeoutMS,WebProxy webProxy=null) + { + HttpResponseMessage response=null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE,$"GetRequestSECEDGAR[ENTER]{strRequest}"); + + var handler=new HttpClientHandler() + { + AllowAutoRedirect=true, + AutomaticDecompression=DecompressionMethods.GZip|DecompressionMethods.Deflate|DecompressionMethods.Brotli, + UseCookies=true, + CookieContainer=new CookieContainer() + }; + + if(null!=webProxy){handler.Proxy=webProxy;handler.UseProxy=true;} + + using(HttpClient client=new HttpClient(handler)) + { + client.Timeout=TimeSpan.FromMilliseconds(webRequestTimeoutMS); + Uri uri=new Uri(strRequest); + + client.DefaultRequestHeaders.TryAddWithoutValidation("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding","gzip, deflate, br, zstd"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Language","en-US,en;q=0.9"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Connection","keep-alive"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Priority","u=0, i"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest","document"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode","navigate"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site","none"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User","?1"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC","1"); + client.DefaultRequestHeaders.TryAddWithoutValidation("TE","trailers"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests","1"); + client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0"); + + response=client.GetAsync(strRequest).GetAwaiter().GetResult(); + string html=response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + + return new HttpNetResponse(html,strRequest,null,handler.CookieContainer.GetCookies(uri),response.IsSuccessStatusCode); + } + } + catch(WebException webException){return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);} + catch(Exception exception){return new HttpNetResponse(null,strRequest,false,exception.Message);} + finally{MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestSECEDGAR[LEAVE]");} + } + private static HttpNetResponse ProcessWebResponse(String strRequest,HttpWebResponse webResponse) { try