diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs index 7f6b385..cac0b56 100755 --- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs @@ -5240,23 +5240,29 @@ namespace MarketData.Helper return prices; } + /// + /// The main BigChart Feed which is no longer working + /// + /// + /// + /// public static Price GetPriceAsOf(String symbol, DateTime asOf) { - HttpNetResponse httpNetResponse=null; + HttpNetResponse httpNetResponse = null; try { String strRequest; StringBuilder sb = null; if (null == symbol) return null; - CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol); - if(null!=companyProfile && companyProfile.FreezePricing) + CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); + if (null != companyProfile && companyProfile.FreezePricing) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); return null; } - String requestSymbol=symbol; - if(requestSymbol.StartsWith("^"))requestSymbol=requestSymbol.Substring(1); + String requestSymbol = symbol; + if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1); sb = new StringBuilder(); sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb="); sb.Append(requestSymbol); @@ -5265,14 +5271,13 @@ namespace MarketData.Helper sb.Append("%2F"); sb.Append(asOf.Day.ToString()); sb.Append("%2F"); - sb.Append((asOf.Year-2000).ToString()); + sb.Append((asOf.Year - 2000).ToString()); sb.Append("&x=38&y=25"); strRequest = sb.ToString(); - - httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest); - if(!httpNetResponse.Success) + httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest); + if (!httpNetResponse.Success) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); @@ -5283,51 +5288,137 @@ namespace MarketData.Helper if (null == tables || 0 == tables.Count) return null; HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); if (rows.Count < 7) return null; - Price price=new Price(); - price.Source=Price.PriceSource.BigCharts; + Price price = new Price(); + price.Source = Price.PriceSource.BigCharts; price.Symbol = symbol.ToUpper(); - price.Date=FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText); - price.Close=FeedParser.ParseValueFromMarketWatch("Closing Price:",rows[2].InnerText); + price.Date = FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText); + price.Close = FeedParser.ParseValueFromMarketWatch("Closing Price:", rows[2].InnerText); price.AdjClose = price.Close; price.Open = FeedParser.ParseValueFromMarketWatch("Open:", rows[3].InnerText); price.High = FeedParser.ParseValueFromMarketWatch("High:", rows[4].InnerText); price.Low = FeedParser.ParseValueFromMarketWatch("Low:", rows[5].InnerText); price.Volume = FeedParser.ParseLongValueFromMarketWatch("Volume:", rows[6].InnerText); - if(!(price.Date.Date.Equals(asOf.Date.Date))) + if (!(price.Date.Date.Equals(asOf.Date.Date))) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}",symbol,price.Date.ToShortDateString(),asOf.ToShortDateString())); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}", symbol, price.Date.ToShortDateString(), asOf.ToShortDateString())); return null; } return price; } - catch(Exception) + catch (Exception) { return null; } finally { - if(null!=httpNetResponse)httpNetResponse.Dispose(); + if (null != httpNetResponse) httpNetResponse.Dispose(); } } -//******************************************************************************************************************************************************************************************************** -// ******************************************************************************** H I S T O R I C A L P R I C I N G Y A H O O ********************************************************************* -//******************************************************************************************************************************************************************************************************** + - public static Price GetDailyPrice(String symbol,DateTime pricingDate) + + public static Price GetPriceAsOfV2(String symbol, DateTime asOf) { - if(null==symbol)return null; - CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol); - if(null!=companyProfile && companyProfile.FreezePricing) + HttpNetResponse httpNetResponse = null; + try + { + String strRequest; + StringBuilder sb = null; + + if (null == symbol) return null; + CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); + if (null != companyProfile && companyProfile.FreezePricing) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); + return null; + } + String requestSymbol = symbol; + if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1); + sb = new StringBuilder(); + // sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb="); + // sb.Append(requestSymbol); + // sb.Append("&closeDate="); + // sb.Append(asOf.Month.ToString()); + // sb.Append("%2F"); + // sb.Append(asOf.Day.ToString()); + // sb.Append("%2F"); + // sb.Append((asOf.Year - 2000).ToString()); + // sb.Append("&x=38&y=25");0000000000000000000000000000000000000000 + // strRequest = sb.ToString(); + + + + + //https://www.marketwatch.com/investing/stock/TSCDY/download-data + sb.Append("https://www.marketwatch.com/investing/stock/").Append(symbol); + sb.Append("/downloaddatapartial?startdate="); + sb.Append(asOf.Month.ToString()).Append("/").Append(asOf.Day.ToString()).Append("/").Append(asOf.Year.ToString()); + sb.Append("%2000:00:00&enddate="); + sb.Append(asOf.Month.ToString()).Append("/").Append(asOf.Day.ToString()).Append("/").Append(asOf.Year.ToString()); + sb.Append("%2023:59:59&daterange=d30&frequency=p1d&csvdownload=true&downloadpartial=false&newdates=false"); + + + strRequest = sb.ToString(); + + httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest); + if (!httpNetResponse.Success) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); + return null; + } + byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); + MemoryStream memoryStream = new MemoryStream(streamBytes); + HtmlDocument htmlDocument = new HtmlDocument(); + htmlDocument.Load(memoryStream); + HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"historicalquote fatbottomed\"]"); + if (null == tables || 0 == tables.Count) return null; + HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); + if (rows.Count < 7) return null; + Price price = new Price(); + price.Source = Price.PriceSource.BigCharts; + price.Symbol = symbol.ToUpper(); + price.Date = FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText); + price.Close = FeedParser.ParseValueFromMarketWatch("Closing Price:", rows[2].InnerText); + price.AdjClose = price.Close; + price.Open = FeedParser.ParseValueFromMarketWatch("Open:", rows[3].InnerText); + price.High = FeedParser.ParseValueFromMarketWatch("High:", rows[4].InnerText); + price.Low = FeedParser.ParseValueFromMarketWatch("Low:", rows[5].InnerText); + price.Volume = FeedParser.ParseLongValueFromMarketWatch("Volume:", rows[6].InnerText); + if (!(price.Date.Date.Equals(asOf.Date.Date))) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}", symbol, price.Date.ToShortDateString(), asOf.ToShortDateString())); + return null; + } + return price; + } + catch (Exception) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol)); return null; } - Prices prices=GetDailyPrices(symbol,pricingDate,pricingDate); - if(null==prices||0==prices.Count)return null; - Price price=prices.FirstOrDefault(); - if(!price.Date.Date.Equals(pricingDate.Date)) + finally { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetDailyPrice: The pricing date returned for '{0}' was different than the date requested. Date requested '{1}', date returned '{2}'",symbol,pricingDate.ToShortDateString(),price.Date.Date.ToShortDateString())); + if (null != httpNetResponse) httpNetResponse.Dispose(); + } + } +//******************************************************************************************************************************************************************************************************** + // ******************************************************************************** H I S T O R I C A L P R I C I N G Y A H O O ********************************************************************* + //******************************************************************************************************************************************************************************************************** + + public static Price GetDailyPrice(String symbol, DateTime pricingDate) + { + if (null == symbol) return null; + CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); + if (null != companyProfile && companyProfile.FreezePricing) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); + return null; + } + Prices prices = GetDailyPrices(symbol, pricingDate, pricingDate); + if (null == prices || 0 == prices.Count) return null; + Price price = prices.FirstOrDefault(); + if (!price.Date.Date.Equals(pricingDate.Date)) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetDailyPrice: The pricing date returned for '{0}' was different than the date requested. Date requested '{1}', date returned '{2}'", symbol, pricingDate.ToShortDateString(), price.Date.Date.ToShortDateString())); return null; } return price; diff --git a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs index 278a1c8..de7185e 100755 --- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs +++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs @@ -406,24 +406,24 @@ namespace MarketData.Integration } } - public static HttpNetResponse GetRequestCSV(String strRequest,String referer) + public static HttpNetResponse GetRequestNoEncodingV2(String strRequest,WebProxy webProxy=null) { HttpWebResponse webResponse=null; try { - MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestCSV[ENTER]{0}",strRequest)); + ServicePointManager.Expect100Continue = true; + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",strRequest)); int charCount = 0; byte[] buffer = new byte[8192]; StringBuilder sb = new StringBuilder(); HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; webRequest.Timeout = REQUEST_TIMEOUT; - webRequest.Accept="text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + webRequest.Accept = "text/html, text/csv"; webRequest.ContentType="text/csv; charset=utf-8"; - webRequest.Headers.Add("Accept-Encoding: identity"); // "gzip, deflate" - webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); - webRequest.Headers.Add("Upgrade-Insecure-Requests: 1"); - webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0"; - webRequest.Referer=referer; + webRequest.Headers.Add("Accept-Encoding: None"); + webRequest.Headers.Add("Accept-Language: en-US"); + webRequest.UserAgent = "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36"; webRequest.KeepAlive = true; try{webResponse = (HttpWebResponse)webRequest.GetResponse();} catch(WebException webException) @@ -439,7 +439,7 @@ namespace MarketData.Integration while (true) { charCount = responseStream.Read(buffer, 0, buffer.Length); - if (0 == charCount) break; + if (0 >= charCount) break; sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); } return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true); @@ -459,7 +459,64 @@ namespace MarketData.Integration webResponse.Close(); webResponse.Dispose(); } - MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestCSV[LEAVE]"); + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]"); + } + } + + public static HttpNetResponse GetRequestCSV(String strRequest, String referer) + { + HttpWebResponse webResponse = null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestCSV[ENTER]{0}", strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + webRequest.Timeout = REQUEST_TIMEOUT; + webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + webRequest.ContentType = "text/csv; charset=utf-8"; + webRequest.Headers.Add("Accept-Encoding: identity"); // "gzip, deflate" + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Upgrade-Insecure-Requests: 1"); + webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0"; + webRequest.Referer = referer; + webRequest.KeepAlive = true; + try { webResponse = (HttpWebResponse)webRequest.GetResponse(); } + catch (WebException webException) + { + if (IsMovedException(webException)) + { + webRequest = Redirect(webRequest, webException); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + } + else throw; + } + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + return new HttpNetResponse(sb.ToString(), strRequest, webResponse, true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); + } + finally + { + if (null != webResponse) + { + webResponse.Close(); + webResponse.Dispose(); + } + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestCSV[LEAVE]"); } } @@ -475,9 +532,10 @@ namespace MarketData.Integration HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); webRequest.Timeout = REQUEST_TIMEOUT; webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); - webRequest.Headers.Add("Accept-Encoding: None"); - webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0"; - webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; + webRequest.ContentType = "text/csv; charset=utf-8"; + webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br, zstd"); + webRequest.UserAgent = "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36"; + webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"; webRequest.KeepAlive = true; try{webResponse = (HttpWebResponse)webRequest.GetResponse();} catch(WebException webException)