diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs
index b7ad74f..a683d10 100755
--- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs
+++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs
@@ -5240,23 +5240,29 @@ namespace MarketData.Helper
return prices;
}
+ ///
+ /// The main BigChart Feed which is no longer working
+ ///
+ ///
+ ///
+ ///
public static Price GetPriceAsOf(String symbol, DateTime asOf)
{
- HttpNetResponse httpNetResponse=null;
+ HttpNetResponse httpNetResponse = null;
try
{
String strRequest;
StringBuilder sb = null;
if (null == symbol) return null;
- CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
- if(null!=companyProfile && companyProfile.FreezePricing)
+ CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol);
+ if (null != companyProfile && companyProfile.FreezePricing)
{
- MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol));
+ MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol));
return null;
}
- String requestSymbol=symbol;
- if(requestSymbol.StartsWith("^"))requestSymbol=requestSymbol.Substring(1);
+ String requestSymbol = symbol;
+ if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1);
sb = new StringBuilder();
sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb=");
sb.Append(requestSymbol);
@@ -5265,14 +5271,13 @@ namespace MarketData.Helper
sb.Append("%2F");
sb.Append(asOf.Day.ToString());
sb.Append("%2F");
- sb.Append((asOf.Year-2000).ToString());
+ sb.Append((asOf.Year - 2000).ToString());
sb.Append("&x=38&y=25");
strRequest = sb.ToString();
-
- httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
- if(!httpNetResponse.Success)
+ httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest);
+ if (!httpNetResponse.Success)
{
- MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
+ MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
@@ -5283,51 +5288,165 @@ namespace MarketData.Helper
if (null == tables || 0 == tables.Count) return null;
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
if (rows.Count < 7) return null;
- Price price=new Price();
- price.Source=Price.PriceSource.BigCharts;
+ Price price = new Price();
+ price.Source = Price.PriceSource.BigCharts;
price.Symbol = symbol.ToUpper();
- price.Date=FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText);
- price.Close=FeedParser.ParseValueFromMarketWatch("Closing Price:",rows[2].InnerText);
+ price.Date = FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText);
+ price.Close = FeedParser.ParseValueFromMarketWatch("Closing Price:", rows[2].InnerText);
price.AdjClose = price.Close;
price.Open = FeedParser.ParseValueFromMarketWatch("Open:", rows[3].InnerText);
price.High = FeedParser.ParseValueFromMarketWatch("High:", rows[4].InnerText);
price.Low = FeedParser.ParseValueFromMarketWatch("Low:", rows[5].InnerText);
price.Volume = FeedParser.ParseLongValueFromMarketWatch("Volume:", rows[6].InnerText);
- if(!(price.Date.Date.Equals(asOf.Date.Date)))
+ if (!(price.Date.Date.Equals(asOf.Date.Date)))
{
- MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}",symbol,price.Date.ToShortDateString(),asOf.ToShortDateString()));
+ MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}", symbol, price.Date.ToShortDateString(), asOf.ToShortDateString()));
return null;
}
return price;
}
- catch(Exception)
+ catch (Exception)
{
return null;
}
finally
{
- if(null!=httpNetResponse)httpNetResponse.Dispose();
+ if (null != httpNetResponse) httpNetResponse.Dispose();
}
}
-//********************************************************************************************************************************************************************************************************
-// ******************************************************************************** H I S T O R I C A L P R I C I N G Y A H O O *********************************************************************
-//********************************************************************************************************************************************************************************************************
- public static Price GetDailyPrice(String symbol,DateTime pricingDate)
+ ///
+ /// This is a modified version of the above query. It uses a cookie collection that contains the datadome cookie.
+ /// If this query starts to fail then you should try the query in developer tools in the chromium browser and capture the
+ /// datadome value and update it here in the GetCookieCollection() below.
+ ///
+ ///
+ ///
+ ///
+ public static Price GetPriceAsOfV2(String symbol, DateTime asOf)
{
- if(null==symbol)return null;
- CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
- if(null!=companyProfile && companyProfile.FreezePricing)
+ HttpNetResponse httpNetResponse = null;
+ try
{
- MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol));
+ String strRequest;
+ StringBuilder sb = null;
+
+ if (null == symbol) return null;
+ CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol);
+ if (null != companyProfile && companyProfile.FreezePricing)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol));
+ return null;
+ }
+ String requestSymbol = symbol;
+ if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1);
+ sb = new StringBuilder();
+ sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb=");
+ sb.Append(requestSymbol);
+ sb.Append("&closeDate=");
+ sb.Append(asOf.Month.ToString());
+ sb.Append("%2F");
+ sb.Append(asOf.Day.ToString());
+ sb.Append("%2F");
+ sb.Append((asOf.Year - 2000).ToString());
+ sb.Append("&x=38&y=25");
+ strRequest = sb.ToString();
+
+ MDTrace.WriteLine(LogLevel.DEBUG,$"{strRequest}");
+
+ CookieCollection cookieCollection = GetCookieCollection("www.marketwatch.com");
+
+ httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest,cookieCollection,new Uri("https://www.marketwatch.com/investing"));
+ if (!httpNetResponse.Success)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
+ return null;
+ }
+ byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
+ MemoryStream memoryStream = new MemoryStream(streamBytes);
+ HtmlDocument htmlDocument = new HtmlDocument();
+ htmlDocument.Load(memoryStream);
+
+ HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"table table--overflow align--center\"]");
+
+ if (null == tables || 0 == tables.Count) return null;
+ HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
+ if (rows.Count < 2) return null;
+ HtmlNodeCollection data = rows[1].SelectNodes(".//td");
+ if (data.Count != 6)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the correct number of data elements (expected 6 data elements).", symbol));
+ return null;
+ }
+ Price price = new Price();
+ price.Source = Price.PriceSource.BigCharts;
+ price.Symbol = symbol.ToUpper();
+ String[] dateElements = data[0].InnerText.Trim().Split(" ");
+
+ price.Date = Utility.ParseDate(dateElements[0].Replace("\n",null));
+ price.Open = Utility.ParseCurrency(data[1].InnerText.Trim().Replace("\n",null));
+ price.High = Utility.ParseCurrency(data[2].InnerText.Trim().Replace("\n",null));
+ price.Low = Utility.ParseCurrency(data[3].InnerText.Trim().Replace("\n",null));
+ price.Close = price.AdjClose = Utility.ParseCurrency(data[4].InnerText.Trim().Replace("\n",null));
+ price.Volume = FeedParser.ParseValueLong(data[5].InnerText.Trim());
+ if (!(price.Date.Date.Equals(asOf.Date.Date)))
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}", symbol, price.Date.ToShortDateString(), asOf.ToShortDateString()));
+ return null;
+ }
+ return price;
+ }
+ catch (Exception exception)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception : {0}", exception.ToString()));
return null;
}
- Prices prices=GetDailyPrices(symbol,pricingDate,pricingDate);
- if(null==prices||0==prices.Count)return null;
- Price price=prices.FirstOrDefault();
- if(!price.Date.Date.Equals(pricingDate.Date))
+ finally
{
- MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetDailyPrice: The pricing date returned for '{0}' was different than the date requested. Date requested '{1}', date returned '{2}'",symbol,pricingDate.ToShortDateString(),price.Date.Date.ToShortDateString()));
+ if (null != httpNetResponse) httpNetResponse.Dispose();
+ }
+ }
+
+ private static CookieCollection GetCookieCollection(String cookieDomain)
+ {
+ String[] cookies = {
+ "datadome=VLSWQ1PWL4SV2rljkJEV7GUOK3T11Sm73RP17IwDEtRzfWHykYwy7ZoeSPrjwmbwvGRAenazmLVTpyCB6Yqlw6vUL6Wbqq4zML5DGuHLxNU4LgAQ7Ko9tztglqjIGLZE",
+ };
+ CookieCollection cookieCollection = new CookieCollection();
+ for (int index = 0; index < cookies.Count(); index++)
+ {
+ String strCookie = cookies[index];
+ if (strCookie.EndsWith(";")) strCookie = strCookie.Substring(0, strCookie.Length - 1);
+ String[] pairs = strCookie.Split('=');
+ cookieCollection.Add(new Cookie()
+ {
+ Name = pairs[0],
+ Value = pairs[1],
+ Domain = cookieDomain
+ });
+ }
+ return cookieCollection;
+ }
+
+//********************************************************************************************************************************************************************************************************
+ // ******************************************************************************** H I S T O R I C A L P R I C I N G Y A H O O *********************************************************************
+ //********************************************************************************************************************************************************************************************************
+
+ public static Price GetDailyPrice(String symbol, DateTime pricingDate)
+ {
+ if (null == symbol) return null;
+ CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol);
+ if (null != companyProfile && companyProfile.FreezePricing)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol));
+ return null;
+ }
+ Prices prices = GetDailyPrices(symbol, pricingDate, pricingDate);
+ if (null == prices || 0 == prices.Count) return null;
+ Price price = prices.FirstOrDefault();
+ if (!price.Date.Date.Equals(pricingDate.Date))
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetDailyPrice: The pricing date returned for '{0}' was different than the date requested. Date requested '{1}', date returned '{2}'", symbol, pricingDate.ToShortDateString(), price.Date.Date.ToShortDateString()));
return null;
}
return price;
diff --git a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs
index 278a1c8..74d9ad5 100755
--- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs
+++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs
@@ -406,32 +406,135 @@ namespace MarketData.Integration
}
}
- public static HttpNetResponse GetRequestCSV(String strRequest,String referer)
+ ///
+ /// This one is being used in the MarketWatch price fetch
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ public static HttpNetResponse GetRequestNoEncodingV2(String strRequest, CookieCollection cookieCollection, Uri uri, WebProxy webProxy = null)
{
- HttpWebResponse webResponse=null;
+ HttpWebResponse webResponse = null;
try
{
- MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestCSV[ENTER]{0}",strRequest));
+ ServicePointManager.Expect100Continue = true;
+ MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV2[ENTER]{0}", strRequest));
+ int charCount = 0;
+ byte[] buffer = new byte[8192];
+ StringBuilder sb = new StringBuilder();
+ HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
+ if (null != webProxy) webRequest.Proxy = webProxy;
+ webRequest.Timeout = REQUEST_TIMEOUT;
+ webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7";
+ webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br, zstd");
+ webRequest.Headers.Add("Accept-Language: en-US,en;q=0.9");
+ webRequest.UserAgent = "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36";
+ webRequest.Headers.Add("sec-fetch-dest", "document");
+ webRequest.Headers.Add("sec-fetch-mode", "navigate");
+ webRequest.Headers.Add("sec-fetch-site", "none");
+ webRequest.Headers.Add("sec-fetch-user", "?1");
+ webRequest.Headers.Add("upgrade-insecure-requests", "1");
+
+ webRequest.Headers.Add("priority", "u=0, i");
+ webRequest.Headers.Add("sec-ch-device-memory", "8");
+
+ webRequest.Headers.Add("Referrer-Policy", "strict-origin-when-cross-origin");
+ webRequest.AllowAutoRedirect = true;
+
+ if (null != cookieCollection)
+ {
+ CookieContainer cookieContainer = new CookieContainer();
+ foreach (Cookie cookie in cookieCollection)
+ {
+ cookieContainer.Add(uri, cookie);
+ }
+ webRequest.CookieContainer = cookieContainer;
+ }
+
+ try { webResponse = (HttpWebResponse)webRequest.GetResponse(); }
+ catch (WebException webException)
+ {
+ if (IsMovedException(webException))
+ {
+ webRequest = Redirect(webRequest, webException);
+ webResponse = (HttpWebResponse)webRequest.GetResponse();
+ }
+ else throw;
+ }
+ Stream responseStream = webResponse.GetResponseStream();
+
+ if (webResponse.ContentEncoding.ToLower().Contains("gzip"))
+ {
+ responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
+ StreamReader reader = new StreamReader(responseStream, Encoding.Default);
+ sb.Append(reader.ReadToEnd());
+ reader.Close();
+ }
+ else if (webResponse.ContentEncoding.ToLower().Contains("deflate"))
+ {
+ responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
+ StreamReader reader = new StreamReader(responseStream, Encoding.Default);
+ sb.Append(reader.ReadToEnd());
+ reader.Close();
+ }
+ else
+ {
+ while (true)
+ {
+ charCount = responseStream.Read(buffer, 0, buffer.Length);
+ if (0 == charCount) break;
+ sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
+ }
+ }
+ return new HttpNetResponse(sb.ToString(), strRequest, webResponse, true);
+ }
+ catch (WebException webException)
+ {
+ return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
+ }
+ catch (Exception exception)
+ {
+ return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
+ }
+ finally
+ {
+ if (null != webResponse)
+ {
+ webResponse.Close();
+ webResponse.Dispose();
+ }
+ MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncoding[LEAVE]");
+ }
+ }
+
+ public static HttpNetResponse GetRequestCSV(String strRequest, String referer)
+ {
+ HttpWebResponse webResponse = null;
+ try
+ {
+ MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestCSV[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
webRequest.Timeout = REQUEST_TIMEOUT;
- webRequest.Accept="text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
- webRequest.ContentType="text/csv; charset=utf-8";
+ webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
+ webRequest.ContentType = "text/csv; charset=utf-8";
webRequest.Headers.Add("Accept-Encoding: identity"); // "gzip, deflate"
- webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
+ webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Upgrade-Insecure-Requests: 1");
- webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
- webRequest.Referer=referer;
+ webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
+ webRequest.Referer = referer;
webRequest.KeepAlive = true;
- try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
- catch(WebException webException)
+ try { webResponse = (HttpWebResponse)webRequest.GetResponse(); }
+ catch (WebException webException)
{
- if(IsMovedException(webException))
+ if (IsMovedException(webException))
{
- webRequest=Redirect(webRequest, webException);
- webResponse=(HttpWebResponse)webRequest.GetResponse();
+ webRequest = Redirect(webRequest, webException);
+ webResponse = (HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
@@ -442,24 +545,24 @@ namespace MarketData.Integration
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
- return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
+ return new HttpNetResponse(sb.ToString(), strRequest, webResponse, true);
}
catch (WebException webException)
{
- return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
+ return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
- return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
+ return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
- if(null!=webResponse)
+ if (null != webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
- MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestCSV[LEAVE]");
+ MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestCSV[LEAVE]");
}
}
@@ -475,9 +578,10 @@ namespace MarketData.Integration
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
webRequest.Timeout = REQUEST_TIMEOUT;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
- webRequest.Headers.Add("Accept-Encoding: None");
- webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
- webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
+ webRequest.ContentType = "text/csv; charset=utf-8";
+ webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br, zstd");
+ webRequest.UserAgent = "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36";
+ webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7";
webRequest.KeepAlive = true;
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
@@ -516,62 +620,63 @@ namespace MarketData.Integration
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV2[LEAVE]");
}
}
+
// I am using this code specifically on the seeking alpha web site. it seems as though seeking alpha has implemeted some user agent based bot prevention mechanism on their website
// to prevent scrapers. What I do here is to choose from a set of commonly used user agents and then randomly choose a user agent to code into the request. retry logic up to 3 times
// and handling of 404 (not found)
- public static HttpNetResponse GetRequestNoEncodingV3(String strRequest,String referer=null)
+ public static HttpNetResponse GetRequestNoEncodingV3(String strRequest, String referer = null)
{
- HttpWebResponse webResponse=null;
- WebException lastWebException=null;
-
+ HttpWebResponse webResponse = null;
+ WebException lastWebException = null;
+
try
{
- MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3[ENTER]{0}",strRequest));
- Random random=new Random();
- String[] userAgents=
+ MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV3[ENTER]{0}", strRequest));
+ Random random = new Random();
+ String[] userAgents =
{
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537."
};
- int MAX_RETRIES=5;
- int TIMEOUT_BETWEEN_RETRIES=1000;
+ int MAX_RETRIES = 5;
+ int TIMEOUT_BETWEEN_RETRIES = 1000;
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
- for(int count=0;count