Sean/ARM64
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Remove BigCharts pricing feed. Add Robinhood price feed

Browse Source
This commit is contained in:
Sean
2025-10-17 18:04:13 -04:00
parent a864a49cb3
commit 20cdbfa942
7 changed files with 678 additions and 965 deletions
Download Patch File Download Diff File Expand all files Collapse all files

491
MarketData/MarketDataLib/Helper/MarketDataHelper.cs
View File

@@ -17,6 +17,8 @@ using MarketData.DataAccess;
using MarketData.Integration;
using System.Globalization;
using MarketDataLib.Utility;
using MarketData.Configuration;
using System.Text.Json;
//Zacks Rank - Zacks
//Splits - EODDATA
@@ -76,12 +78,8 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No data returned for request {0}",strRequest));
return false;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetWorldTime Received {0} bytes of data.",httpNetResponse.ResponseString.Length));
return true;
}
catch (Exception exception)
{
@@ -94,9 +92,6 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("MarketData.GetWorldTime: Done, took {0}(ms)", profiler.End()));
}
}
// ******************************************************************************************************************************************************************************
// ************************************************************************************ P R E M A R K E T D A T A ***********************************************************
// ******************************************************************************************************************************************************************************
@@ -4574,7 +4569,7 @@ namespace MarketData.Helper
return null;
}
DateTime currentMarketDate=PremarketDA.GetLatestMarketDate();
LatestPriceDelegate[] latestPriceDelegates=new LatestPriceDelegate[]{GetLatestPriceBarChart,GetLatestPriceYahoo,GetLatestPriceBigCharts,GetLatestPriceGoogle};
LatestPriceDelegate[] latestPriceDelegates=new LatestPriceDelegate[]{GetLatestPriceBarChart,GetLatestPriceYahoo,GetLatestPriceRobinhood,GetLatestPriceGoogle};
Price latestPrice=null;
foreach(LatestPriceDelegate latestPriceDelegate in latestPriceDelegates)
{
@@ -4591,10 +4586,7 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The date retrieved for {0} via {1} does not match the current market date. Market Date: {2} Price Date: {3}",symbol,latestPriceDelegate.Method.Name,currentMarketDate.ToShortDateString(),latestPrice.Date.ToShortDateString()));
}
}
}
latestPrice=GetPriceAsOf(symbol,currentMarketDate); // try to get the price from MarketWatch historical feed using todays date
if(null!=latestPrice && latestPrice.Date.Date.Equals(currentMarketDate.Date)) return latestPrice;
}
if(null!=companyProfile && companyProfile.CanRollPrevious)
{
latestPrice=PricingMarketDataHelper.RollPriceForward(symbol);
@@ -4901,113 +4893,6 @@ namespace MarketData.Helper
}
}
// This is used in the intra-day price feed.
public static Price GetLatestPriceBigCharts(String symbol)
{
HttpNetResponse httpNetResponse=null;
MemoryStream memoryStream=null;
try
{
StringBuilder sb = new StringBuilder();
String strRequest;
if(null==symbol) return null;
sb.Append("http://bigcharts.marketwatch.com/quickchart/quickchart.asp?symb=").Append(symbol).Append("&insttype=Stock").Append(symbol);
strRequest = sb.ToString();
WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceBigCharts");
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return null;
}
Price price = new Price();
price.Symbol = symbol.ToUpper();
price.Date = DateTime.Now;
price.Source=Price.PriceSource.BigCharts;
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
memoryStream = new MemoryStream(streamBytes);
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.Load(memoryStream);
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"quickchart\"]");
if(null==table||0==table.Count)return null;
HtmlNodeCollection divSection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"maincontent\"]");
if(null==divSection)return null;
HtmlNodeCollection rows = divSection[0].SelectNodes(".//tr");
if(rows.Count<3)return null;
HtmlNode softTimeNode = htmlDocument.DocumentNode.SelectSingleNode("//*[@class=\"soft time\"]");
if(null!=softTimeNode)
{
DateTime givenDate=ConvertBigChartsDate(softTimeNode.InnerHtml);
if(!Utility.IsEpoch(givenDate))price.Date=givenDate;
}
HtmlNodeCollection dataColumns = rows[2].SelectNodes(".//td");
for(int index=0;index<dataColumns.Count;index++)
{
String innerText=Utility.RemoveControlChars(dataColumns[index].InnerText).Trim();
String[] items=innerText.Split(':');
if(null==items||2!=items.Length)continue;
if("Last".Equals(items[0])){price.Close=FeedParser.ParseValue(items[1]);price.AdjClose=price.Close;}
else if("Open".Equals(items[0]))price.Open=FeedParser.ParseValue(items[1]);
else if("High".Equals(items[0]))price.High=FeedParser.ParseValue(items[1]);
else if("Low".Equals(items[0]))price.Low=FeedParser.ParseValue(items[1]);
else if("Volume".Equals(items[0]))price.Volume=FeedParser.ParseValueLong(items[1]);
}
if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No closing price for {0}",price.Symbol));
return null;
}
if(double.IsNaN(price.Open))price.Open=0.00;
if(double.IsNaN(price.Low))price.Low=0.00;
if(double.IsNaN(price.High))price.High=0.00;
if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low)
{
price.Open=price.High=price.Low=price.Close;
}
return price;
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null;
}
finally
{
if(null!=httpNetResponse)httpNetResponse.Dispose();
if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();}
}
}
// This seems necessary on ARM64. Utility.ParseDate() works on windows with the same exact string.
private static DateTime ConvertBigChartsDate(String strDate)
{
try
{
StringBuilder sb = new StringBuilder();
if(String.IsNullOrEmpty(strDate))return Utility.Epoch;
String[] elements = strDate.Split(' ');
if(3!=elements.Length)return Utility.Epoch;
String[] dateElements = elements[0].Split('/');
if(3!=dateElements.Length)return Utility.Epoch;
sb.Append(Utility.Pad(dateElements[0],'0',2));
sb.Append("/");
sb.Append(Utility.Pad(dateElements[1],'0',2));
sb.Append("/");
sb.Append(Utility.Pad(dateElements[2],'0',2));
sb.Append(" ");
sb.Append(elements[1]);
sb.Append(" ");
sb.Append(elements[2]);
return Convert.ToDateTime(sb.ToString());
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"[ConvertBigChartsDate] Date:{strDate} {exception.ToString()}");
return Utility.Epoch;
}
}
public static Price GetLatestPriceGoogle(String symbol)
{
HttpNetResponse httpNetResponse=null;
@@ -5205,62 +5090,45 @@ namespace MarketData.Helper
return barChartValues;
}
// *******************************************************************************************************************************************************************************
// ************************************************************** H I S T O R I C A L P R I C E S - B I G C H A R T S ********************************************************
// *******************************************************************************************************************************************************************************
// ************************************************************** L A T E S T P R I C E - R O B I N H O O D ******************************************************************
// *******************************************************************************************************************************************************************************
/// <summary>
/// Gets historical pricing from BigCharts
/// This version fetches the latest price from robinhood and ensures that the pricing date is what we expect
/// </summary>
/// <param name="symbol"></param>
/// <param name="startDate">Most recent date</param>
/// <param name="endDate">Most historical date</param>
/// <returns>Prices</returns>
public static Prices GetPricesAsOf(String symbol, DateTime startDate, DateTime endDate)
/// <param name="startDate"></param>
/// <returns></returns>
public static Price GetPriceRobinHood(String symbol, DateTime startDate)
{
Prices prices = new Prices();
if (null == symbol) return null;
CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol);
if (null != companyProfile && companyProfile.FreezePricing)
try
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol));
Price price = GetLatestPriceRobinhood(symbol);
if (null == price) return null;
if (!price.Date.Date.Equals(startDate.Date))
{
MDTrace.WriteLine(LogLevel.DEBUG, $"Error, the price retrieved from Robinhood for : {symbol} is dated {price.Date.Date.ToShortDateString()} , expected {startDate.Date.ToShortDateString()}.");
return null;
}
return price;
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception : {0}", exception.ToString()));
return null;
}
DateGenerator dateGenerator = new DateGenerator();
List<DateTime> dates = dateGenerator.GenerateHistoricalDates(startDate, endDate);
for (int index = 0; index < dates.Count; index++)
{
DateTime currentDate = dates[index];
Price price = GetPriceAsOfV2(symbol, currentDate);
if (null == price)
{
price = GetPriceAsOf(symbol, currentDate);
if (null == price)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("No price for {0} on {1}", symbol, Utility.DateTimeToStringMMHDDHYYYY(currentDate)));
continue;
}
}
prices.Add(price);
}
return prices;
}
/// <summary>
/// The main BigChart Feed which is no longer working
/// This version fetches the latest price from Robinhood
/// </summary>
/// <param name="symbol"></param>
/// <param name="asOf"></param>
/// <returns></returns>
public static Price GetPriceAsOf(String symbol, DateTime asOf)
public static Price GetLatestPriceRobinhood(string symbol)
{
HttpNetResponse httpNetResponse = null;
try
{
String strRequest;
StringBuilder sb = null;
if (null == symbol) return null;
CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol);
if (null != companyProfile && companyProfile.FreezePricing)
@@ -5268,18 +5136,54 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol));
return null;
}
StringBuilder sb = new StringBuilder();
String requestSymbol = symbol;
String strRequest = default;
if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1);
sb = new StringBuilder();
sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb=");
sb.Append(requestSymbol);
sb.Append("&closeDate=");
sb.Append(asOf.Month.ToString());
sb.Append("%2F");
sb.Append(asOf.Day.ToString());
sb.Append("%2F");
sb.Append((asOf.Year - 2000).ToString());
sb.Append("&x=38&y=25");
sb.Append("https://robinhood.com/us/en/stocks/").Append(symbol).Append("/");
strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, $"{strRequest}");
httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest);
if (!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
String strInstrumentId = Utility.BetweenString(httpNetResponse.ResponseString, "<meta content=\"robinhood://instrument?id=", "\"");
if (null == strInstrumentId) return null;
List<String> spanSections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString, "span");
String strOpen = Sections.ScanSectionsFindStartWithReturnIndexAfter(spanSections, "<span class=\"css-eimp3v\">Open price</span>", 1);
if (null == strOpen) return null;
strOpen = Utility.BetweenString(strOpen, ">", "<");
if (null == strOpen) return null;
if (strOpen.Equals("???")) strOpen = null;
String strHigh = Sections.ScanSectionsFindStartWithReturnIndexAfter(spanSections, "<span class=\"css-eimp3v\">High today</span>", 1);
if (null == strHigh) return null;
strHigh = Utility.BetweenString(strHigh, ">", "<");
if (null == strHigh) return null;
if (strHigh.Equals("???")) strHigh = null;
String strLow = Sections.ScanSectionsFindStartWithReturnIndexAfter(spanSections, "<span class=\"css-eimp3v\">Low today</span>", 1);
if (null == strLow) return null;
strLow = Utility.BetweenString(strLow, ">", "<");
if (null == strLow) return null;
if (strLow.Equals("???")) strLow = null;
String strVolume = Sections.ScanSectionsFindStartWithReturnIndexAfter(spanSections, "<span class=\"css-v72tci\">Volume</span>", 1);
if (null == strVolume) return null;
strVolume = Utility.BetweenString(strVolume, ">", "<");
if (null == strVolume) return null;
if (strVolume.Equals("???")) strVolume = null;
// Fetch the current price
httpNetResponse.Dispose();
sb = new StringBuilder();
sb.Append("https://bonfire.robinhood.com/instruments/").Append(strInstrumentId).Append("/detail-page-live-updating-data/?display_span=day&hide_extended_hours=false");
strRequest = sb.ToString();
httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest);
if (!httpNetResponse.Success)
@@ -5287,164 +5191,43 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
MemoryStream memoryStream = new MemoryStream(streamBytes);
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.Load(memoryStream);
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"historicalquote fatbottomed\"]");
if (null == tables || 0 == tables.Count) return null;
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
if (rows.Count < 7) return null;
JsonDocument jsonDocument = JsonDocument.Parse(httpNetResponse.ResponseString);
JsonElement root = jsonDocument.RootElement;
JsonElement chartSection = root.GetProperty("chart_section");
JsonElement quoteSection = chartSection.GetProperty("quote");
String strLastTradePrice = quoteSection.GetProperty("last_trade_price").GetString();
String strUpdatedAt = quoteSection.GetProperty("updated_at").GetString();
String strSymbol = quoteSection.GetProperty("symbol").GetString();
DateTime lastUpdatedAt = DateTime.Parse(strUpdatedAt, null, System.Globalization.DateTimeStyles.RoundtripKind);
Price price = new Price();
price.Source = Price.PriceSource.BigCharts;
price.Symbol = symbol.ToUpper();
price.Date = FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText);
price.Close = FeedParser.ParseValueFromMarketWatch("Closing Price:", rows[2].InnerText);
price.AdjClose = price.Close;
price.Open = FeedParser.ParseValueFromMarketWatch("Open:", rows[3].InnerText);
price.High = FeedParser.ParseValueFromMarketWatch("High:", rows[4].InnerText);
price.Low = FeedParser.ParseValueFromMarketWatch("Low:", rows[5].InnerText);
price.Volume = FeedParser.ParseLongValueFromMarketWatch("Volume:", rows[6].InnerText);
if (!(price.Date.Date.Equals(asOf.Date.Date)))
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}", symbol, price.Date.ToShortDateString(), asOf.ToShortDateString()));
return null;
}
price.Symbol = strSymbol;
price.Date = lastUpdatedAt;
price.Source = Price.PriceSource.Robinhood;
price.Close = price.AdjClose = FeedParser.ParseValueDouble(strLastTradePrice);
price.Open = Utility.ParseCurrency(strOpen);
price.High = Utility.ParseCurrency(strHigh);
price.Low = Utility.ParseCurrency(strLow);
price.Volume = (long)FeedParser.ParseValueDouble(strVolume);
CheckPrice(price);
if (!price.IsValid) return null;
if (!price.Symbol.Equals(symbol)) return null;
MDTrace.WriteLine(LogLevel.DEBUG, $"Got price from Robinhood for {symbol} : {price.ToString()}");
return price;
}
catch (Exception)
{
return null;
}
finally
{
if (null != httpNetResponse) httpNetResponse.Dispose();
}
}
/// <summary>
/// This is a modified version of the above query. It uses a cookie collection that contains the datadome cookie.
/// If this query starts to fail then you should try the query in developer tools in the chromium browser and capture the
/// datadome value and update it here in the GetCookieCollection() below.
/// </summary>
/// <param name="symbol"></param>
/// <param name="asOf"></param>
/// <returns></returns>
public static Price GetPriceAsOfV2(String symbol, DateTime asOf)
{
HttpNetResponse httpNetResponse = null;
try
{
String strRequest;
StringBuilder sb = null;
if (null == symbol) return null;
CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol);
if (null != companyProfile && companyProfile.FreezePricing)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol));
return null;
}
String requestSymbol = symbol;
if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1);
sb = new StringBuilder();
sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb=");
sb.Append(requestSymbol);
sb.Append("&closeDate=");
sb.Append(asOf.Month.ToString());
sb.Append("%2F");
sb.Append(asOf.Day.ToString());
sb.Append("%2F");
sb.Append((asOf.Year - 2000).ToString());
sb.Append("&x=38&y=25");
strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,$"{strRequest}");
CookieCollection cookieCollection = GetCookieCollection("www.marketwatch.com");
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest,cookieCollection,new Uri("https://www.marketwatch.com/investing"));
if (!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
MemoryStream memoryStream = new MemoryStream(streamBytes);
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.Load(memoryStream);
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"table table--overflow align--center\"]");
if (null == tables || 0 == tables.Count) return null;
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
if (rows.Count < 2) return null;
Prices prices = new Prices();
for (int rowIndex = 1; rowIndex < rows.Count; rowIndex++)
{
HtmlNodeCollection data = rows[rowIndex].SelectNodes(".//td");
if (data.Count != 6)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the correct number of data elements (expected 6 data elements).", symbol));
return null;
}
Price price = new Price();
price.Source = Price.PriceSource.BigCharts;
price.Symbol = symbol.ToUpper();
String[] dateElements = data[0].InnerText.Trim().Split(" ");
price.Date = Utility.ParseDate(dateElements[0].Replace("\n", null));
price.Open = Utility.ParseCurrency(data[1].InnerText.Trim().Replace("\n", null));
price.High = Utility.ParseCurrency(data[2].InnerText.Trim().Replace("\n", null));
price.Low = Utility.ParseCurrency(data[3].InnerText.Trim().Replace("\n", null));
price.Close = price.AdjClose = Utility.ParseCurrency(data[4].InnerText.Trim().Replace("\n", null));
price.Volume = FeedParser.ParseValueLong(data[5].InnerText.Trim());
prices.Add(price);
}
Price selectedPrice = prices.Where(x => x.Date.Date.Equals(asOf.Date.Date)).FirstOrDefault();
if (null == selectedPrice)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("No price for {0} on {1}", symbol, asOf.ToShortDateString()));
return null;
}
return selectedPrice;
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception : {0}", exception.ToString()));
return null;
return null;
}
finally
{
if (null != httpNetResponse) httpNetResponse.Dispose();
}
}
private static CookieCollection GetCookieCollection(String cookieDomain)
{
String[] cookies = {
"datadome=VLSWQ1PWL4SV2rljkJEV7GUOK3T11Sm73RP17IwDEtRzfWHykYwy7ZoeSPrjwmbwvGRAenazmLVTpyCB6Yqlw6vUL6Wbqq4zML5DGuHLxNU4LgAQ7Ko9tztglqjIGLZE",
};
CookieCollection cookieCollection = new CookieCollection();
for (int index = 0; index < cookies.Count(); index++)
{
String strCookie = cookies[index];
if (strCookie.EndsWith(";")) strCookie = strCookie.Substring(0, strCookie.Length - 1);
String[] pairs = strCookie.Split('=');
cookieCollection.Add(new Cookie()
{
Name = pairs[0],
Value = pairs[1],
Domain = cookieDomain
});
}
return cookieCollection;
}
//********************************************************************************************************************************************************************************************************
//********************************************************************************************************************************************************************************************************
// ******************************************************************************** H I S T O R I C A L P R I C I N G Y A H O O *********************************************************************
//********************************************************************************************************************************************************************************************************
@@ -5737,43 +5520,57 @@ namespace MarketData.Helper
}
}
public static List<String> FindItemsInTag(String item, List<List<String>> itemTags)
{
List<String> foundItems = new List<String>();
if (null == itemTags || 0 == itemTags.Count) return null;
for (int index = 0; index < itemTags.Count; index++)
{
List<String> itemTag = itemTags[index];
if (itemTag[0].Equals(item)) foundItems.Add(itemTag[1]);
}
return foundItems;
}
private static DateTime ProcessNASDAQRelativeDate(String relativeDate)
{
DateTime referenceDate=DateTime.Now;
string[] items=relativeDate.Split(' ');
if(items.Length<2)return Utility.Epoch;
if("days".Equals(items[1])||"day".Equals(items[1]))
{
TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0,0,0);
referenceDate-=timeSpan;
}
else if("hours".Equals(items[1])||"hour".Equals(items[1]))
{
TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0);
referenceDate-=timeSpan;
}
else if("mins".Equals(items[1]) || "minutes".Equals(items[1]))
{
TimeSpan timeSpan=new TimeSpan(0,int.Parse(items[0]),0);
referenceDate-=timeSpan;
}
else if( (new string[]{"Jul","Aug","Sep","Oct","Nov","Dec","Jan","Feb","Mar","Apr","May","Jun"}).Any(x=>x.Equals(items[0])))
{
DateTime now=DateTime.Now;
StringBuilder sb=new StringBuilder();
sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1],'0',2)).Append(" ").Append(referenceDate.Year);
referenceDate=Utility.ParseDate(sb.ToString());
if(referenceDate>now)
{
sb=new StringBuilder();
sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1],'0',2)).Append(" ").Append(referenceDate.Year-1);
referenceDate=Utility.ParseDate(sb.ToString());
}
}
else referenceDate=Utility.Epoch;
if(referenceDate>DateTime.Now)referenceDate=Utility.Epoch;
return referenceDate;
}
{
DateTime referenceDate = DateTime.Now;
string[] items = relativeDate.Split(' ');
if (items.Length < 2) return Utility.Epoch;
if ("days".Equals(items[1]) || "day".Equals(items[1]))
{
TimeSpan timeSpan = new TimeSpan(int.Parse(items[0]), 0, 0, 0, 0);
referenceDate -= timeSpan;
}
else if ("hours".Equals(items[1]) || "hour".Equals(items[1]))
{
TimeSpan timeSpan = new TimeSpan(int.Parse(items[0]), 0, 0);
referenceDate -= timeSpan;
}
else if ("mins".Equals(items[1]) || "minutes".Equals(items[1]))
{
TimeSpan timeSpan = new TimeSpan(0, int.Parse(items[0]), 0);
referenceDate -= timeSpan;
}
else if ((new string[] { "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", "Jan", "Feb", "Mar", "Apr", "May", "Jun" }).Any(x => x.Equals(items[0])))
{
DateTime now = DateTime.Now;
StringBuilder sb = new StringBuilder();
sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1], '0', 2)).Append(" ").Append(referenceDate.Year);
referenceDate = Utility.ParseDate(sb.ToString());
if (referenceDate > now)
{
sb = new StringBuilder();
sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1], '0', 2)).Append(" ").Append(referenceDate.Year - 1);
referenceDate = Utility.ParseDate(sb.ToString());
}
}
else referenceDate = Utility.Epoch;
if (referenceDate > DateTime.Now) referenceDate = Utility.Epoch;
return referenceDate;
}
// ****************************************************************************************************************************************************************************************
// **************************************************************************** H E L P E R M E T H O D S *******************************************************************************

965
MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs
View File

File diff suppressed because it is too large Load Diff

6
MarketData/MarketDataLib/MarketDataModel/Prices.cs
View File

@@ -316,7 +316,7 @@ namespace MarketData.MarketDataModel
// ***************************************************************************************************************************************************************************
public class Price
{
public enum PriceSource{Other=0,BigCharts=1,Yahoo=2,Fidelity=3,Google=4,BarChart=5}; // !!IMPORTANT only add to this list.
public enum PriceSource{Other=0,BigCharts=1,Yahoo=2,Fidelity=3,Google=4,BarChart=5,Robinhood=6}; // !!IMPORTANT only add to this list.
private String symbol;
private DateTime date;
private double open;
@@ -379,7 +379,9 @@ namespace MarketData.MarketDataModel
return "Google";
case PriceSource.BarChart :
return "BarChart";
default :
case PriceSource.Robinhood :
return "Robinhood";
default:
return Constants.CONST_QUESTION;
}
}

145
MarketData/MarketDataLib/Utility/FeedParser.cs
View File

@@ -14,16 +14,16 @@ namespace MarketData.Utils
private FeedParser()
{
}
public static TimeSeriesCollection ParseTimeSeries(String symbol, List<DateTime> reportDates, TimeSeriesElement.ElementType elementType,String strLine)
public static TimeSeriesCollection ParseTimeSeries(String symbol, List<DateTime> reportDates, TimeSeriesElement.ElementType elementType, String strLine)
{
TimeSeriesCollection timeSeriesCollection = new TimeSeriesCollection();
Dictionary<DateTime,DateTime> dates = new Dictionary<DateTime,DateTime>();
Dictionary<DateTime, DateTime> dates = new Dictionary<DateTime, DateTime>();
try
{
if (null == reportDates || 0 == reportDates.Count) return null;
String[] lineItems = CSVLineHelper.ParseLine(strLine);
if (null == lineItems) return null;
for (int index = 0; index < reportDates.Count;index++)
for (int index = 0; index < reportDates.Count; index++)
{
TimeSeriesElement timeSeriesElement = new TimeSeriesElement();
timeSeriesElement.Type = elementType;
@@ -46,11 +46,11 @@ namespace MarketData.Utils
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
MDTrace.WriteLine(LogLevel.DEBUG, exception.ToString());
return null;
}
}
//"Book Value Per Share USD,3.31,3.22,2.64,2.58,2.28,2.10,1.39,2.64,3.88,3.94,3.94"
//"Book Value Per Share USD,3.31,3.22,2.64,2.58,2.28,2.10,1.39,2.64,3.88,3.94,3.94"
public static List<DateTime> ParsePeriodEndingDates(String periodEnding)
{
List<DateTime> periodEndingDates = new List<DateTime>();
@@ -58,7 +58,7 @@ namespace MarketData.Utils
periodEnding = periodEnding.Replace("Period Ending", "");
int length = periodEnding.Length;
int index = 0;
while (index<length)
while (index < length)
{
String strMonth = periodEnding.Substring(index, 3);
char ch;
@@ -69,7 +69,7 @@ namespace MarketData.Utils
sb = new StringBuilder();
for (int yIndex = 0; yIndex < 5; yIndex++) sb.Append(periodEnding[index++]);
String strYear = sb.ToString().Trim();
DateTime periodEndingDate=DateTime.ParseExact(strMonth+Utility.Pad(strDay,'0',2)+strYear, "MMMddyyyy", new System.Globalization.CultureInfo("en-US"));
DateTime periodEndingDate = DateTime.ParseExact(strMonth + Utility.Pad(strDay, '0', 2) + strYear, "MMMddyyyy", new System.Globalization.CultureInfo("en-US"));
periodEndingDates.Add(periodEndingDate);
}
return periodEndingDates;
@@ -78,11 +78,11 @@ namespace MarketData.Utils
{
try
{
if(strText.Contains("Est"))
if (strText.Contains("Est"))
{
strText=Utility.RemoveAfter(strText,'-');
strText = Utility.RemoveAfter(strText, '-');
strText = strText.Trim();
strText = strText + " "+DateTime.Now.Year.ToString();
strText = strText + " " + DateTime.Now.Year.ToString();
string[] items = strText.Split(' ');
strText = items[0] + "-" + Utility.Pad(items[1], '0', 2) + "-" + items[2];
return DateTime.ParseExact(strText, "MMM-dd-yyyy", new System.Globalization.CultureInfo("en-US"));
@@ -93,22 +93,22 @@ namespace MarketData.Utils
return DateTime.ParseExact(strText, "dd-MMM-yy", new System.Globalization.CultureInfo("en-US"));
}
}
catch(Exception)
catch (Exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[ParseValueDateTime] Error parsing date '"+strText+"'");
MDTrace.WriteLine(LogLevel.DEBUG, "[ParseValueDateTime] Error parsing date '" + strText + "'");
return DateTime.Parse("01-01-0001");
}
}
// "Middleby Corp. Fri, Aug 2, 2013"
// "Middleby Corp. Fri, Aug 2, 2013"
public static DateTime ParseValueDateTimeMonthFormatFromMarketWatch(String strItem)
{
strItem = Utility.RemoveControlChars(strItem.Trim());
String[] strItems = strItem.Split(' ');
if (strItems.Length < 3) return DateTime.Parse("01-01-0001"); ;
if (strItems.Length < 3) return DateTime.Parse("01-01-0001");
String strDate = strItems[strItems.Length - 3] + " " + strItems[strItems.Length - 2] + " " + strItems[strItems.Length - 1];
return ParseValueDateTimeMonthFormat(strDate);
}
public static double ParseValueFromMarketWatch(String identifier,String strItem)
public static double ParseValueFromMarketWatch(String identifier, String strItem)
{
try
{
@@ -146,15 +146,15 @@ namespace MarketData.Utils
{
try
{
if(null==strText)return Utility.Epoch;
if (null == strText) return Utility.Epoch;
String[] strArray = strText.Split(' ');
if (3 != strArray.Length) return DateTime.Parse("01-01-0001");
strArray[1] = strArray[1].Replace(",", "");
return ParseValueDateTime(Utility.Pad(strArray[1],'0',2)+"-"+strArray[0]+"-"+strArray[2].Substring(2,2));
return ParseValueDateTime(Utility.Pad(strArray[1], '0', 2) + "-" + strArray[0] + "-" + strArray[2].Substring(2, 2));
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[ParseValueDateTimeMonthFormat] Error parsing date '" + strText + "', "+exception.ToString());
MDTrace.WriteLine(LogLevel.DEBUG, "[ParseValueDateTimeMonthFormat] Error parsing date '" + strText + "', " + exception.ToString());
return DateTime.Parse("01-01-0001");
}
}
@@ -164,23 +164,23 @@ namespace MarketData.Utils
{
try
{
if(null==strText)return DateTime.Parse("01-01-0001");
if (null == strText) return DateTime.Parse("01-01-0001");
if(strText.EndsWith("am") || strText.EndsWith("pm"))
if (strText.EndsWith("am") || strText.EndsWith("pm"))
{
return DateTime.Now;
}
else if(strText.Contains("."))
else if (strText.Contains("."))
{
strText=strText.Replace(".",null);
String[] subItems=strText.Split(' ');
strText = strText.Replace(".", null);
String[] subItems = strText.Split(' ');
DateTime currentDate = DateTime.Now;
StringBuilder sb=new StringBuilder();
StringBuilder sb = new StringBuilder();
sb.Append(subItems[0]).Append(" ");
sb.Append(subItems[1]).Append(",").Append(" ");
sb.Append(currentDate.Year.ToString());
DateTime resultingDate = ParseValueDateTimeMonthFormat(sb.ToString());
if(resultingDate>currentDate)resultingDate = new DateTime(resultingDate.Year-1,resultingDate.Month, resultingDate.Day);
if (resultingDate > currentDate) resultingDate = new DateTime(resultingDate.Year - 1, resultingDate.Month, resultingDate.Day);
return resultingDate;
}
else
@@ -201,16 +201,16 @@ namespace MarketData.Utils
{
try
{
if(null==strText)return Utility.Epoch;
if (null == strText) return Utility.Epoch;
String[] strArray = strText.Split(' ');
if (3 > strArray.Length) return DateTime.Parse("01-01-0001");
strArray[1] = strArray[1].Replace(",", null);
strArray[0] = strArray[0].Replace(".", null);
return ParseValueDateTime(Utility.Pad(strArray[1],'0',2)+"-"+strArray[0]+"-"+strArray[2].Substring(2,2));
return ParseValueDateTime(Utility.Pad(strArray[1], '0', 2) + "-" + strArray[0] + "-" + strArray[2].Substring(2, 2));
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[ParseValueDateTimeMonthFormat] Error parsing date '" + strText + "', "+exception.ToString());
MDTrace.WriteLine(LogLevel.DEBUG, "[ParseValueDateTimeMonthFormat] Error parsing date '" + strText + "', " + exception.ToString());
return DateTime.Parse("01-01-0001");
}
}
@@ -219,33 +219,33 @@ namespace MarketData.Utils
{
try
{
DateTime today=DateTime.Now;
if(strDate.Contains("day"))
DateTime today = DateTime.Now;
if (strDate.Contains("day"))
{
DateGenerator dateGenerator=new DateGenerator();
String[] items=strDate.Split(' ');
DateTime relativeDate=dateGenerator.GenerateHistoricalDate(today,int.Parse(items[0])+1);
DateGenerator dateGenerator = new DateGenerator();
String[] items = strDate.Split(' ');
DateTime relativeDate = dateGenerator.GenerateHistoricalDate(today, int.Parse(items[0]) + 1);
return relativeDate;
}
else if(strDate.Contains("hour"))
else if (strDate.Contains("hour"))
{
String[] items=strDate.Split(' ');
TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0);
DateTime relativeDate=today.Subtract(timeSpan);
String[] items = strDate.Split(' ');
TimeSpan timeSpan = new TimeSpan(int.Parse(items[0]), 0, 0);
DateTime relativeDate = today.Subtract(timeSpan);
return relativeDate;
}
else if(strDate.Contains("minute"))
else if (strDate.Contains("minute"))
{
String[] items=strDate.Split(' ');
TimeSpan timeSpan=new TimeSpan(0,int.Parse(items[0]),0);
DateTime relativeDate=today.Subtract(timeSpan);
String[] items = strDate.Split(' ');
TimeSpan timeSpan = new TimeSpan(0, int.Parse(items[0]), 0);
DateTime relativeDate = today.Subtract(timeSpan);
return relativeDate;
}
return ParseValueDateTimeMonthFormat(strDate);
}
catch(Exception)
catch (Exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Unable to parse date '{0}'",strDate));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Unable to parse date '{0}'", strDate));
return null;
}
}
@@ -256,15 +256,15 @@ namespace MarketData.Utils
try
{
if (null == strText || 0 == strText.Length) return double.NaN;
strText=strText.Trim();
if(strText.StartsWith("--"))return double.NaN;
if(strText.Equals("N/A"))return double.NaN;
strText=strText.Replace("%", "");
strText = strText.Trim();
if (strText.StartsWith("--")) return double.NaN;
if (strText.Equals("N/A")) return double.NaN;
strText = strText.Replace("%", "");
strText = strText.Replace("$", "");
if (strText.Contains("("))
{
strText=strText.Replace("(", "");
strText=strText.Replace(")", "");
strText = strText.Replace("(", "");
strText = strText.Replace(")", "");
multiplier = -1.00;
}
if (strText.Equals("-")) return double.NaN;
@@ -286,7 +286,7 @@ namespace MarketData.Utils
value = double.Parse(strText);
value *= 1000000;
}
else if (strText[strText.Length - 1].Equals('K')||strText[strText.Length - 1].Equals('k'))
else if (strText[strText.Length - 1].Equals('K') || strText[strText.Length - 1].Equals('k'))
{
strText = strText.Replace("K", "");
strText = strText.Replace("k", "");
@@ -299,7 +299,7 @@ namespace MarketData.Utils
}
catch (Exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[ParseValue] Error parsing '"+strText+"'");
MDTrace.WriteLine(LogLevel.DEBUG, "[ParseValue] Error parsing '" + strText + "'");
return double.NaN;
}
}
@@ -309,8 +309,8 @@ namespace MarketData.Utils
long value;
try
{
if(null==strText)return 0;
strText=strText.Replace("%","");
if (null == strText) return 0;
strText = strText.Replace("%", "");
strText = strText.Replace(",", "");
if (strText.Equals("-")) return 0;
if (strText[strText.Length - 1].Equals('B'))
@@ -336,9 +336,46 @@ namespace MarketData.Utils
}
catch (Exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[ParseValueLong] Error parsing '" + strText + "'");
MDTrace.WriteLine(LogLevel.DEBUG, "[ParseValueLong] Error parsing '" + strText + "'");
return 0;
}
}
public static double ParseValueDouble(String strText)
{
double value;
try
{
if(null==strText)return 0;
strText=strText.Replace("%","");
strText = strText.Replace(",", "");
if (strText.Equals("-")) return 0;
if (strText[strText.Length - 1].Equals('B'))
{
strText = strText.Replace("B", "");
value = double.Parse(strText);
value *= 1000000000;
}
else if (strText[strText.Length - 1].Equals('M'))
{
strText = strText.Replace("M", "");
value = double.Parse(strText);
value *= 1000000;
}
else if (strText[strText.Length - 1].Equals('K'))
{
strText = strText.Replace("K", "");
value = double.Parse(strText);
value *= 1000;
}
else value = double.Parse(strText);
return value;
}
catch (Exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[ParseValueDouble] Error parsing '" + strText + "'");
return 0;
}
}
}
}

15
MarketData/MarketDataLib/Utility/Sections.cs
View File

@@ -36,7 +36,7 @@ namespace MarketDataLib.Utility
while (true)
{
String itemsInSection = GetItemsInSection(strInput, sectionName, ref searchIndex);
if(null==itemsInSection)break;
if (null == itemsInSection) break;
sectionItems.Add(itemsInSection);
searchIndex++;
}
@@ -106,18 +106,19 @@ namespace MarketDataLib.Utility
}
return false;
}
public static bool FindInSectionsReverse(List<String> sections,String startsWith,int startingIndex,ref int indexOfItem,bool findExact=true)
public static bool FindInSectionsReverse(List<String> sections, String startsWith, int startingIndex, ref int indexOfItem, bool findExact = true)
{
for(int index=startingIndex;index>=0;index--)
for (int index = startingIndex; index >= 0; index--)
{
if(!findExact &&sections[index].StartsWith(startsWith))
if (!findExact && sections[index].StartsWith(startsWith))
{
indexOfItem=index;
indexOfItem = index;
return true;
}
else if(sections[index].Equals(startsWith))
else if (sections[index].Equals(startsWith))
{
indexOfItem=index;
indexOfItem = index;
return true;
}
}

3
MarketData/MarketDataLib/Utility/Utility.cs
View File

@@ -609,7 +609,8 @@ namespace MarketData.Utils
public static DateTime ParseDate(String strDate)
{
System.Globalization.CultureInfo cultureInfo = new System.Globalization.CultureInfo("en-US");
String[] formats=new[] { "MM/dd/yyyy HH:MM tt", "yyyy-MM-dd","yy-MM-dd","yyyy-MM-dd hh:mm:ss tt","dddd, MMMM dd","MMM dd yyyy","yyyy-MM","ddd, MMM. d","ddd, MMM. dd","yyyy/MM/dd","M-d-yyyy","dd-MM-yyyy","MM-dd-yyyy","M.d.yyyy","dd.MM.yyyy","MM.dd.yyyy","yyyyMMdd" }.Union(cultureInfo.DateTimeFormat.GetAllDateTimePatterns()).ToArray();
String[] formats=new[] { "MMMM dd,yyyy" ,"yyyy-MM-dd HH:mm:ss","MM/dd/yyyy HH:MM tt", "yyyy-MM-dd","yy-MM-dd","yyyy-MM-dd hh:mm:ss tt","dddd, MMMM dd","MMM dd yyyy","yyyy-MM","ddd, MMM. d","ddd, MMM. dd","yyyy/MM/dd","M-d-yyyy","dd-MM-yyyy","MM-dd-yyyy","M.d.yyyy","dd.MM.yyyy","MM.dd.yyyy","yyyyMMdd" }.Union(cultureInfo.DateTimeFormat.GetAllDateTimePatterns()).ToArray();
strDate = strDate.Trim();
DateTime dateTime=DateTime.ParseExact(strDate, formats, cultureInfo, DateTimeStyles.AssumeLocal);
return dateTime;