From 20cdbfa942dc56edb85160733cb8d080d4cc0381 Mon Sep 17 00:00:00 2001 From: Sean Date: Fri, 17 Oct 2025 18:04:13 -0400 Subject: [PATCH] Remove BigCharts pricing feed. Add Robinhood price feed --- MarketData/MarketData/Services/MainService.cs | 18 +- .../MarketDataLib/Helper/MarketDataHelper.cs | 491 +++------ .../Helper/PricingMarketDataHelper.cs | 965 ++++++++---------- .../MarketDataLib/MarketDataModel/Prices.cs | 6 +- .../MarketDataLib/Utility/FeedParser.cs | 145 ++- MarketData/MarketDataLib/Utility/Sections.cs | 15 +- MarketData/MarketDataLib/Utility/Utility.cs | 3 +- 7 files changed, 678 insertions(+), 965 deletions(-) diff --git a/MarketData/MarketData/Services/MainService.cs b/MarketData/MarketData/Services/MainService.cs index 4606064..6aadef7 100755 --- a/MarketData/MarketData/Services/MainService.cs +++ b/MarketData/MarketData/Services/MainService.cs @@ -557,8 +557,8 @@ namespace MarketData.Services // Here is the start of the real workers ThreadPool.QueueUserWorkItem(delegate { - UpdatePricesBigCharts(startDate); // bigcharts.marketwatch.com UpdatePricesYahooSweep(startDate); // The sweep variation of the method is intended to be used after the BigCharts update because the sweep will take pricing_source into consideration when fetching prices. + UpdatePricesRobinhood(startDate); // UpdatePricesBarChartSweep(startDate); // barchart. This sweep will pull any prices we failed to retrieve from BigCharts resetEvents[STAGE_1].Set(); SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_1 UPDATEPRICESBIGCHARTS/YAHOO done.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); @@ -647,22 +647,22 @@ namespace MarketData.Services MDTrace.WriteLine(LogLevel.DEBUG,$"Done."); } - public static void UpdatePricesBigCharts(DateTime startDate) - { - PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper(); - pricingMarketDataHelper.UpdatePricesBigCharts(startDate); - } - public static void UpdatePricesYahooSweep(DateTime startDate) { PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper(); pricingMarketDataHelper.UpdatePricesYahooSweep(startDate); } - public static void UpdatePricesBarChartSweep(DateTime startDate) + public static void UpdatePricesRobinhood(DateTime startDate) { PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper(); - pricingMarketDataHelper.UpdatePricesBarChartSweep(startDate); + pricingMarketDataHelper.UpdatePricesRobinhood(startDate); + } + + public static void UpdatePricesBarChartSweep(DateTime startDate) + { + PricingMarketDataHelper pricingMarketDataHelper = new PricingMarketDataHelper(); + pricingMarketDataHelper.UpdatePricesBarChartSweep(startDate); } public static void DeletePriceWatchList(String watchListName, String strDate) diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs index 0867e85..bf2f13c 100755 --- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs @@ -17,6 +17,8 @@ using MarketData.DataAccess; using MarketData.Integration; using System.Globalization; using MarketDataLib.Utility; +using MarketData.Configuration; +using System.Text.Json; //Zacks Rank - Zacks //Splits - EODDATA @@ -76,12 +78,8 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No data returned for request {0}",strRequest)); return false; } - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetWorldTime Received {0} bytes of data.",httpNetResponse.ResponseString.Length)); - - return true; - } catch (Exception exception) { @@ -94,9 +92,6 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG, String.Format("MarketData.GetWorldTime: Done, took {0}(ms)", profiler.End())); } } - - - // ****************************************************************************************************************************************************************************** // ************************************************************************************ P R E M A R K E T D A T A *********************************************************** // ****************************************************************************************************************************************************************************** @@ -4574,7 +4569,7 @@ namespace MarketData.Helper return null; } DateTime currentMarketDate=PremarketDA.GetLatestMarketDate(); - LatestPriceDelegate[] latestPriceDelegates=new LatestPriceDelegate[]{GetLatestPriceBarChart,GetLatestPriceYahoo,GetLatestPriceBigCharts,GetLatestPriceGoogle}; + LatestPriceDelegate[] latestPriceDelegates=new LatestPriceDelegate[]{GetLatestPriceBarChart,GetLatestPriceYahoo,GetLatestPriceRobinhood,GetLatestPriceGoogle}; Price latestPrice=null; foreach(LatestPriceDelegate latestPriceDelegate in latestPriceDelegates) { @@ -4591,10 +4586,7 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The date retrieved for {0} via {1} does not match the current market date. Market Date: {2} Price Date: {3}",symbol,latestPriceDelegate.Method.Name,currentMarketDate.ToShortDateString(),latestPrice.Date.ToShortDateString())); } } - } - latestPrice=GetPriceAsOf(symbol,currentMarketDate); // try to get the price from MarketWatch historical feed using todays date - if(null!=latestPrice && latestPrice.Date.Date.Equals(currentMarketDate.Date)) return latestPrice; - + } if(null!=companyProfile && companyProfile.CanRollPrevious) { latestPrice=PricingMarketDataHelper.RollPriceForward(symbol); @@ -4901,113 +4893,6 @@ namespace MarketData.Helper } } -// This is used in the intra-day price feed. - public static Price GetLatestPriceBigCharts(String symbol) - { - HttpNetResponse httpNetResponse=null; - MemoryStream memoryStream=null; - try - { - StringBuilder sb = new StringBuilder(); - String strRequest; - if(null==symbol) return null; - sb.Append("http://bigcharts.marketwatch.com/quickchart/quickchart.asp?symb=").Append(symbol).Append("&insttype=Stock").Append(symbol); - strRequest = sb.ToString(); - - WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceBigCharts"); - httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); - if(!httpNetResponse.Success) - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); - return null; - } - Price price = new Price(); - price.Symbol = symbol.ToUpper(); - price.Date = DateTime.Now; - price.Source=Price.PriceSource.BigCharts; - byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); - memoryStream = new MemoryStream(streamBytes); - HtmlDocument htmlDocument = new HtmlDocument(); - htmlDocument.Load(memoryStream); - HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"quickchart\"]"); - if(null==table||0==table.Count)return null; - HtmlNodeCollection divSection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"maincontent\"]"); - if(null==divSection)return null; - HtmlNodeCollection rows = divSection[0].SelectNodes(".//tr"); - if(rows.Count<3)return null; - HtmlNode softTimeNode = htmlDocument.DocumentNode.SelectSingleNode("//*[@class=\"soft time\"]"); - if(null!=softTimeNode) - { - DateTime givenDate=ConvertBigChartsDate(softTimeNode.InnerHtml); - if(!Utility.IsEpoch(givenDate))price.Date=givenDate; - } - HtmlNodeCollection dataColumns = rows[2].SelectNodes(".//td"); - for(int index=0;index - /// Gets historical pricing from BigCharts + /// This version fetches the latest price from robinhood and ensures that the pricing date is what we expect /// /// - /// Most recent date - /// Most historical date - /// Prices - public static Prices GetPricesAsOf(String symbol, DateTime startDate, DateTime endDate) + /// + /// + public static Price GetPriceRobinHood(String symbol, DateTime startDate) { - Prices prices = new Prices(); - - if (null == symbol) return null; - CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); - if (null != companyProfile && companyProfile.FreezePricing) + try { - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); + Price price = GetLatestPriceRobinhood(symbol); + if (null == price) return null; + if (!price.Date.Date.Equals(startDate.Date)) + { + MDTrace.WriteLine(LogLevel.DEBUG, $"Error, the price retrieved from Robinhood for : {symbol} is dated {price.Date.Date.ToShortDateString()} , expected {startDate.Date.ToShortDateString()}."); + return null; + } + return price; + } + catch (Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception : {0}", exception.ToString())); return null; } - DateGenerator dateGenerator = new DateGenerator(); - List dates = dateGenerator.GenerateHistoricalDates(startDate, endDate); - for (int index = 0; index < dates.Count; index++) - { - DateTime currentDate = dates[index]; - Price price = GetPriceAsOfV2(symbol, currentDate); - if (null == price) - { - price = GetPriceAsOf(symbol, currentDate); - if (null == price) - { - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("No price for {0} on {1}", symbol, Utility.DateTimeToStringMMHDDHYYYY(currentDate))); - continue; - } - } - prices.Add(price); - } - return prices; } /// - /// The main BigChart Feed which is no longer working + /// This version fetches the latest price from Robinhood /// /// - /// /// - public static Price GetPriceAsOf(String symbol, DateTime asOf) + public static Price GetLatestPriceRobinhood(string symbol) { HttpNetResponse httpNetResponse = null; try { - String strRequest; - StringBuilder sb = null; - if (null == symbol) return null; CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); if (null != companyProfile && companyProfile.FreezePricing) @@ -5268,18 +5136,54 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); return null; } + + StringBuilder sb = new StringBuilder(); String requestSymbol = symbol; + String strRequest = default; if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1); sb = new StringBuilder(); - sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb="); - sb.Append(requestSymbol); - sb.Append("&closeDate="); - sb.Append(asOf.Month.ToString()); - sb.Append("%2F"); - sb.Append(asOf.Day.ToString()); - sb.Append("%2F"); - sb.Append((asOf.Year - 2000).ToString()); - sb.Append("&x=38&y=25"); + sb.Append("https://robinhood.com/us/en/stocks/").Append(symbol).Append("/"); + strRequest = sb.ToString(); + MDTrace.WriteLine(LogLevel.DEBUG, $"{strRequest}"); + httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest); + if (!httpNetResponse.Success) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); + return null; + } + String strInstrumentId = Utility.BetweenString(httpNetResponse.ResponseString, " spanSections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString, "span"); + + String strOpen = Sections.ScanSectionsFindStartWithReturnIndexAfter(spanSections, "Open price", 1); + if (null == strOpen) return null; + strOpen = Utility.BetweenString(strOpen, ">", "<"); + if (null == strOpen) return null; + if (strOpen.Equals("???")) strOpen = null; + + String strHigh = Sections.ScanSectionsFindStartWithReturnIndexAfter(spanSections, "High today", 1); + if (null == strHigh) return null; + strHigh = Utility.BetweenString(strHigh, ">", "<"); + if (null == strHigh) return null; + if (strHigh.Equals("???")) strHigh = null; + + String strLow = Sections.ScanSectionsFindStartWithReturnIndexAfter(spanSections, "Low today", 1); + if (null == strLow) return null; + strLow = Utility.BetweenString(strLow, ">", "<"); + if (null == strLow) return null; + if (strLow.Equals("???")) strLow = null; + + String strVolume = Sections.ScanSectionsFindStartWithReturnIndexAfter(spanSections, "Volume", 1); + if (null == strVolume) return null; + strVolume = Utility.BetweenString(strVolume, ">", "<"); + if (null == strVolume) return null; + if (strVolume.Equals("???")) strVolume = null; + + // Fetch the current price + httpNetResponse.Dispose(); + sb = new StringBuilder(); + sb.Append("https://bonfire.robinhood.com/instruments/").Append(strInstrumentId).Append("/detail-page-live-updating-data/?display_span=day&hide_extended_hours=false"); strRequest = sb.ToString(); httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest); if (!httpNetResponse.Success) @@ -5287,164 +5191,43 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } - byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); - MemoryStream memoryStream = new MemoryStream(streamBytes); - HtmlDocument htmlDocument = new HtmlDocument(); - htmlDocument.Load(memoryStream); - HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"historicalquote fatbottomed\"]"); - if (null == tables || 0 == tables.Count) return null; - HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); - if (rows.Count < 7) return null; + + JsonDocument jsonDocument = JsonDocument.Parse(httpNetResponse.ResponseString); + JsonElement root = jsonDocument.RootElement; + JsonElement chartSection = root.GetProperty("chart_section"); + JsonElement quoteSection = chartSection.GetProperty("quote"); + String strLastTradePrice = quoteSection.GetProperty("last_trade_price").GetString(); + String strUpdatedAt = quoteSection.GetProperty("updated_at").GetString(); + String strSymbol = quoteSection.GetProperty("symbol").GetString(); + + DateTime lastUpdatedAt = DateTime.Parse(strUpdatedAt, null, System.Globalization.DateTimeStyles.RoundtripKind); Price price = new Price(); - price.Source = Price.PriceSource.BigCharts; - price.Symbol = symbol.ToUpper(); - price.Date = FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText); - price.Close = FeedParser.ParseValueFromMarketWatch("Closing Price:", rows[2].InnerText); - price.AdjClose = price.Close; - price.Open = FeedParser.ParseValueFromMarketWatch("Open:", rows[3].InnerText); - price.High = FeedParser.ParseValueFromMarketWatch("High:", rows[4].InnerText); - price.Low = FeedParser.ParseValueFromMarketWatch("Low:", rows[5].InnerText); - price.Volume = FeedParser.ParseLongValueFromMarketWatch("Volume:", rows[6].InnerText); - if (!(price.Date.Date.Equals(asOf.Date.Date))) - { - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}", symbol, price.Date.ToShortDateString(), asOf.ToShortDateString())); - return null; - } + price.Symbol = strSymbol; + price.Date = lastUpdatedAt; + price.Source = Price.PriceSource.Robinhood; + price.Close = price.AdjClose = FeedParser.ParseValueDouble(strLastTradePrice); + price.Open = Utility.ParseCurrency(strOpen); + price.High = Utility.ParseCurrency(strHigh); + price.Low = Utility.ParseCurrency(strLow); + price.Volume = (long)FeedParser.ParseValueDouble(strVolume); + CheckPrice(price); + if (!price.IsValid) return null; + if (!price.Symbol.Equals(symbol)) return null; + MDTrace.WriteLine(LogLevel.DEBUG, $"Got price from Robinhood for {symbol} : {price.ToString()}"); return price; } - catch (Exception) - { - return null; - } - finally - { - if (null != httpNetResponse) httpNetResponse.Dispose(); - } - } - - /// - /// This is a modified version of the above query. It uses a cookie collection that contains the datadome cookie. - /// If this query starts to fail then you should try the query in developer tools in the chromium browser and capture the - /// datadome value and update it here in the GetCookieCollection() below. - /// - /// - /// - /// - public static Price GetPriceAsOfV2(String symbol, DateTime asOf) - { - HttpNetResponse httpNetResponse = null; - try - { - String strRequest; - StringBuilder sb = null; - - if (null == symbol) return null; - CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); - if (null != companyProfile && companyProfile.FreezePricing) - { - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); - return null; - } - String requestSymbol = symbol; - if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1); - sb = new StringBuilder(); - sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb="); - sb.Append(requestSymbol); - sb.Append("&closeDate="); - sb.Append(asOf.Month.ToString()); - sb.Append("%2F"); - sb.Append(asOf.Day.ToString()); - sb.Append("%2F"); - sb.Append((asOf.Year - 2000).ToString()); - sb.Append("&x=38&y=25"); - strRequest = sb.ToString(); - - MDTrace.WriteLine(LogLevel.DEBUG,$"{strRequest}"); - - CookieCollection cookieCollection = GetCookieCollection("www.marketwatch.com"); - - httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest,cookieCollection,new Uri("https://www.marketwatch.com/investing")); - if (!httpNetResponse.Success) - { - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); - return null; - } - byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); - MemoryStream memoryStream = new MemoryStream(streamBytes); - HtmlDocument htmlDocument = new HtmlDocument(); - htmlDocument.Load(memoryStream); - - HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"table table--overflow align--center\"]"); - - if (null == tables || 0 == tables.Count) return null; - HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); - if (rows.Count < 2) return null; - - Prices prices = new Prices(); - for (int rowIndex = 1; rowIndex < rows.Count; rowIndex++) - { - HtmlNodeCollection data = rows[rowIndex].SelectNodes(".//td"); - if (data.Count != 6) - { - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the correct number of data elements (expected 6 data elements).", symbol)); - return null; - } - Price price = new Price(); - price.Source = Price.PriceSource.BigCharts; - price.Symbol = symbol.ToUpper(); - String[] dateElements = data[0].InnerText.Trim().Split(" "); - price.Date = Utility.ParseDate(dateElements[0].Replace("\n", null)); - price.Open = Utility.ParseCurrency(data[1].InnerText.Trim().Replace("\n", null)); - price.High = Utility.ParseCurrency(data[2].InnerText.Trim().Replace("\n", null)); - price.Low = Utility.ParseCurrency(data[3].InnerText.Trim().Replace("\n", null)); - price.Close = price.AdjClose = Utility.ParseCurrency(data[4].InnerText.Trim().Replace("\n", null)); - price.Volume = FeedParser.ParseValueLong(data[5].InnerText.Trim()); - prices.Add(price); - } - - Price selectedPrice = prices.Where(x => x.Date.Date.Equals(asOf.Date.Date)).FirstOrDefault(); - - if (null == selectedPrice) - { - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("No price for {0} on {1}", symbol, asOf.ToShortDateString())); - return null; - } - - return selectedPrice; - } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception : {0}", exception.ToString())); - return null; + return null; } finally { if (null != httpNetResponse) httpNetResponse.Dispose(); } - } - - private static CookieCollection GetCookieCollection(String cookieDomain) - { - String[] cookies = { - "datadome=VLSWQ1PWL4SV2rljkJEV7GUOK3T11Sm73RP17IwDEtRzfWHykYwy7ZoeSPrjwmbwvGRAenazmLVTpyCB6Yqlw6vUL6Wbqq4zML5DGuHLxNU4LgAQ7Ko9tztglqjIGLZE", - }; - CookieCollection cookieCollection = new CookieCollection(); - for (int index = 0; index < cookies.Count(); index++) - { - String strCookie = cookies[index]; - if (strCookie.EndsWith(";")) strCookie = strCookie.Substring(0, strCookie.Length - 1); - String[] pairs = strCookie.Split('='); - cookieCollection.Add(new Cookie() - { - Name = pairs[0], - Value = pairs[1], - Domain = cookieDomain - }); - } - return cookieCollection; } -//******************************************************************************************************************************************************************************************************** + //******************************************************************************************************************************************************************************************************** // ******************************************************************************** H I S T O R I C A L P R I C I N G Y A H O O ********************************************************************* //******************************************************************************************************************************************************************************************************** @@ -5737,43 +5520,57 @@ namespace MarketData.Helper } } + public static List FindItemsInTag(String item, List> itemTags) + { + List foundItems = new List(); + + if (null == itemTags || 0 == itemTags.Count) return null; + for (int index = 0; index < itemTags.Count; index++) + { + List itemTag = itemTags[index]; + if (itemTag[0].Equals(item)) foundItems.Add(itemTag[1]); + } + return foundItems; + } + + private static DateTime ProcessNASDAQRelativeDate(String relativeDate) - { - DateTime referenceDate=DateTime.Now; - string[] items=relativeDate.Split(' '); - if(items.Length<2)return Utility.Epoch; - if("days".Equals(items[1])||"day".Equals(items[1])) - { - TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0,0,0); - referenceDate-=timeSpan; - } - else if("hours".Equals(items[1])||"hour".Equals(items[1])) - { - TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0); - referenceDate-=timeSpan; - } - else if("mins".Equals(items[1]) || "minutes".Equals(items[1])) - { - TimeSpan timeSpan=new TimeSpan(0,int.Parse(items[0]),0); - referenceDate-=timeSpan; - } - else if( (new string[]{"Jul","Aug","Sep","Oct","Nov","Dec","Jan","Feb","Mar","Apr","May","Jun"}).Any(x=>x.Equals(items[0]))) - { - DateTime now=DateTime.Now; - StringBuilder sb=new StringBuilder(); - sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1],'0',2)).Append(" ").Append(referenceDate.Year); - referenceDate=Utility.ParseDate(sb.ToString()); - if(referenceDate>now) - { - sb=new StringBuilder(); - sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1],'0',2)).Append(" ").Append(referenceDate.Year-1); - referenceDate=Utility.ParseDate(sb.ToString()); - } - } - else referenceDate=Utility.Epoch; - if(referenceDate>DateTime.Now)referenceDate=Utility.Epoch; - return referenceDate; - } + { + DateTime referenceDate = DateTime.Now; + string[] items = relativeDate.Split(' '); + if (items.Length < 2) return Utility.Epoch; + if ("days".Equals(items[1]) || "day".Equals(items[1])) + { + TimeSpan timeSpan = new TimeSpan(int.Parse(items[0]), 0, 0, 0, 0); + referenceDate -= timeSpan; + } + else if ("hours".Equals(items[1]) || "hour".Equals(items[1])) + { + TimeSpan timeSpan = new TimeSpan(int.Parse(items[0]), 0, 0); + referenceDate -= timeSpan; + } + else if ("mins".Equals(items[1]) || "minutes".Equals(items[1])) + { + TimeSpan timeSpan = new TimeSpan(0, int.Parse(items[0]), 0); + referenceDate -= timeSpan; + } + else if ((new string[] { "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", "Jan", "Feb", "Mar", "Apr", "May", "Jun" }).Any(x => x.Equals(items[0]))) + { + DateTime now = DateTime.Now; + StringBuilder sb = new StringBuilder(); + sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1], '0', 2)).Append(" ").Append(referenceDate.Year); + referenceDate = Utility.ParseDate(sb.ToString()); + if (referenceDate > now) + { + sb = new StringBuilder(); + sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1], '0', 2)).Append(" ").Append(referenceDate.Year - 1); + referenceDate = Utility.ParseDate(sb.ToString()); + } + } + else referenceDate = Utility.Epoch; + if (referenceDate > DateTime.Now) referenceDate = Utility.Epoch; + return referenceDate; + } // **************************************************************************************************************************************************************************************** // **************************************************************************** H E L P E R M E T H O D S ******************************************************************************* diff --git a/MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs b/MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs index 705450d..978ff06 100755 --- a/MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs @@ -36,44 +36,46 @@ namespace MarketData.Helper get{return startDate;} } } + public class PricingMarketDataHelper - { - private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads; - private static int WaitBetweenRequests=125; - private List symbols; - private int currentIndex = 0; + { + private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads; + private static int WaitBetweenRequests = 125; + private List symbols; + private int currentIndex = 0; + + public PricingMarketDataHelper() + { + } + public static Price RollPriceForward(String symbol) + { + try + { + DateGenerator dateGenerator = new DateGenerator(); + DateTime latestDate = PremarketDA.GetLatestMarketDate(); + if (Utility.IsEpoch(latestDate)) return null; + Price latestPrice = PricingDA.GetPrice(symbol); + if (null == latestPrice) return null; + DateTime latestPriceDateSymbol = latestPrice.Date; + latestPrice.Date = latestDate; + PricingDA.InsertPrice(latestPrice); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("{0} price rolled forward from {1} to {2}", symbol, latestPriceDateSymbol.ToShortDateString(), latestDate.ToShortDateString())); + return latestPrice; + } + catch (Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception rolling price forward for {0}. Exception was {1}", symbol, exception.ToString())); + return null; + } + } - public PricingMarketDataHelper() - { - } - public static Price RollPriceForward(String symbol) - { - try - { - DateGenerator dateGenerator=new DateGenerator(); - DateTime latestDate=PremarketDA.GetLatestMarketDate(); - if(Utility.IsEpoch(latestDate))return null; - Price latestPrice=PricingDA.GetPrice(symbol); - if(null==latestPrice)return null; - DateTime latestPriceDateSymbol=latestPrice.Date; - latestPrice.Date=latestDate; - PricingDA.InsertPrice(latestPrice); - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} price rolled forward from {1} to {2}",symbol,latestPriceDateSymbol.ToShortDateString(),latestDate.ToShortDateString())); - return latestPrice; - } - catch(Exception exception) - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception rolling price forward for {0}. Exception was {1}",symbol,exception.ToString())); - return null; - } - } public bool UpdateLatestPriceAll() { - Profiler profiler=new Profiler(); + Profiler profiler = new Profiler(); try { symbols = PricingDA.GetSymbols(); - currentIndex=0; + currentIndex = 0; while (true) { List queueSymbols = GetQueueSymbols(); @@ -85,28 +87,82 @@ namespace MarketData.Helper } for (int index = 0; index < queueSymbols.Count; index++) { - PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index],resetEvents[index]); + PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index]); ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdateLatestPriceAll, pricingThreadHelper); - try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;} + try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; } } - MDTrace.WriteLine(LogLevel.DEBUG,"UpdateLatestPriceAll, waiting for queued items to complete."); + MDTrace.WriteLine(LogLevel.DEBUG, "UpdateLatestPriceAll, waiting for queued items to complete."); WaitHandle.WaitAll(resetEvents); } return true; } finally { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateLatestPriceAll]End, total took {0}(ms)",profiler.End())); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdateLatestPriceAll]End, total took {0}(ms)", profiler.End())); } } - public bool UpdatePricesBigCharts(DateTime? startDate=null) // get prices from BigCharts + + // ************************************************************************************************************************************** + // ************************************************************* R O B I N H O O D ****************************************************** + // ************************************************************************************************************************************** + /// + /// Robinhood will only give us latest price. The price will have a date associated with it.. todays date. so the date provided needs to match. + /// I am planning to use this feed in UpdateDaily2 + /// + /// + /// + public bool UpdatePricesRobinhood(DateTime startDate) { - Profiler profiler=new Profiler(); + Profiler profiler = new Profiler(); + int maxThreads = 15; // I am tweaking these to back into a total runtime of 40 minutes for all stocks. maxThreads=10 waitBetweenRequests=500 = 86 minutes + int waitBetweenRequests = 250; + try { symbols = PricingDA.GetSymbols(); - currentIndex=0; - if(null==startDate)startDate=DateTime.Now; + currentIndex = 0; + while (true) + { + List queueSymbols = GetQueueSymbols(maxThreads); + if (null == queueSymbols || 0 == queueSymbols.Count) break; + ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count]; + for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++) + { + resetEvents[eventIndex] = new ManualResetEvent(false); + } + for (int index = 0; index < queueSymbols.Count; index++) + { + PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], startDate, resetEvents[index]); + ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesRobinhood, pricingThreadHelper); + try { Thread.Sleep(waitBetweenRequests); } catch (Exception) {; } + } + MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesRobinhood, waiting for queued items to complete."); + WaitHandle.WaitAll(resetEvents); + } + return true; + } + finally + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesRobinhood]End, total took {0} (ms)", profiler.End())); + } + } + + // ************************************************************************************************************************************** + // ************************************************************* E N D R O B I N H O O D *********************************************** + // ************************************************************************************************************************************** + + // ************************************************************************************************************************************** + // ******************************************************************* Y A H O O ******************************************************* + // ************************************************************************************************************************************** + + public bool UpdatePricesYahoo(DateTime? startDate = null) // get prices from Yahoo + { + Profiler profiler = new Profiler(); + try + { + symbols = PricingDA.GetSymbols(); + currentIndex = 0; + if (null == startDate) startDate = DateTime.Now; while (true) { List queueSymbols = GetQueueSymbols(); @@ -118,53 +174,20 @@ namespace MarketData.Helper } for (int index = 0; index < queueSymbols.Count; index++) { - PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index],startDate.Value,resetEvents[index]); - ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesBigCharts, pricingThreadHelper); - try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;} + PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], startDate.Value, resetEvents[index]); + ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesYahoo, pricingThreadHelper); + try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; } } - MDTrace.WriteLine(LogLevel.DEBUG,"UpdatePricesBigCharts, waiting for queued items to complete."); + MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesYahoo, waiting for queued items to complete."); WaitHandle.WaitAll(resetEvents); } return true; } finally { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesBigCharts]End, total took {0}(ms)",profiler.End())); - } + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesBigCharts]End, total took {0}(ms)", profiler.End())); + } } - public bool UpdatePricesYahoo(DateTime? startDate=null) // get prices from Yahoo - { - Profiler profiler=new Profiler(); - try - { - symbols=PricingDA.GetSymbols(); - currentIndex=0; - if(null==startDate) startDate=DateTime.Now; - while(true) - { - List queueSymbols=GetQueueSymbols(); - if(null==queueSymbols||0==queueSymbols.Count) break; - ManualResetEvent[] resetEvents=new ManualResetEvent[queueSymbols.Count]; - for(int eventIndex=0;eventIndex /// UpdatePricesBarChartSweep - This feed is intended to supplement BigCharts nightly price feed in UPDATEDAILY2 by filling any gaps that may occur /// in that feed. This feed will only pull a price for which we had a price yesterday (findprevbusinessday) and for which do not have @@ -209,7 +242,7 @@ namespace MarketData.Helper /// /// /// - public bool UpdatePricesBarChartSweep(DateTime? startDate = null) + public bool UpdatePricesBarChartSweep(DateTime? startDate = null) { Profiler profiler = new Profiler(); try @@ -262,8 +295,11 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesBarChartSweep]End, total took {0}(ms)", profiler.End())); } } +// ******************************************************************************************************************************************** +// ********************************************************************** E N D B A R C H A R T *************************************************** +// ******************************************************************************************************************************************** - public bool UpdatePricesAsOfSymbol(String symbol, DateTime asOf) // get prices from BigCharts + public bool UpdatePriceAsOfSymbolYahoo(String symbol, DateTime asOf) // get prices from Yahoo { Profiler profiler = new Profiler(); try @@ -282,8 +318,8 @@ namespace MarketData.Helper } for (int index = 0; index < queueSymbols.Count; index++) { - PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index], asOf); - ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesAsOf, pricingThreadHelper); + PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], asOf, resetEvents[index]); + ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesYahoo, pricingThreadHelper); try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; } } MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesAsOfSymbol, waiting for queued items to complete."); @@ -296,79 +332,15 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesAsOfSymbol]End, total took {0}(ms)", profiler.End())); } } - public bool UpdatePriceAsOfSymbolYahoo(String symbol,DateTime asOf) // get prices from Yahoo - { - Profiler profiler=new Profiler(); - try - { - symbols=new List(); - symbols.Add(symbol); - currentIndex=0; - while(true) - { - List queueSymbols=GetQueueSymbols(); - if(null==queueSymbols||0==queueSymbols.Count) break; - ManualResetEvent[] resetEvents=new ManualResetEvent[queueSymbols.Count]; - for(int eventIndex=0;eventIndex queueSymbols = GetQueueSymbols(); - if (null == queueSymbols || 0 == queueSymbols.Count) break; - ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count]; - for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++) - { - resetEvents[eventIndex] = new ManualResetEvent(false); - } - for (int index = 0; index < queueSymbols.Count; index++) - { - PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index],asOf); - ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesAsOf, pricingThreadHelper); - try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;} - } - MDTrace.WriteLine(LogLevel.DEBUG,"UpdatePricesAsOf, waiting for queued items to complete."); - WaitHandle.WaitAll(resetEvents); - } - return true; - } - finally - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesAsOf]End, total took {0}(ms)",profiler.End())); - } - } - public bool GetMissingPricesSymbol(String symbol,DateTime? startDate=null) - { - Profiler profiler=new Profiler(); - try - { - List symbols=new List(); - symbols.Add(symbol); - this.symbols=symbols; + List symbols = new List(); + symbols.Add(symbol); + this.symbols = symbols; currentIndex = 0; while (true) { @@ -381,26 +353,27 @@ namespace MarketData.Helper } for (int index = 0; index < queueSymbols.Count; index++) { - PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index],startDate); + PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index], startDate); ThreadPool.QueueUserWorkItem(ThreadPoolCallbackGetMissingPrice, pricingThreadHelper); - try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;} + try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; } } - MDTrace.WriteLine(LogLevel.DEBUG,"Get missing prices, waiting for queued items to complete."); + MDTrace.WriteLine(LogLevel.DEBUG, "Get missing prices, waiting for queued items to complete."); WaitHandle.WaitAll(resetEvents); } return true; } finally { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetMissingPrices]End, total took {0}(ms)",profiler.End())); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetMissingPrices]End, total took {0}(ms)", profiler.End())); } } - public bool GetMissingPrices(DateTime? startDate=null) + + public bool GetMissingPrices(DateTime? startDate = null) { - Profiler profiler=new Profiler(); + Profiler profiler = new Profiler(); try { - this.symbols=PricingDA.GetSymbols(); + this.symbols = PricingDA.GetSymbols(); currentIndex = 0; while (true) { @@ -413,28 +386,28 @@ namespace MarketData.Helper } for (int index = 0; index < queueSymbols.Count; index++) { - PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index],startDate); + PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index], startDate); ThreadPool.QueueUserWorkItem(ThreadPoolCallbackGetMissingPrice, pricingThreadHelper); - try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;} + try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; } } - MDTrace.WriteLine(LogLevel.DEBUG,"Get missing prices, waiting for queued items to complete."); + MDTrace.WriteLine(LogLevel.DEBUG, "Get missing prices, waiting for queued items to complete."); WaitHandle.WaitAll(resetEvents); } return true; } finally { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetMissingPrices]End, total took {0}(ms)",profiler.End())); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetMissingPrices]End, total took {0}(ms)", profiler.End())); } } - + public bool UpdateLatestPriceSymbol(String symbol) { List symbols = new List(); symbols.Add(symbol); return UpdateLatestPrices(symbols); } - + public bool UpdateLatestPrices(List symbols) { Profiler profiler = new Profiler(); @@ -468,371 +441,333 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdateLatestPrices]End, total took {0}(ms)", profiler.End())); } } - public bool LoadPrices(List symbols) - { - Profiler profiler=new Profiler(); - try - { - DateTime pricingDate=DateTime.Now; - this.symbols = symbols; - currentIndex = 0; - while (true) - { - List queueSymbols = GetQueueSymbols(); - if (null == queueSymbols || 0 == queueSymbols.Count) break; - ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count]; - for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++) - { - resetEvents[eventIndex] = new ManualResetEvent(false); - } - for (int index = 0; index < queueSymbols.Count; index++) - { - PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], pricingDate,resetEvents[index]); - ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadPrices, pricingThreadHelper); - try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;} - } - MDTrace.WriteLine(LogLevel.DEBUG,"Load prices, waiting for queued items to complete."); - WaitHandle.WaitAll(resetEvents); - } - return true; - } - finally - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadPrices]End, total took {0}(ms)",profiler.End())); - } - } -// ************************************************************************************************************************************ -// *************************************************** T H R E A D P O O L C A L L B A C K ****************************************** -// ************************************************************************************************************************************ + + // ************************************************************************************************************************************ + // *************************************************** T H R E A D P O O L C A L L B A C K ****************************************** + // ************************************************************************************************************************************ public void ThreadPoolCallbackUpdateLatestPriceAll(Object pricingThreadHelperContext) { - PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; - try - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("UpdateLatestPriceAll, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); - UpdateLatestPriceAllEx(pricingThreadHelper.Symbol); - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("UpdateLatestPriceAll, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); - } - finally - { - pricingThreadHelper.ResetEvent.Set(); - } - } - public void ThreadPoolCallbackUpdatePricesBigCharts(Object pricingThreadHelperContext) - { - PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; - try - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (BigCharts), Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); - UpdatePriceBigChartsEx(pricingThreadHelper.Symbol,pricingThreadHelper.PricingDate); - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (BigCharts), Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); - } - finally - { - pricingThreadHelper.ResetEvent.Set(); - } - } - public void ThreadPoolCallbackUpdatePricesYahoo(Object pricingThreadHelperContext) - { - PricingThreadHelper pricingThreadHelper=(PricingThreadHelper)pricingThreadHelperContext; - try - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (Yahoo), Thread {0} started for {1}...",Thread.CurrentThread.ManagedThreadId,pricingThreadHelper.Symbol)); - UpdatePriceYahoo(pricingThreadHelper.Symbol,pricingThreadHelper.PricingDate); - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (Yahoo), Thread {0} ended for {1}",Thread.CurrentThread.ManagedThreadId,pricingThreadHelper.Symbol)); - } - finally - { - pricingThreadHelper.ResetEvent.Set(); - } - } - public void ThreadPoolCallbackUpdatePricesYahooSweep(Object pricingThreadHelperContext) - { - PricingThreadHelper pricingThreadHelper=(PricingThreadHelper)pricingThreadHelperContext; - try - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (Yahoo), Thread {0} started for {1}...",Thread.CurrentThread.ManagedThreadId,pricingThreadHelper.Symbol)); - UpdatePriceYahooSweep(pricingThreadHelper.Symbol,pricingThreadHelper.PricingDate); - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (Yahoo), Thread {0} ended for {1}",Thread.CurrentThread.ManagedThreadId,pricingThreadHelper.Symbol)); - } - finally - { - pricingThreadHelper.ResetEvent.Set(); - } - } - public void ThreadPoolCallbackUpdatePricesBarChartSweep(Object pricingThreadHelperContext) - { - PricingThreadHelper pricingThreadHelper=(PricingThreadHelper)pricingThreadHelperContext; - try - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (BarChart), Thread {0} started for {1}...",Thread.CurrentThread.ManagedThreadId,pricingThreadHelper.Symbol)); - UpdatePriceBarChartSweep(pricingThreadHelper.Symbol,pricingThreadHelper.PricingDate); - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (BarChart), Thread {0} ended for {1}",Thread.CurrentThread.ManagedThreadId,pricingThreadHelper.Symbol)); - } - finally - { - pricingThreadHelper.ResetEvent.Set(); - } - } - public void ThreadPoolCallbackUpdatePricesAsOf(Object pricingThreadHelperContext) - { PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; try { - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); - UpdatePriceAsOfEx(pricingThreadHelper.Symbol, pricingThreadHelper.StartDate.Value); - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("UpdateLatestPriceAll, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); + UpdateLatestPriceAllEx(pricingThreadHelper.Symbol); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("UpdateLatestPriceAll, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); } finally { pricingThreadHelper.ResetEvent.Set(); } } - public void ThreadPoolCallbackGetMissingPrice(Object pricingThreadHelperContext) + + public void ThreadPoolCallbackUpdatePricesRobinhood(Object pricingThreadHelperContext) { - PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; + PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; try { - GetMissingPricesEx(pricingThreadHelper.Symbol,pricingThreadHelper.StartDate==null?new DateTime():pricingThreadHelper.StartDate.Value); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Robinhood), Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); + UpdatePriceRobinhoodEx(pricingThreadHelper.Symbol, pricingThreadHelper.PricingDate); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Robinhood), Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); } finally { - pricingThreadHelper.ResetEvent.Set(); - } + pricingThreadHelper.ResetEvent.Set(); + } } - public void ThreadPoolCallbackUpdateLatestPrice(Object pricingThreadHelperContext) - { - PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; + + public void ThreadPoolCallbackUpdatePricesYahoo(Object pricingThreadHelperContext) + { + PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; + try + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Yahoo), Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); + UpdatePriceYahoo(pricingThreadHelper.Symbol, pricingThreadHelper.PricingDate); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Yahoo), Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); + } + finally + { + pricingThreadHelper.ResetEvent.Set(); + } + } + + public void ThreadPoolCallbackUpdatePricesYahooSweep(Object pricingThreadHelperContext) + { + PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; + try + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Yahoo), Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); + UpdatePriceYahooSweep(pricingThreadHelper.Symbol, pricingThreadHelper.PricingDate); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Yahoo), Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); + } + finally + { + pricingThreadHelper.ResetEvent.Set(); + } + } + + public void ThreadPoolCallbackUpdatePricesBarChartSweep(Object pricingThreadHelperContext) + { + PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; + try + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (BarChart), Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); + UpdatePriceBarChartSweep(pricingThreadHelper.Symbol, pricingThreadHelper.PricingDate); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (BarChart), Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); + } + finally + { + pricingThreadHelper.ResetEvent.Set(); + } + } + + public void ThreadPoolCallbackGetMissingPrice(Object pricingThreadHelperContext) + { + PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; + try + { + GetMissingPricesEx(pricingThreadHelper.Symbol, pricingThreadHelper.StartDate == null ? new DateTime() : pricingThreadHelper.StartDate.Value); + } + finally + { + pricingThreadHelper.ResetEvent.Set(); + } + } + + public void ThreadPoolCallbackUpdateLatestPrice(Object pricingThreadHelperContext) + { + PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; try { Price latestPrice = MarketDataHelper.GetLatestPrice(pricingThreadHelper.Symbol); if (null == latestPrice) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Error retrieving latest price for {0} *****",pricingThreadHelper.Symbol)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("***** Error retrieving latest price for {0} *****", pricingThreadHelper.Symbol)); return; } - MDTrace.WriteLine(LogLevel.DEBUG,Price.Header); - MDTrace.WriteLine(LogLevel.DEBUG,latestPrice.ToString()); - MDTrace.WriteLine(LogLevel.DEBUG,"Removing price for '"+pricingThreadHelper.Symbol+"' on "+Utility.DateTimeToStringMMSDDSYYYY(latestPrice.Date)); - PricingDA.DeletePrice(pricingThreadHelper.Symbol,latestPrice.Date); - MDTrace.WriteLine(LogLevel.DEBUG,"Adding price for '" + pricingThreadHelper.Symbol + "' on " + Utility.DateTimeToStringMMSDDSYYYY(latestPrice.Date)); + MDTrace.WriteLine(LogLevel.DEBUG, Price.Header); + MDTrace.WriteLine(LogLevel.DEBUG, latestPrice.ToString()); + MDTrace.WriteLine(LogLevel.DEBUG, "Removing price for '" + pricingThreadHelper.Symbol + "' on " + Utility.DateTimeToStringMMSDDSYYYY(latestPrice.Date)); + PricingDA.DeletePrice(pricingThreadHelper.Symbol, latestPrice.Date); + MDTrace.WriteLine(LogLevel.DEBUG, "Adding price for '" + pricingThreadHelper.Symbol + "' on " + Utility.DateTimeToStringMMSDDSYYYY(latestPrice.Date)); PricingDA.InsertPrice(latestPrice); } finally { - pricingThreadHelper.ResetEvent.Set(); + pricingThreadHelper.ResetEvent.Set(); } - } - public void ThreadPoolCallbackLoadPrices(Object pricingThreadHelperContext) - { - PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; - try - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); - LoadPricesSymbolEx(pricingThreadHelper.Symbol,pricingThreadHelper.PricingDate); - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); - } - finally - { - pricingThreadHelper.ResetEvent.Set(); - } - } -// **************************************************************************************************************************************** -// **************************************************************************************************************************************** -// **************************************************************************************************************************************** - private List GetQueueSymbols() - { - List queueSymbols = new List(); - int index = currentIndex; - for (; index < currentIndex + MaxThreads && index < symbols.Count; index++) - { - queueSymbols.Add(symbols[index]); - } - currentIndex = index; - return queueSymbols; - } -// ******************************************************************************************************************************************** -// ************************************************************ W O R K E R S ***************************************************************** -// ******************************************************************************************************************************************** - public static void GetMissingPricesEx(String symbol,DateTime startDate) - { - try - { - DateGenerator dateGenerator=new DateGenerator(); - TimeSpan oneDay=new TimeSpan(1,0,0,0); - DateTime? historicalDate = null; - DateTime minDate=Constants.MIN_PRICING_DATE; + } - List pricingDatesList = PricingDA.GetPricingDatesForSymbol(symbol); - if (null == pricingDatesList || 0 == pricingDatesList.Count) return; - pricingDatesList=pricingDatesList.Where(x => x.Date>=minDate.Date).ToList(); - if(null==pricingDatesList||0==pricingDatesList.Count) return; - DateTime beginDate = pricingDatesList[0]; - DateTime endDate = pricingDatesList[pricingDatesList.Count - 1]; - Dictionary pricingDates = new Dictionary(); - for (int listIndex = 0; listIndex < pricingDatesList.Count; listIndex++) - { - DateTime pricingDate=pricingDatesList[listIndex]; - pricingDates.Add(pricingDate.Date, pricingDate.Date); - } - Dictionary marketPrices = new Dictionary(); - if (Utility.IsEpoch(startDate)) historicalDate = beginDate; - else historicalDate = startDate; - MDTrace.WriteLine(LogLevel.DEBUG,"Working on '" + symbol + "' from "+Utility.DateTimeToStringMMHDDHYYYY(historicalDate.Value)+" to "+Utility.DateTimeToStringMMHDDHYYYY(endDate)); - Prices prices = MarketDataHelper.GetDailyPrices(symbol, historicalDate.Value, endDate); // yahoo finance - if(null==prices||0==prices.Count)prices=MarketDataHelper.GetPricesAsOf(symbol,historicalDate.Value,endDate); // try big charts - if (null == prices||0==prices.Count)return; - for (int listIndex = 0; listIndex < prices.Count; listIndex++) - { - Price price = prices[listIndex]; - if(!marketPrices.ContainsKey(price.Date.Date))marketPrices.Add(price.Date,price); - } - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Retrieved {0} prices from feed for {1}",prices.Count,symbol)); - bool added=false; - for (DateTime date = historicalDate.Value.Date; date < endDate.Date; date += oneDay) - { - if (pricingDates.ContainsKey(date.Date)) continue; - if (dateGenerator.IsWeekend(date)) continue; - Price price = null; - if (!marketPrices.ContainsKey(date.Date))continue; - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Inserting price {0}...",price.ToString())); - price = marketPrices[date.Date]; - MDTrace.WriteLine(LogLevel.DEBUG,price.ToString()); - PricingDA.InsertPrice(price); - added=true; - } - if(!added) - { - MDTrace.WriteLine(LogLevel.DEBUG,"There were no new prices to add for '" + symbol); - } - } - catch(Exception exception) - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception {0}",exception.ToString())); - } - } - public static void UpdateLatestPriceAllEx(String symbol) - { - try - { - symbol = symbol.ToUpper(); - Price latestPrice = MarketDataHelper.GetLatestPrice(symbol); - if (null == latestPrice) - { - MDTrace.WriteLine(LogLevel.DEBUG,"Error retrieving latest price."); - return; - } - MDTrace.WriteLine(LogLevel.DEBUG,Price.Header); - MDTrace.WriteLine(LogLevel.DEBUG,latestPrice.ToString()); - MDTrace.WriteLine(LogLevel.DEBUG,"Removing price for '" + symbol + "' on " + Utility.DateTimeToStringMMSDDSYYYY(latestPrice.Date)); - PricingDA.DeletePrice(symbol, latestPrice.Date); - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Inserting price {0}...",latestPrice.ToString())); - PricingDA.InsertPrice(latestPrice); - } - catch(Exception exception) - { - MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); - } - } - public static void UpdatePriceBigChartsEx(String symbol,DateTime pricingDate) + // **************************************************************************************************************************************** + // **************************************************************************************************************************************** + // **************************************************************************************************************************************** + private List GetQueueSymbols() + { + List queueSymbols = new List(); + int index = currentIndex; + for (; index < currentIndex + MaxThreads && index < symbols.Count; index++) + { + queueSymbols.Add(symbols[index]); + } + currentIndex = index; + return queueSymbols; + } + + private List GetQueueSymbols(int maxThreads) + { + List queueSymbols = new List(); + int index = currentIndex; + for (; index < currentIndex + maxThreads && index < symbols.Count; index++) + { + queueSymbols.Add(symbols[index]); + } + currentIndex = index; + return queueSymbols; + } + + // ******************************************************************************************************************************************** + // ************************************************************ W O R K E R S ***************************************************************** + // ******************************************************************************************************************************************** + public static void GetMissingPricesEx(String symbol, DateTime startDate) { try { - Price price=PricingDA.GetPrice(symbol,pricingDate); - if(null!=price) + DateGenerator dateGenerator = new DateGenerator(); + TimeSpan oneDay = new TimeSpan(1, 0, 0, 0); + DateTime? historicalDate = null; + DateTime minDate = Constants.MIN_PRICING_DATE; + + List pricingDatesList = PricingDA.GetPricingDatesForSymbol(symbol); + if (null == pricingDatesList || 0 == pricingDatesList.Count) return; + pricingDatesList = pricingDatesList.Where(x => x.Date >= minDate.Date).ToList(); + if (null == pricingDatesList || 0 == pricingDatesList.Count) return; + DateTime beginDate = pricingDatesList[0]; + DateTime endDate = pricingDatesList[pricingDatesList.Count - 1]; + Dictionary pricingDates = new Dictionary(); + for (int listIndex = 0; listIndex < pricingDatesList.Count; listIndex++) { - MDTrace.WriteLine(LogLevel.DEBUG,"Already have latest price for '" + symbol + "'"); + DateTime pricingDate = pricingDatesList[listIndex]; + pricingDates.Add(pricingDate.Date, pricingDate.Date); + } + Dictionary marketPrices = new Dictionary(); + if (Utility.IsEpoch(startDate)) historicalDate = beginDate; + else historicalDate = startDate; + MDTrace.WriteLine(LogLevel.DEBUG, "Working on '" + symbol + "' from " + Utility.DateTimeToStringMMHDDHYYYY(historicalDate.Value) + " to " + Utility.DateTimeToStringMMHDDHYYYY(endDate)); + Prices prices = MarketDataHelper.GetDailyPrices(symbol, historicalDate.Value, endDate); // yahoo finance + // if (null == prices || 0 == prices.Count) prices = MarketDataHelper.GetPricesAsOf(symbol, historicalDate.Value, endDate); // try big charts + if (null == prices || 0 == prices.Count) return; + for (int listIndex = 0; listIndex < prices.Count; listIndex++) + { + Price price = prices[listIndex]; + if (!marketPrices.ContainsKey(price.Date.Date)) marketPrices.Add(price.Date, price); + } + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Retrieved {0} prices from feed for {1}", prices.Count, symbol)); + bool added = false; + for (DateTime date = historicalDate.Value.Date; date < endDate.Date; date += oneDay) + { + if (pricingDates.ContainsKey(date.Date)) continue; + if (dateGenerator.IsWeekend(date)) continue; + Price price = null; + if (!marketPrices.ContainsKey(date.Date)) continue; + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting price {0}...", price.ToString())); + price = marketPrices[date.Date]; + MDTrace.WriteLine(LogLevel.DEBUG, price.ToString()); + PricingDA.InsertPrice(price); + added = true; + } + if (!added) + { + MDTrace.WriteLine(LogLevel.DEBUG, "There were no new prices to add for '" + symbol); + } + } + catch (Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception {0}", exception.ToString())); + } + } + + public static void UpdateLatestPriceAllEx(String symbol) + { + try + { + symbol = symbol.ToUpper(); + Price latestPrice = MarketDataHelper.GetLatestPrice(symbol); + if (null == latestPrice) + { + MDTrace.WriteLine(LogLevel.DEBUG, "Error retrieving latest price."); return; } - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Requesting price for {0} for start date:{1} end date:{2}",symbol,pricingDate,pricingDate)); - Prices prices = MarketDataHelper.GetPricesAsOf(symbol, pricingDate,pricingDate); - if (null == prices||0==prices.Count) + MDTrace.WriteLine(LogLevel.DEBUG, Price.Header); + MDTrace.WriteLine(LogLevel.DEBUG, latestPrice.ToString()); + MDTrace.WriteLine(LogLevel.DEBUG, "Removing price for '" + symbol + "' on " + Utility.DateTimeToStringMMSDDSYYYY(latestPrice.Date)); + PricingDA.DeletePrice(symbol, latestPrice.Date); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting price {0}...", latestPrice.ToString())); + PricingDA.InsertPrice(latestPrice); + } + catch (Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG, exception.ToString()); + } + } + + public static void UpdatePriceRobinhoodEx(String symbol, DateTime pricingDate) + { + try + { + Price price = PricingDA.GetPrice(symbol, pricingDate); + if (null != price) { - MDTrace.WriteLine(LogLevel.DEBUG,"No prices (GetPricesAsOf) for '" + symbol+ "'"); + MDTrace.WriteLine(LogLevel.DEBUG, "Already have latest price for '" + symbol + "'"); + return; + } + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Requesting price for {0} for start date:{1}", symbol, pricingDate)); + price = MarketDataHelper.GetPriceRobinHood(symbol, pricingDate); + if (null == price) + { + MDTrace.WriteLine(LogLevel.DEBUG, "No price (GetPriceRobinHood) for '" + symbol + "'"); return; } else { - for (int secIndex = 0; secIndex < prices.Count; secIndex++) - { - MDTrace.WriteLine(LogLevel.DEBUG,prices[secIndex].ToString()); - } - MDTrace.WriteLine(LogLevel.DEBUG,"Inserting...."); - PricingDA.InsertPrices(prices); - MDTrace.WriteLine(LogLevel.DEBUG,"Done."); + MDTrace.WriteLine(LogLevel.DEBUG, price.ToString()); + MDTrace.WriteLine(LogLevel.DEBUG, "Inserting...."); + PricingDA.InsertPrice(price); + MDTrace.WriteLine(LogLevel.DEBUG, "Done."); } } - catch(Exception exception) + catch (Exception exception) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception {0}",exception.ToString())); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception {0}", exception.ToString())); } } - public static void UpdatePriceYahoo(String symbol,DateTime pricingDate) - { - try - { - Price price=PricingDA.GetPrice(symbol,pricingDate); - if(null!=price) - { - MDTrace.WriteLine(LogLevel.DEBUG,"Already have latest price for '"+symbol+"'"); - return; - } - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Requesting price for {0} for start date:{1} end date:{2}",symbol,pricingDate,pricingDate)); - price=MarketDataHelper.GetDailyPrice(symbol,pricingDate); - if(null==price) - { - MDTrace.WriteLine(LogLevel.DEBUG,"No price (UpdatePriceYahoo) for '"+symbol+"'"); - return; - } - else - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Inserting price {0}...",price.ToString())); - PricingDA.InsertPrice(price); - } - } - catch(Exception exception) - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception {0}",exception.ToString())); - } - } - public static void UpdatePriceYahooSweep(String symbol,DateTime pricingDate) - { - try - { - CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol); - Price price=PricingDA.GetPrice(symbol,pricingDate); - if(null!=price) - { - if(companyProfile==null || !companyProfile.PricingSourceEnum.Equals(CompanyProfile.EnumPricingSource.YAHOO)) - { - MDTrace.WriteLine(LogLevel.DEBUG,"Already have latest price for '"+symbol+"'"); - return; - } - } - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("UpdatePriceYahooSweep: Requesting price for {0} for start date:{1} end date:{2}",symbol,pricingDate,pricingDate)); - price=MarketDataHelper.GetDailyPrice(symbol,pricingDate); - if(null==price) - { - MDTrace.WriteLine(LogLevel.DEBUG,"No price (UpdatePriceYahooSweep) for '"+symbol+"'"); - return; - } - else if(!price.IsValid) - { - MDTrace.WriteLine(LogLevel.DEBUG,"Invalid price (UpdatePriceYahooSweep) for '"+symbol+"'"); - return; - } - else - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Inserting price {0}...",price.ToString())); - PricingDA.InsertPrice(price); - } - } - catch(Exception exception) - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception {0}",exception.ToString())); - } - } + + public static void UpdatePriceYahoo(String symbol, DateTime pricingDate) + { + try + { + Price price = PricingDA.GetPrice(symbol, pricingDate); + if (null != price) + { + MDTrace.WriteLine(LogLevel.DEBUG, "Already have latest price for '" + symbol + "'"); + return; + } + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Requesting price for {0} for start date:{1} end date:{2}", symbol, pricingDate, pricingDate)); + price = MarketDataHelper.GetDailyPrice(symbol, pricingDate); + if (null == price) + { + MDTrace.WriteLine(LogLevel.DEBUG, "No price (UpdatePriceYahoo) for '" + symbol + "'"); + return; + } + else + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting price {0}...", price.ToString())); + PricingDA.InsertPrice(price); + } + } + catch (Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception {0}", exception.ToString())); + } + } + + public static void UpdatePriceYahooSweep(String symbol, DateTime pricingDate) + { + try + { + CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); + Price price = PricingDA.GetPrice(symbol, pricingDate); + if (null != price) + { + if (companyProfile == null || !companyProfile.PricingSourceEnum.Equals(CompanyProfile.EnumPricingSource.YAHOO)) + { + MDTrace.WriteLine(LogLevel.DEBUG, "Already have latest price for '" + symbol + "'"); + return; + } + } + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("UpdatePriceYahooSweep: Requesting price for {0} for start date:{1} end date:{2}", symbol, pricingDate, pricingDate)); + price = MarketDataHelper.GetDailyPrice(symbol, pricingDate); + if (null == price) + { + MDTrace.WriteLine(LogLevel.DEBUG, "No price (UpdatePriceYahooSweep) for '" + symbol + "'"); + return; + } + else if (!price.IsValid) + { + MDTrace.WriteLine(LogLevel.DEBUG, "Invalid price (UpdatePriceYahooSweep) for '" + symbol + "'"); + return; + } + else + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting price {0}...", price.ToString())); + PricingDA.InsertPrice(price); + } + } + catch (Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception {0}", exception.ToString())); + } + } + /// /// This feed is intended to supplement BigCharts and Yahoo feed. /// It should run in the UPDATEDAILY2 chain after Yahoo during the nightly price capture. @@ -842,7 +777,7 @@ namespace MarketData.Helper /// /// /// - public static void UpdatePriceBarChartSweep(String symbol, DateTime pricingDate) + public static void UpdatePriceBarChartSweep(String symbol, DateTime pricingDate) { try { @@ -859,73 +794,13 @@ namespace MarketData.Helper return; } MDTrace.WriteLine(LogLevel.DEBUG, String.Format("(UpdatePriceBarChartSweep) Inserting price {0}...", price.ToString())); - PricingDA.InsertPrice(price); + PricingDA.InsertPrice(price); } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("(UpdatePriceBarChartSweep) Exception {0}", exception.ToString())); } } - - public static void UpdatePriceAsOfEx(String symbol, DateTime asOf) - { - try - { - DateTime latestDate = PricingDA.GetLatestDate(symbol); - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Latest pricing date for {0} is {1}", symbol, latestDate.ToShortDateString())); - if (latestDate.Equals(asOf)) - { - MDTrace.WriteLine(LogLevel.DEBUG, "Already have price for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(asOf)); - return; - } - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Retrieving price for {0} on {1} from BigCharts", symbol, asOf.ToShortDateString())); - Price price = MarketDataHelper.GetPriceAsOf(symbol, asOf); - if (null == price) - { - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Error retrieving price for {0} on {1}", symbol, asOf.ToShortDateString())); - return; - } - MDTrace.WriteLine(LogLevel.DEBUG, Price.Header); - MDTrace.WriteLine(LogLevel.DEBUG, price.ToString()); - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting price {0}...", price.ToString())); - PricingDA.InsertPrice(price); - MDTrace.WriteLine(LogLevel.DEBUG, ""); - } - catch (Exception exception) - { - MDTrace.WriteLine(LogLevel.DEBUG, exception.ToString()); - } - } - - public static void LoadPricesSymbolEx(String symbol,DateTime pricingDate) - { - DateGenerator dateGenerator = new DateGenerator(); - DateTime startDate = pricingDate; // the least recent date - DateTime endDate = pricingDate; // the most recent date - endDate = dateGenerator.GetPrevBusinessDay(endDate); - startDate = PricingDA.GetLatestDate(symbol); - startDate = dateGenerator.FindNextBusinessDay(startDate); - if (startDate.Date > endDate.Date) - { - MDTrace.WriteLine(LogLevel.DEBUG,"Already have latest price for '" + symbol + "'"); - return; - } - Prices prices=MarketDataHelper.GetPricesAsOf(symbol,startDate,endDate); - if (null == prices||0==prices.Count) - { - MDTrace.WriteLine(LogLevel.DEBUG,"No prices (GetDailyPrices) for '" + symbol + "'"); - return; - } - else - { - for (int secIndex = 0; secIndex < prices.Count; secIndex++) - { - MDTrace.WriteLine(LogLevel.DEBUG,prices[secIndex].ToString()); - } - PricingDA.InsertPrices(prices); - MDTrace.WriteLine(LogLevel.DEBUG,"Done."); - } - } - } + } } diff --git a/MarketData/MarketDataLib/MarketDataModel/Prices.cs b/MarketData/MarketDataLib/MarketDataModel/Prices.cs index fa3abf4..6816d46 100755 --- a/MarketData/MarketDataLib/MarketDataModel/Prices.cs +++ b/MarketData/MarketDataLib/MarketDataModel/Prices.cs @@ -316,7 +316,7 @@ namespace MarketData.MarketDataModel // *************************************************************************************************************************************************************************** public class Price { - public enum PriceSource{Other=0,BigCharts=1,Yahoo=2,Fidelity=3,Google=4,BarChart=5}; // !!IMPORTANT only add to this list. + public enum PriceSource{Other=0,BigCharts=1,Yahoo=2,Fidelity=3,Google=4,BarChart=5,Robinhood=6}; // !!IMPORTANT only add to this list. private String symbol; private DateTime date; private double open; @@ -379,7 +379,9 @@ namespace MarketData.MarketDataModel return "Google"; case PriceSource.BarChart : return "BarChart"; - default : + case PriceSource.Robinhood : + return "Robinhood"; + default: return Constants.CONST_QUESTION; } } diff --git a/MarketData/MarketDataLib/Utility/FeedParser.cs b/MarketData/MarketDataLib/Utility/FeedParser.cs index 7028e74..b2e06e6 100755 --- a/MarketData/MarketDataLib/Utility/FeedParser.cs +++ b/MarketData/MarketDataLib/Utility/FeedParser.cs @@ -14,16 +14,16 @@ namespace MarketData.Utils private FeedParser() { } - public static TimeSeriesCollection ParseTimeSeries(String symbol, List reportDates, TimeSeriesElement.ElementType elementType,String strLine) + public static TimeSeriesCollection ParseTimeSeries(String symbol, List reportDates, TimeSeriesElement.ElementType elementType, String strLine) { TimeSeriesCollection timeSeriesCollection = new TimeSeriesCollection(); - Dictionary dates = new Dictionary(); + Dictionary dates = new Dictionary(); try { if (null == reportDates || 0 == reportDates.Count) return null; String[] lineItems = CSVLineHelper.ParseLine(strLine); if (null == lineItems) return null; - for (int index = 0; index < reportDates.Count;index++) + for (int index = 0; index < reportDates.Count; index++) { TimeSeriesElement timeSeriesElement = new TimeSeriesElement(); timeSeriesElement.Type = elementType; @@ -46,11 +46,11 @@ namespace MarketData.Utils } catch (Exception exception) { - MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); + MDTrace.WriteLine(LogLevel.DEBUG, exception.ToString()); return null; } } -//"Book Value Per Share USD,3.31,3.22,2.64,2.58,2.28,2.10,1.39,2.64,3.88,3.94,3.94" + //"Book Value Per Share USD,3.31,3.22,2.64,2.58,2.28,2.10,1.39,2.64,3.88,3.94,3.94" public static List ParsePeriodEndingDates(String periodEnding) { List periodEndingDates = new List(); @@ -58,7 +58,7 @@ namespace MarketData.Utils periodEnding = periodEnding.Replace("Period Ending", ""); int length = periodEnding.Length; int index = 0; - while (indexcurrentDate)resultingDate = new DateTime(resultingDate.Year-1,resultingDate.Month, resultingDate.Day); + if (resultingDate > currentDate) resultingDate = new DateTime(resultingDate.Year - 1, resultingDate.Month, resultingDate.Day); return resultingDate; } else @@ -201,16 +201,16 @@ namespace MarketData.Utils { try { - if(null==strText)return Utility.Epoch; + if (null == strText) return Utility.Epoch; String[] strArray = strText.Split(' '); if (3 > strArray.Length) return DateTime.Parse("01-01-0001"); strArray[1] = strArray[1].Replace(",", null); strArray[0] = strArray[0].Replace(".", null); - return ParseValueDateTime(Utility.Pad(strArray[1],'0',2)+"-"+strArray[0]+"-"+strArray[2].Substring(2,2)); + return ParseValueDateTime(Utility.Pad(strArray[1], '0', 2) + "-" + strArray[0] + "-" + strArray[2].Substring(2, 2)); } catch (Exception exception) { - MDTrace.WriteLine(LogLevel.DEBUG,"[ParseValueDateTimeMonthFormat] Error parsing date '" + strText + "', "+exception.ToString()); + MDTrace.WriteLine(LogLevel.DEBUG, "[ParseValueDateTimeMonthFormat] Error parsing date '" + strText + "', " + exception.ToString()); return DateTime.Parse("01-01-0001"); } } @@ -219,33 +219,33 @@ namespace MarketData.Utils { try { - DateTime today=DateTime.Now; - if(strDate.Contains("day")) + DateTime today = DateTime.Now; + if (strDate.Contains("day")) { - DateGenerator dateGenerator=new DateGenerator(); - String[] items=strDate.Split(' '); - DateTime relativeDate=dateGenerator.GenerateHistoricalDate(today,int.Parse(items[0])+1); + DateGenerator dateGenerator = new DateGenerator(); + String[] items = strDate.Split(' '); + DateTime relativeDate = dateGenerator.GenerateHistoricalDate(today, int.Parse(items[0]) + 1); return relativeDate; } - else if(strDate.Contains("hour")) + else if (strDate.Contains("hour")) { - String[] items=strDate.Split(' '); - TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0); - DateTime relativeDate=today.Subtract(timeSpan); + String[] items = strDate.Split(' '); + TimeSpan timeSpan = new TimeSpan(int.Parse(items[0]), 0, 0); + DateTime relativeDate = today.Subtract(timeSpan); return relativeDate; } - else if(strDate.Contains("minute")) + else if (strDate.Contains("minute")) { - String[] items=strDate.Split(' '); - TimeSpan timeSpan=new TimeSpan(0,int.Parse(items[0]),0); - DateTime relativeDate=today.Subtract(timeSpan); + String[] items = strDate.Split(' '); + TimeSpan timeSpan = new TimeSpan(0, int.Parse(items[0]), 0); + DateTime relativeDate = today.Subtract(timeSpan); return relativeDate; } return ParseValueDateTimeMonthFormat(strDate); } - catch(Exception) + catch (Exception) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Unable to parse date '{0}'",strDate)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Unable to parse date '{0}'", strDate)); return null; } } @@ -256,15 +256,15 @@ namespace MarketData.Utils try { if (null == strText || 0 == strText.Length) return double.NaN; - strText=strText.Trim(); - if(strText.StartsWith("--"))return double.NaN; - if(strText.Equals("N/A"))return double.NaN; - strText=strText.Replace("%", ""); + strText = strText.Trim(); + if (strText.StartsWith("--")) return double.NaN; + if (strText.Equals("N/A")) return double.NaN; + strText = strText.Replace("%", ""); strText = strText.Replace("$", ""); if (strText.Contains("(")) { - strText=strText.Replace("(", ""); - strText=strText.Replace(")", ""); + strText = strText.Replace("(", ""); + strText = strText.Replace(")", ""); multiplier = -1.00; } if (strText.Equals("-")) return double.NaN; @@ -286,7 +286,7 @@ namespace MarketData.Utils value = double.Parse(strText); value *= 1000000; } - else if (strText[strText.Length - 1].Equals('K')||strText[strText.Length - 1].Equals('k')) + else if (strText[strText.Length - 1].Equals('K') || strText[strText.Length - 1].Equals('k')) { strText = strText.Replace("K", ""); strText = strText.Replace("k", ""); @@ -299,7 +299,7 @@ namespace MarketData.Utils } catch (Exception) { - MDTrace.WriteLine(LogLevel.DEBUG,"[ParseValue] Error parsing '"+strText+"'"); + MDTrace.WriteLine(LogLevel.DEBUG, "[ParseValue] Error parsing '" + strText + "'"); return double.NaN; } } @@ -309,8 +309,8 @@ namespace MarketData.Utils long value; try { - if(null==strText)return 0; - strText=strText.Replace("%",""); + if (null == strText) return 0; + strText = strText.Replace("%", ""); strText = strText.Replace(",", ""); if (strText.Equals("-")) return 0; if (strText[strText.Length - 1].Equals('B')) @@ -336,9 +336,46 @@ namespace MarketData.Utils } catch (Exception) { - MDTrace.WriteLine(LogLevel.DEBUG,"[ParseValueLong] Error parsing '" + strText + "'"); + MDTrace.WriteLine(LogLevel.DEBUG, "[ParseValueLong] Error parsing '" + strText + "'"); return 0; } } + + public static double ParseValueDouble(String strText) + { + double value; + try + { + if(null==strText)return 0; + strText=strText.Replace("%",""); + strText = strText.Replace(",", ""); + if (strText.Equals("-")) return 0; + if (strText[strText.Length - 1].Equals('B')) + { + strText = strText.Replace("B", ""); + value = double.Parse(strText); + value *= 1000000000; + } + else if (strText[strText.Length - 1].Equals('M')) + { + strText = strText.Replace("M", ""); + value = double.Parse(strText); + value *= 1000000; + } + else if (strText[strText.Length - 1].Equals('K')) + { + strText = strText.Replace("K", ""); + value = double.Parse(strText); + value *= 1000; + } + else value = double.Parse(strText); + return value; + } + catch (Exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,"[ParseValueDouble] Error parsing '" + strText + "'"); + return 0; + } + } } } diff --git a/MarketData/MarketDataLib/Utility/Sections.cs b/MarketData/MarketDataLib/Utility/Sections.cs index 474dc7e..3ef607e 100755 --- a/MarketData/MarketDataLib/Utility/Sections.cs +++ b/MarketData/MarketDataLib/Utility/Sections.cs @@ -36,7 +36,7 @@ namespace MarketDataLib.Utility while (true) { String itemsInSection = GetItemsInSection(strInput, sectionName, ref searchIndex); - if(null==itemsInSection)break; + if (null == itemsInSection) break; sectionItems.Add(itemsInSection); searchIndex++; } @@ -106,18 +106,19 @@ namespace MarketDataLib.Utility } return false; } - public static bool FindInSectionsReverse(List sections,String startsWith,int startingIndex,ref int indexOfItem,bool findExact=true) + + public static bool FindInSectionsReverse(List sections, String startsWith, int startingIndex, ref int indexOfItem, bool findExact = true) { - for(int index=startingIndex;index>=0;index--) + for (int index = startingIndex; index >= 0; index--) { - if(!findExact &§ions[index].StartsWith(startsWith)) + if (!findExact && sections[index].StartsWith(startsWith)) { - indexOfItem=index; + indexOfItem = index; return true; } - else if(sections[index].Equals(startsWith)) + else if (sections[index].Equals(startsWith)) { - indexOfItem=index; + indexOfItem = index; return true; } } diff --git a/MarketData/MarketDataLib/Utility/Utility.cs b/MarketData/MarketDataLib/Utility/Utility.cs index 7b8a35b..f3d515d 100755 --- a/MarketData/MarketDataLib/Utility/Utility.cs +++ b/MarketData/MarketDataLib/Utility/Utility.cs @@ -609,7 +609,8 @@ namespace MarketData.Utils public static DateTime ParseDate(String strDate) { System.Globalization.CultureInfo cultureInfo = new System.Globalization.CultureInfo("en-US"); - String[] formats=new[] { "MM/dd/yyyy HH:MM tt", "yyyy-MM-dd","yy-MM-dd","yyyy-MM-dd hh:mm:ss tt","dddd, MMMM dd","MMM dd yyyy","yyyy-MM","ddd, MMM. d","ddd, MMM. dd","yyyy/MM/dd","M-d-yyyy","dd-MM-yyyy","MM-dd-yyyy","M.d.yyyy","dd.MM.yyyy","MM.dd.yyyy","yyyyMMdd" }.Union(cultureInfo.DateTimeFormat.GetAllDateTimePatterns()).ToArray(); + + String[] formats=new[] { "MMMM dd,yyyy" ,"yyyy-MM-dd HH:mm:ss","MM/dd/yyyy HH:MM tt", "yyyy-MM-dd","yy-MM-dd","yyyy-MM-dd hh:mm:ss tt","dddd, MMMM dd","MMM dd yyyy","yyyy-MM","ddd, MMM. d","ddd, MMM. dd","yyyy/MM/dd","M-d-yyyy","dd-MM-yyyy","MM-dd-yyyy","M.d.yyyy","dd.MM.yyyy","MM.dd.yyyy","yyyyMMdd" }.Union(cultureInfo.DateTimeFormat.GetAllDateTimePatterns()).ToArray(); strDate = strDate.Trim(); DateTime dateTime=DateTime.ParseExact(strDate, formats, cultureInfo, DateTimeStyles.AssumeLocal); return dateTime;