From d3ad9e918b0403824246d089681432bb6e500771 Mon Sep 17 00:00:00 2001 From: Sean Date: Fri, 17 Oct 2025 21:37:42 -0400 Subject: [PATCH] Make RobinHood feed a sweep --- MarketData/MarketData/Services/MainService.cs | 6 +-- .../Helper/PricingMarketDataHelper.cs | 48 +++++++++++++------ 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/MarketData/MarketData/Services/MainService.cs b/MarketData/MarketData/Services/MainService.cs index 3ae678c..fea3dfc 100755 --- a/MarketData/MarketData/Services/MainService.cs +++ b/MarketData/MarketData/Services/MainService.cs @@ -558,7 +558,7 @@ namespace MarketData.Services ThreadPool.QueueUserWorkItem(delegate { UpdatePricesYahooSweep(startDate); // This was a sweep but now it is the main price feed since the BigCharts feed issue. Robinhood runs to fill the gaps anbd then BarChart - UpdatePricesRobinhood(startDate); // This the new Robinhood price feed. It will only pull current day prices for items that have not been pulled from above feed. + UpdatePricesRobinhoodSweep(startDate); // This the new Robinhood price feed. It will only pull current day prices for items that have not been pulled from above feed. UpdatePricesBarChartSweep(startDate); // barchart. This sweep will pull any prices we failed to retrieve from Yahoo and Robinhood. resetEvents[STAGE_1].Set(); SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_1 UPDATEPRICES YAHOO/ROBINHOOD/BARCHART done.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); @@ -653,10 +653,10 @@ namespace MarketData.Services pricingMarketDataHelper.UpdatePricesYahooSweep(startDate); } - public static void UpdatePricesRobinhood(DateTime startDate) + public static void UpdatePricesRobinhoodSweep(DateTime startDate) { PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper(); - pricingMarketDataHelper.UpdatePricesRobinhood(startDate); + pricingMarketDataHelper.UpdatePricesRobinhoodSweep(startDate); } public static void UpdatePricesBarChartSweep(DateTime startDate) diff --git a/MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs b/MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs index e0f5911..efb74d6 100755 --- a/MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs @@ -106,20 +106,39 @@ namespace MarketData.Helper // ************************************************************* R O B I N H O O D ****************************************************** // ************************************************************************************************************************************** /// - /// Robinhood will only give us latest price. The price will have a date associated with it.. todays date. so the date provided needs to match. - /// I am planning to use this feed in UpdateDaily2 + /// Robinhood will only give us latest price. The price will have a date associated with it.. todays date. so the date provided needs to match. + /// This feed works the same way as the BarChartSweep in that we will only attempt to fetch prices that are missing today for which we had prices yesterday. /// /// /// - public bool UpdatePricesRobinhood(DateTime startDate) + public bool UpdatePricesRobinhoodSweep(DateTime startDate) { Profiler profiler = new Profiler(); - int maxThreads = 20; // I am tweaking these to back into a total runtime of 40 minutes for all stocks. maxThreads=10 waitBetweenRequests=500 = 86 minutes - int waitBetweenRequests = 200; - try { - symbols = PricingDA.GetSymbols(); + int maxThreads = 20; // I am tweaking these to back into a total runtime of 40 minutes for all stocks. maxThreads=10 waitBetweenRequests=500 = 86 minutes + int waitBetweenRequests = 200; + MDTrace.WriteLine(LogLevel.DEBUG, "[UpdatePricesRobinhoodSweep]Enter"); + DateGenerator dateGenerator = new DateGenerator(); + startDate = dateGenerator.GetPrevBusinessDay(startDate); // make sure we are given a valid business date + DateTime requiredDate = dateGenerator.FindPrevBusinessDay(startDate); // now go and fetch yesterday (prev business day) + MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesRobinhoodSweep] Determining what prices we need on {startDate.ToShortDateString()} by examing what prices we had on {requiredDate.ToShortDateString()}"); + MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesRobinhoodSweep] Retrieve universe of symbols..."); + symbols = PricingDA.GetSymbols(); // get the list of symbols in the universe + MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesRobinhoodSweep] Retrieve latest dates..."); + Dictionary latestDates = PricingDA.GetLatestDates(symbols); // get the latest pricing date for these symbols + List symbolsToFetch = new List(); + List keys = latestDates.Keys.ToList(); + MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesRobinhoodSweep] Filtering..."); + foreach (String key in keys) // go through the latest dates and select those for which + { // we have a price on previous business date + if (latestDates[key].Date.Date.Equals(requiredDate.Date)) + { + symbolsToFetch.Add(key); + } + } + symbols = symbolsToFetch; // These are the symbols that we know we need to fetch + MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesRobinhoodSweep] Preparing to supplement {symbolsToFetch.Count} prices for pricing date {startDate.ToShortDateString()} ."); currentIndex = 0; while (true) { @@ -133,20 +152,19 @@ namespace MarketData.Helper for (int index = 0; index < queueSymbols.Count; index++) { PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], startDate, resetEvents[index]); - ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesRobinhood, pricingThreadHelper); + ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesRobinhoodSweep, pricingThreadHelper); try { Thread.Sleep(waitBetweenRequests); } catch (Exception) {; } } - MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesRobinhood, waiting for queued items to complete."); + MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesRobinhoodSweep, waiting for queued items to complete."); WaitHandle.WaitAll(resetEvents); } return true; } finally { - MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesRobinhood]End, total took {0} (ms)", profiler.End())); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesRobinhoodSweep] End, total took {0}(ms)", profiler.End())); } } - // ************************************************************************************************************************************** // ************************************************************* E N D R O B I N H O O D *********************************************** // ************************************************************************************************************************************** @@ -268,7 +286,7 @@ namespace MarketData.Helper } } symbols = symbolsToFetch; // These are the symbols that we know we need to fetch - MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweep] Preparing to supplement {symbolsToFetch.Count} for pricing date {startDate.Value.ToShortDateString()} ."); + MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweep] Preparing to supplement {symbolsToFetch.Count} prices for pricing date {startDate.Value.ToShortDateString()} ."); currentIndex = 0; while (true) { @@ -460,13 +478,13 @@ namespace MarketData.Helper } } - public void ThreadPoolCallbackUpdatePricesRobinhood(Object pricingThreadHelperContext) + public void ThreadPoolCallbackUpdatePricesRobinhoodSweep(Object pricingThreadHelperContext) { PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext; try { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Robinhood), Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); - UpdatePriceRobinhoodEx(pricingThreadHelper.Symbol, pricingThreadHelper.PricingDate); + UpdatePriceRobinhoodSweepEx(pricingThreadHelper.Symbol, pricingThreadHelper.PricingDate); MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Robinhood), Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol)); } finally @@ -670,7 +688,7 @@ namespace MarketData.Helper } } - public static void UpdatePriceRobinhoodEx(String symbol, DateTime pricingDate) + public static void UpdatePriceRobinhoodSweepEx(String symbol, DateTime pricingDate) { try {