From 8c73faf59f31d28a73d204b315481e5ffb550505 Mon Sep 17 00:00:00 2001 From: Sean Date: Sat, 28 Jun 2025 12:21:47 -0400 Subject: [PATCH] Optimize the BarChart pull --- .../MarketDataLib/Helper/MarketDataHelper.cs | 4 +- .../Helper/PricingMarketDataHelper.cs | 60 +++++++++++-------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs index 4336e1f..eb19efc 100755 --- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs @@ -5113,6 +5113,8 @@ namespace MarketData.Helper HttpNetResponse httpNetResponse = null; try { + DateGenerator dateGenerator = new DateGenerator(); + DateTime currentPricingDate = dateGenerator.GetPrevBusinessDay(DateTime.Now); StringBuilder sb = new StringBuilder(); String strRequest; if (null == symbol) return null; @@ -5138,7 +5140,7 @@ namespace MarketData.Helper Dictionary values = CreateBarChartValues(pairs); Price price = new Price(); price.Symbol = symbol; - price.Date = DateTime.Now; + price.Date = currentPricingDate; price.Source = Price.PriceSource.BarChart; if (values.ContainsKey("lowPrice")) price.Low = values["lowPrice"]; if (values.ContainsKey("highPrice")) price.High = values["highPrice"]; diff --git a/MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs b/MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs index efe4f09..59b7351 100755 --- a/MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs @@ -165,8 +165,8 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesBigCharts]End, total took {0}(ms)",profiler.End())); } } - - public bool UpdatePricesYahooSweep(DateTime? startDate = null) // get prices from Yahoo + + public bool UpdatePricesYahooSweep(DateTime? startDate = null) // get prices from Yahoo { Profiler profiler = new Profiler(); try @@ -199,19 +199,46 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesYahooSweep]End, total took {0}(ms)", profiler.End())); } } - + /// - /// UpdatePricesBarChartSweep - This feed will retrieve latest price. It will not use a date + /// UpdatePricesBarChartSweep - This feed is intended to supplement BigCharts nightly price feed in UPDATEDAILY2 by filling any gaps that may occur + /// in that feed. This feed will only pull a price for which we had a price yesterday (findprevbusinessday) and for which do not have + /// a price tonight. If BigCharts was successful then we can expect this method to retrieve nothing. If BigCharts fails to give us a price + /// tonight BUT did in fact give us a price yesterday then this method will attempt to rerieve that symbol from BarChart and insert that + /// price into the database with the appropriate date (tonight) and with the appropriate Source (BarChart). /// /// /// - public bool UpdatePricesBarChartSweep(DateTime? startDate = null) // get prices from BarCharts + public bool UpdatePricesBarChartSweep(DateTime? startDate = null) { Profiler profiler = new Profiler(); try { - symbols = PricingDA.GetSymbols(); + MDTrace.WriteLine(LogLevel.DEBUG, "[UpdatePricesBarChartSweep]Enter"); + DateGenerator dateGenerator = new DateGenerator(); if (null == startDate) startDate = DateTime.Now; + startDate = dateGenerator.GetPrevBusinessDay(startDate.Value); // make sure we are given a valid business date + DateTime requiredDate = dateGenerator.FindPrevBusinessDay(startDate.Value); // now go and fetch yesterday (prev business day) + MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweep] Determining what prices we need on {startDate.Value.ToShortDateString()} by examing what prices we had on {requiredDate.ToShortDateString()}"); + MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweep] Retrieve universe of symbols..."); + symbols = PricingDA.GetSymbols(); // get the list of symbols in the universe + MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweep] Retrieve latest dates..."); + Dictionary latestDates = PricingDA.GetLatestDates(symbols); // get the latest pricing date for these symbols + List symbolsToFetch = new List(); + List keys = latestDates.Keys.ToList(); + MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweep] Filtering..."); + foreach (String key in keys) // go through the latest dates and select those for which + { // we have a price on previous business date + if (latestDates[key].Date.Date.Equals(requiredDate.Date)) + { + symbolsToFetch.Add(key); + } + } + symbols = symbolsToFetch; // These are the symbols that we know we need to fetch + if (0 == symbolsToFetch.Count) + { + MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweep] Does not need to supplement any pricing data for {startDate.Value.ToShortDateString()} ."); + } currentIndex = 0; while (true) { @@ -814,6 +841,7 @@ namespace MarketData.Helper /// It should run in the UPDATEDAILY2 chain after Yahoo during the nightly price capture. /// It will only retrieve price for the current date AND it expects us to have a price for the previous business date. /// In other words it will not fill pricing gaps, it will only add "tonights" price to an already established pricing history + /// and only if we are missing a price. See the method UpdatePricesBarChartSweep to understand the context. /// /// /// @@ -821,26 +849,8 @@ namespace MarketData.Helper { try { - DateGenerator dateGenerator = new DateGenerator(); - DateTime prevBusinessDay = dateGenerator.FindPrevBusinessDay(pricingDate); - Price price = PricingDA.GetPrice(symbol); // retrieve the latest price for this symbol - if (null == price) // we don't have a price for the symbol so return - { - return; - } - if (price.Date.Date.Equals(pricingDate.Date)) - { - MDTrace.WriteLine(LogLevel.DEBUG, $"Already have latest price for {symbol} on {price.Date.ToShortDateString()}"); - return; - } - if (!prevBusinessDay.Date.Equals(price.Date.Date)) - { - MDTrace.WriteLine(LogLevel.DEBUG, $"Expected a date of {prevBusinessDay.ToShortDateString()} for {symbol} but found {price.Date.ToShortDateString()}"); - return; - } - MDTrace.WriteLine(LogLevel.DEBUG,$"Latest price for {symbol} is {price.Date.Date.ToShortDateString()}"); MDTrace.WriteLine(LogLevel.DEBUG, String.Format("UpdatePriceBarChartSweep: Requesting latest price for {0} ", symbol)); - price = MarketDataHelper.GetLatestPriceBarChart(symbol); + Price price = MarketDataHelper.GetLatestPriceBarChart(symbol); if (null == price) { MDTrace.WriteLine(LogLevel.DEBUG, "(UpdatePriceBarChartSweep) No price for '" + symbol + "'");