Sean/ARM64
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Add BarChart supplemental feed

Browse Source
This commit is contained in:
Sean
2025-06-27 21:38:37 -04:00
parent bd93d7af44
commit 3fb36991ff
4 changed files with 308 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
MarketData/MarketDataLib/Helper/MarketDataHelper.cs
View File

@@ -5101,9 +5101,98 @@ namespace MarketData.Helper
}
}
/// <summary>
/// GetLatestPriceBarChart - This is a new feed I am implementing because I am having difficulty retrieving prices from MarketWatch(BigCharts)
/// I am planning to use this field initially as a sweep to run after the BigCharts feed and prior to the Yahoo Sweep
/// </summary>
/// <param name="symbol"></param>
/// <returns></returns>
public static Price GetLatestPriceBarChart(String symbol)
{
HttpNetResponse httpNetResponse = null;
try
{
StringBuilder sb = new StringBuilder();
String strRequest;
if (null == symbol) return null;
sb = new StringBuilder();
sb.Append("https://www.barchart.com/stocks/quotes/").Append(symbol).Append("/overview");
strRequest = sb.ToString();
WebProxy webProxy = HttpNetRequest.GetProxy("GetLatestPriceBarChart");
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3C(strRequest, webProxy);
if (!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetLatestPriceBarChart: Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
String beginMarker = "data-ng-init='init(\"" + symbol + "\"";
String endMarker = ">";
String item = Utility.BetweenString(httpNetResponse.ResponseString, beginMarker, endMarker);
if (null == item) return null;
item = item.Replace("&quot;", "\"");
item = item.Replace("{", null);
item = item.Replace("}", null);
String[] pairs = item.Split(",");
Dictionary<String, double> values = CreateBarChartValues(pairs);
Price price = new Price();
price.Symbol = symbol;
price.Date = DateTime.Now;
price.Source = Price.PriceSource.BarChart;
if (values.ContainsKey("lowPrice")) price.Low = values["lowPrice"];
if (values.ContainsKey("highPrice")) price.High = values["highPrice"];
if (values.ContainsKey("openPrice")) price.Open = values["openPrice"];
if (values.ContainsKey("lastPrice")) price.Close = values["lastPrice"];
if (values.ContainsKey("volume")) price.Volume = (long)values["volume"];
if (values.ContainsKey("prevClose")) price.PrevClose = values["prevClose"];
else price.PrevClose = price.Close;
price.AdjClose = price.Close;
MarketDataHelper.CheckPrice(price);
if (double.IsNaN(price.Open) && double.IsNaN(price.High) && double.IsNaN(price.Low) && double.IsNaN(price.Close))
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("*** No closing price for {0}", price.Symbol));
return null;
}
if (double.IsNaN(price.Open)) price.Open = 0.00;
if (double.IsNaN(price.Low)) price.Low = 0.00;
if (double.IsNaN(price.High)) price.High = 0.00;
if (0 != price.Close && 0 == price.Open && 0 == price.High && 0 == price.Low)
{
price.Open = price.High = price.Low = price.Close;
}
return price;
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG, exception.ToString());
return null;
}
}
private static Dictionary<String, double> CreateBarChartValues(String[] pairs)
{
Dictionary<String, double> barChartValues = new Dictionary<String, double>();
if (null == pairs || 0 == pairs.Length) return barChartValues;
foreach (String item in pairs)
{
String[] nameValue = item.Split(":");
if (null == nameValue || 2 != nameValue.Length) continue;
String name = nameValue[0].Replace("\"", null);
String value = nameValue[1].Replace("\"",null);
double result = 0.00;
if (barChartValues.ContainsKey(name)) continue;
if (double.TryParse(value, out result))
{
barChartValues.Add(name, result);
}
}
return barChartValues;
}
// *******************************************************************************************************************************************************************************
// ************************************************************** H I S T O R I C A L P R I C E S - B I G C H A R T S ********************************************************
// *******************************************************************************************************************************************************************************
// ************************************************************** H I S T O R I C A L P R I C E S - B I G C H A R T S ********************************************************
// *******************************************************************************************************************************************************************************
/// <summary>
/// Gets historical pricing from BigCharts
/// </summary>
@@ -5111,24 +5200,25 @@ namespace MarketData.Helper
/// <param name="startDate">Most recent date</param>
/// <param name="endDate">Most historical date</param>
/// <returns>Prices</returns>
public static Prices GetPricesAsOf(String symbol, DateTime startDate,DateTime endDate)
public static Prices GetPricesAsOf(String symbol, DateTime startDate, DateTime endDate)
{
Prices prices=new Prices();
Prices prices = new Prices();
if(null==symbol)return null;
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
if(null!=companyProfile && companyProfile.FreezePricing)
if (null == symbol) return null;
CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol);
if (null != companyProfile && companyProfile.FreezePricing)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol));
return null;
}
DateGenerator dateGenerator=new DateGenerator();
List<DateTime> dates=dateGenerator.GenerateHistoricalDates(startDate,endDate);
for(int index=0;index<dates.Count;index++)
DateGenerator dateGenerator = new DateGenerator();
List<DateTime> dates = dateGenerator.GenerateHistoricalDates(startDate, endDate);
for (int index = 0; index < dates.Count; index++)
{
DateTime currentDate=dates[index];
Price price=GetPriceAsOf(symbol,currentDate);
if(null==price){MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No data for {0} on {1}",symbol,Utility.DateTimeToStringMMHDDHYYYY(currentDate)));continue;};
DateTime currentDate = dates[index];
Price price = GetPriceAsOf(symbol, currentDate);
if (null == price) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("No data for {0} on {1}", symbol, Utility.DateTimeToStringMMHDDHYYYY(currentDate))); continue; }
;
prices.Add(price);
}
return prices;

230
MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs
View File

@@ -165,47 +165,15 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesBigCharts]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdatePricesYahooSweep(DateTime? startDate=null) // get prices from Yahoo
{
Profiler profiler=new Profiler();
try
{
symbols=PricingDA.GetSymbols();
currentIndex=0;
if(null==startDate) startDate=DateTime.Now;
while(true)
{
List<String> queueSymbols=GetQueueSymbols();
if(null==queueSymbols||0==queueSymbols.Count) break;
ManualResetEvent[] resetEvents=new ManualResetEvent[queueSymbols.Count];
for(int eventIndex=0;eventIndex<resetEvents.Length;eventIndex++)
{
resetEvents[eventIndex]=new ManualResetEvent(false);
}
for(int index=0;index<queueSymbols.Count;index++)
{
PricingThreadHelper pricingThreadHelper=new PricingThreadHelper(queueSymbols[index],startDate.Value,resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesYahooSweep,pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"UpdatePricesYahooSweep, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesYahooSweep]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdatePricesAsOfSymbol(String symbol,DateTime asOf) // get prices from BigCharts
public bool UpdatePricesYahooSweep(DateTime? startDate = null) // get prices from Yahoo
{
Profiler profiler=new Profiler();
Profiler profiler = new Profiler();
try
{
symbols=new List<String>();
symbols.Add(symbol);
currentIndex=0;
symbols = PricingDA.GetSymbols();
currentIndex = 0;
if (null == startDate) startDate = DateTime.Now;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
@@ -217,19 +185,92 @@ namespace MarketData.Helper
}
for (int index = 0; index < queueSymbols.Count; index++)
{
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index],asOf);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesAsOf, pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], startDate.Value, resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesYahooSweep, pricingThreadHelper);
try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; }
}
MDTrace.WriteLine(LogLevel.DEBUG,"UpdatePricesAsOfSymbol, waiting for queued items to complete.");
MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesYahooSweep, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesAsOfSymbol]End, total took {0}(ms)",profiler.End()));
}
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesYahooSweep]End, total took {0}(ms)", profiler.End()));
}
}
/// <summary>
/// UpdatePricesBarChartSweep - This feed will retrieve latest price. It will not use a date
/// </summary>
/// <param name="startDate"></param>
/// <returns></returns>
public bool UpdatePricesBarChartSweep(DateTime? startDate = null) // get prices from BarCharts
{
Profiler profiler = new Profiler();
try
{
symbols = PricingDA.GetSymbols();
if (null == startDate) startDate = DateTime.Now;
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], startDate.Value, resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesBarChartSweep, pricingThreadHelper);
try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; }
}
MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesBarChartSweep, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesBarChartSweep]End, total took {0}(ms)", profiler.End()));
}
}
public bool UpdatePricesAsOfSymbol(String symbol, DateTime asOf) // get prices from BigCharts
{
Profiler profiler = new Profiler();
try
{
symbols = new List<String>();
symbols.Add(symbol);
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index], asOf);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesAsOf, pricingThreadHelper);
try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; }
}
MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesAsOfSymbol, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesAsOfSymbol]End, total took {0}(ms)", profiler.End()));
}
}
public bool UpdatePriceAsOfSymbolYahoo(String symbol,DateTime asOf) // get prices from Yahoo
{
@@ -492,19 +533,33 @@ namespace MarketData.Helper
{
pricingThreadHelper.ResetEvent.Set();
}
}
public void ThreadPoolCallbackUpdatePricesBarChartSweep(Object pricingThreadHelperContext)
{
PricingThreadHelper pricingThreadHelper=(PricingThreadHelper)pricingThreadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (BarChart), Thread {0} started for {1}...",Thread.CurrentThread.ManagedThreadId,pricingThreadHelper.Symbol));
UpdatePriceBarChartSweep(pricingThreadHelper.Symbol,pricingThreadHelper.PricingDate);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (BarChart), Thread {0} ended for {1}",Thread.CurrentThread.ManagedThreadId,pricingThreadHelper.Symbol));
}
finally
{
pricingThreadHelper.ResetEvent.Set();
}
}
public void ThreadPoolCallbackUpdatePricesAsOf(Object pricingThreadHelperContext)
{
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
UpdatePriceAsOfEx(pricingThreadHelper.Symbol,pricingThreadHelper.StartDate.Value);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
UpdatePriceAsOfEx(pricingThreadHelper.Symbol, pricingThreadHelper.StartDate.Value);
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
}
finally
{
pricingThreadHelper.ResetEvent.Set();
pricingThreadHelper.ResetEvent.Set();
}
}
public void ThreadPoolCallbackGetMissingPrice(Object pricingThreadHelperContext)
@@ -733,7 +788,7 @@ namespace MarketData.Helper
price=MarketDataHelper.GetDailyPrice(symbol,pricingDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No price (UpdatePriceYahoo) for '"+symbol+"'");
MDTrace.WriteLine(LogLevel.DEBUG,"No price (UpdatePriceYahooSweep) for '"+symbol+"'");
return;
}
else if(!price.IsValid)
@@ -752,35 +807,86 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception {0}",exception.ToString()));
}
}
public static void UpdatePriceAsOfEx(String symbol,DateTime asOf)
/// <summary>
/// This feed is intended to supplement BigCharts and Yahoo feed.
/// It should run in the UPDATEDAILY2 chain after Yahoo during the nightly price capture.
/// It will only retrieve price for the current date AND it expects us to have a price for the previous business date.
/// In other words it will not fill pricing gaps, it will only add "tonights" price to an already established pricing history
/// </summary>
/// <param name="symbol"></param>
/// <param name="pricingDate"></param>
public static void UpdatePriceBarChartSweep(String symbol, DateTime pricingDate)
{
try
{
DateGenerator dateGenerator = new DateGenerator();
DateTime prevBusinessDay = dateGenerator.FindPrevBusinessDay(pricingDate);
Price price = PricingDA.GetPrice(symbol); // retrieve the latest price for this symbol
if (null == price) // we don't have a price for the symbol so return
{
return;
}
if (price.Date.Date.Equals(pricingDate.Date))
{
MDTrace.WriteLine(LogLevel.DEBUG, $"Already have latest price for {symbol} on {price.Date.ToShortDateString()}");
return;
}
if (!prevBusinessDay.Date.Equals(price.Date.Date))
{
MDTrace.WriteLine(LogLevel.DEBUG, $"Expected a date of {prevBusinessDay.ToShortDateString()} for {symbol} but found {price.Date.ToShortDateString()}");
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,$"Latest price for {symbol} is {price.Date.Date.ToShortDateString()}");
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("UpdatePriceBarChartSweep: Requesting latest price for {0} ", symbol));
price = MarketDataHelper.GetLatestPriceBarChart(symbol);
if (null == price)
{
MDTrace.WriteLine(LogLevel.DEBUG, "(UpdatePriceBarChartSweep) No price for '" + symbol + "'");
return;
}
else if (!price.IsValid)
{
MDTrace.WriteLine(LogLevel.DEBUG, "(UpdatePriceBarChartSweep) Invalid price for '" + symbol + "'");
return;
}
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("(UpdatePriceBarChartSweep) Inserting price {0}...", price.ToString()));
PricingDA.InsertPrice(price);
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("(UpdatePriceBarChartSweep) Exception {0}", exception.ToString()));
}
}
public static void UpdatePriceAsOfEx(String symbol, DateTime asOf)
{
try
{
DateTime latestDate = PricingDA.GetLatestDate(symbol);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Latest pricing date for {0} is {1}",symbol,latestDate.ToShortDateString()));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Latest pricing date for {0} is {1}", symbol, latestDate.ToShortDateString()));
if (latestDate.Equals(asOf))
{
MDTrace.WriteLine(LogLevel.DEBUG,"Already have price for '" + symbol + "' on "+Utility.DateTimeToStringMMHDDHYYYY(asOf));
MDTrace.WriteLine(LogLevel.DEBUG, "Already have price for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(asOf));
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Retrieving price for {0} on {1} from BigCharts",symbol,asOf.ToShortDateString()));
Price price = MarketDataHelper.GetPriceAsOf(symbol,asOf);
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Retrieving price for {0} on {1} from BigCharts", symbol, asOf.ToShortDateString()));
Price price = MarketDataHelper.GetPriceAsOf(symbol, asOf);
if (null == price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Error retrieving price for {0} on {1}",symbol,asOf.ToShortDateString()));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Error retrieving price for {0} on {1}", symbol, asOf.ToShortDateString()));
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,Price.Header);
MDTrace.WriteLine(LogLevel.DEBUG,price.ToString());
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Inserting price {0}...",price.ToString()));
MDTrace.WriteLine(LogLevel.DEBUG, Price.Header);
MDTrace.WriteLine(LogLevel.DEBUG, price.ToString());
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting price {0}...", price.ToString()));
PricingDA.InsertPrice(price);
MDTrace.WriteLine(LogLevel.DEBUG,"");
MDTrace.WriteLine(LogLevel.DEBUG, "");
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
MDTrace.WriteLine(LogLevel.DEBUG, exception.ToString());
}
}
}
public static void LoadPricesSymbolEx(String symbol,DateTime pricingDate)
{