873 lines
39 KiB
C#
Executable File
873 lines
39 KiB
C#
Executable File
using MarketData.MarketDataModel;
|
|
using MarketData.DataAccess;
|
|
using MarketData.Utils;
|
|
|
|
namespace MarketData.Helper
|
|
{
|
|
public class PricingThreadHelper : ThreadHelper
|
|
{
|
|
private DateTime pricingDate=DateTime.Parse("12-31-9999");
|
|
private DateTime? startDate;
|
|
|
|
public PricingThreadHelper(String symbol,ManualResetEvent resetEvent)
|
|
{
|
|
Symbol=symbol;
|
|
ResetEvent=resetEvent;
|
|
}
|
|
public PricingThreadHelper(String symbol, DateTime pricingDate,ManualResetEvent resetEvent)
|
|
{
|
|
Symbol = symbol;
|
|
this.pricingDate=pricingDate;
|
|
ResetEvent = resetEvent;
|
|
}
|
|
public PricingThreadHelper(String symbol, ManualResetEvent resetEvent,DateTime? startDate=null)
|
|
{
|
|
Symbol = symbol;
|
|
this.pricingDate=DateTime.Now;
|
|
this.startDate=startDate;
|
|
ResetEvent = resetEvent;
|
|
}
|
|
public DateTime PricingDate
|
|
{
|
|
get{return pricingDate;}
|
|
}
|
|
public DateTime? StartDate
|
|
{
|
|
get{return startDate;}
|
|
}
|
|
}
|
|
|
|
public class PricingMarketDataHelper
|
|
{
|
|
private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads;
|
|
private static int WaitBetweenRequests = 125;
|
|
private List<String> symbols;
|
|
private int currentIndex = 0;
|
|
|
|
public PricingMarketDataHelper()
|
|
{
|
|
}
|
|
public static Price RollPriceForward(String symbol)
|
|
{
|
|
try
|
|
{
|
|
DateGenerator dateGenerator = new DateGenerator();
|
|
DateTime latestDate = PremarketDA.GetLatestMarketDate();
|
|
if (Utility.IsEpoch(latestDate)) return null;
|
|
Price latestPrice = PricingDA.GetPrice(symbol);
|
|
if (null == latestPrice) return null;
|
|
DateTime latestPriceDateSymbol = latestPrice.Date;
|
|
latestPrice.Date = latestDate;
|
|
PricingDA.InsertPrice(latestPrice);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("{0} price rolled forward from {1} to {2}", symbol, latestPriceDateSymbol.ToShortDateString(), latestDate.ToShortDateString()));
|
|
return latestPrice;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception rolling price forward for {0}. Exception was {1}", symbol, exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public bool UpdateLatestPriceAll()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
symbols = PricingDA.GetSymbols();
|
|
currentIndex = 0;
|
|
while (true)
|
|
{
|
|
List<String> queueSymbols = GetQueueSymbols();
|
|
if (null == queueSymbols || 0 == queueSymbols.Count) break;
|
|
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
|
|
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
|
|
{
|
|
resetEvents[eventIndex] = new ManualResetEvent(false);
|
|
}
|
|
for (int index = 0; index < queueSymbols.Count; index++)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index]);
|
|
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdateLatestPriceAll, pricingThreadHelper);
|
|
try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; }
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "UpdateLatestPriceAll, waiting for queued items to complete.");
|
|
WaitHandle.WaitAll(resetEvents);
|
|
}
|
|
return true;
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdateLatestPriceAll]End, total took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
|
|
// **************************************************************************************************************************************
|
|
// ************************************************************* R O B I N H O O D ******************************************************
|
|
// **************************************************************************************************************************************
|
|
/// <summary>
|
|
/// Robinhood will only give us latest price. The price will have a date associated with it.. todays date. so the date provided needs to match.
|
|
/// This feed works the same way as the BarChartSweep in that we will only attempt to fetch prices that are missing today for which we had prices yesterday.
|
|
/// </summary>
|
|
/// <param name="startDate"></param>
|
|
/// <returns></returns>
|
|
public bool UpdatePricesRobinhoodSweep(DateTime startDate)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
int maxThreads = 20; // I am tweaking these to back into a total runtime of 40 minutes for all stocks. maxThreads=10 waitBetweenRequests=500 = 86 minutes
|
|
int waitBetweenRequests = 200;
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "[UpdatePricesRobinhoodSweep]Enter");
|
|
DateGenerator dateGenerator = new DateGenerator();
|
|
startDate = dateGenerator.GetPrevBusinessDay(startDate); // make sure we are given a valid business date
|
|
DateTime requiredDate = dateGenerator.FindPrevBusinessDay(startDate); // now go and fetch yesterday (prev business day)
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesRobinhoodSweep] Determining what prices we need on {startDate.ToShortDateString()} by examing what prices we had on {requiredDate.ToShortDateString()}");
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesRobinhoodSweep] Retrieve universe of symbols...");
|
|
symbols = PricingDA.GetSymbols(); // get the list of symbols in the universe
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesRobinhoodSweep] Retrieve latest dates...");
|
|
Dictionary<String, DateTime> latestDates = PricingDA.GetLatestDates(symbols); // get the latest pricing date for these symbols
|
|
List<String> symbolsToFetch = new List<String>();
|
|
List<String> keys = latestDates.Keys.ToList();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesRobinhoodSweep] Filtering...");
|
|
foreach (String key in keys) // go through the latest dates and select those for which
|
|
{ // we have a price on previous business date
|
|
if (latestDates[key].Date.Date.Equals(requiredDate.Date))
|
|
{
|
|
symbolsToFetch.Add(key);
|
|
}
|
|
}
|
|
symbols = symbolsToFetch; // These are the symbols that we know we need to fetch
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesRobinhoodSweep] Preparing to supplement {symbolsToFetch.Count} prices for pricing date {startDate.ToShortDateString()} .");
|
|
currentIndex = 0;
|
|
while (true)
|
|
{
|
|
List<String> queueSymbols = GetQueueSymbols(maxThreads);
|
|
if (null == queueSymbols || 0 == queueSymbols.Count) break;
|
|
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
|
|
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
|
|
{
|
|
resetEvents[eventIndex] = new ManualResetEvent(false);
|
|
}
|
|
for (int index = 0; index < queueSymbols.Count; index++)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], startDate, resetEvents[index]);
|
|
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesRobinhoodSweep, pricingThreadHelper);
|
|
try { Thread.Sleep(waitBetweenRequests); } catch (Exception) {; }
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesRobinhoodSweep, waiting for queued items to complete.");
|
|
WaitHandle.WaitAll(resetEvents);
|
|
}
|
|
return true;
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesRobinhoodSweep] End, total took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
// **************************************************************************************************************************************
|
|
// ************************************************************* E N D R O B I N H O O D ***********************************************
|
|
// **************************************************************************************************************************************
|
|
|
|
// **************************************************************************************************************************************
|
|
// ******************************************************************* Y A H O O *******************************************************
|
|
// **************************************************************************************************************************************
|
|
|
|
public bool UpdatePricesYahoo(DateTime? startDate = null) // get prices from Yahoo
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
symbols = PricingDA.GetSymbols();
|
|
currentIndex = 0;
|
|
if (null == startDate) startDate = DateTime.Now;
|
|
while (true)
|
|
{
|
|
List<String> queueSymbols = GetQueueSymbols();
|
|
if (null == queueSymbols || 0 == queueSymbols.Count) break;
|
|
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
|
|
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
|
|
{
|
|
resetEvents[eventIndex] = new ManualResetEvent(false);
|
|
}
|
|
for (int index = 0; index < queueSymbols.Count; index++)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], startDate.Value, resetEvents[index]);
|
|
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesYahoo, pricingThreadHelper);
|
|
try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; }
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesYahoo, waiting for queued items to complete.");
|
|
WaitHandle.WaitAll(resetEvents);
|
|
}
|
|
return true;
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesYahoo]End, total took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
|
|
public bool UpdatePricesYahooSweep(DateTime? startDate = null) // get prices from Yahoo
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
symbols = PricingDA.GetSymbols();
|
|
currentIndex = 0;
|
|
if (null == startDate) startDate = DateTime.Now;
|
|
while (true)
|
|
{
|
|
List<String> queueSymbols = GetQueueSymbols();
|
|
if (null == queueSymbols || 0 == queueSymbols.Count) break;
|
|
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
|
|
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
|
|
{
|
|
resetEvents[eventIndex] = new ManualResetEvent(false);
|
|
}
|
|
for (int index = 0; index < queueSymbols.Count; index++)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], startDate.Value, resetEvents[index]);
|
|
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesYahooSweep, pricingThreadHelper);
|
|
try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; }
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesYahooSweep, waiting for queued items to complete.");
|
|
WaitHandle.WaitAll(resetEvents);
|
|
}
|
|
return true;
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesYahooSweep]End, total took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
|
|
// ********************************************************************************************************************************************
|
|
// ******************************************************************* E N D Y A H O O *******************************************************
|
|
// ********************************************************************************************************************************************
|
|
|
|
|
|
// ********************************************************************************************************************************************
|
|
// ********************************************************************** B A R C H A R T ***************************************************
|
|
// ********************************************************************************************************************************************
|
|
/// <summary>
|
|
/// UpdatePricesBarChartSweep - This feed is intended to supplement BigCharts nightly price feed in UPDATEDAILY2 by filling any gaps that may occur
|
|
/// in that feed. This feed will only pull a price for which we had a price yesterday (findprevbusinessday) and for which do not have
|
|
/// a price tonight. If BigCharts was successful then we can expect this method to retrieve nothing. If BigCharts fails to give us a price
|
|
/// tonight BUT did in fact give us a price yesterday then this method will attempt to rerieve that symbol from BarChart and insert that
|
|
/// price into the database with the appropriate date (tonight) and with the appropriate Source (BarChart).
|
|
/// </summary>
|
|
/// <param name="startDate"></param>
|
|
/// <returns></returns>
|
|
public bool UpdatePricesBarChartSweep(DateTime? startDate = null)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "[UpdatePricesBarChartSweep]Enter");
|
|
DateGenerator dateGenerator = new DateGenerator();
|
|
if (null == startDate) startDate = DateTime.Now;
|
|
startDate = dateGenerator.GetPrevBusinessDay(startDate.Value); // make sure we are given a valid business date
|
|
DateTime requiredDate = dateGenerator.FindPrevBusinessDay(startDate.Value); // now go and fetch yesterday (prev business day)
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweep] Determining what prices we need on {startDate.Value.ToShortDateString()} by examing what prices we had on {requiredDate.ToShortDateString()}");
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweep] Retrieve universe of symbols...");
|
|
symbols = PricingDA.GetSymbols(); // get the list of symbols in the universe
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweep] Retrieve latest dates...");
|
|
Dictionary<String, DateTime> latestDates = PricingDA.GetLatestDates(symbols); // get the latest pricing date for these symbols
|
|
List<String> symbolsToFetch = new List<String>();
|
|
List<String> keys = latestDates.Keys.ToList();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweep] Filtering...");
|
|
foreach (String key in keys) // go through the latest dates and select those for which
|
|
{ // we have a price on previous business date
|
|
if (latestDates[key].Date.Date.Equals(requiredDate.Date))
|
|
{
|
|
symbolsToFetch.Add(key);
|
|
}
|
|
}
|
|
symbols = symbolsToFetch; // These are the symbols that we know we need to fetch
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweep] Preparing to supplement {symbolsToFetch.Count} prices for pricing date {startDate.Value.ToShortDateString()} .");
|
|
currentIndex = 0;
|
|
while (true)
|
|
{
|
|
List<String> queueSymbols = GetQueueSymbols();
|
|
if (null == queueSymbols || 0 == queueSymbols.Count) break;
|
|
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
|
|
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
|
|
{
|
|
resetEvents[eventIndex] = new ManualResetEvent(false);
|
|
}
|
|
for (int index = 0; index < queueSymbols.Count; index++)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], startDate.Value, resetEvents[index]);
|
|
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesBarChartSweep, pricingThreadHelper);
|
|
try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; }
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesBarChartSweep, waiting for queued items to complete.");
|
|
WaitHandle.WaitAll(resetEvents);
|
|
}
|
|
return true;
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesBarChartSweep]End, total took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// UpdatePricesBarChart - This feed is used by UPDATEDAILY2 as a final sweep for open positions.
|
|
/// If we already have the price for the given date then it will not be fetched.
|
|
/// </summary>
|
|
/// <param name="symbolsToFetch">The symbols to fetch. This should be a list of symbols for which we need pricing for startDate</param>
|
|
/// <param name="startDate">The date we are requesting</param>
|
|
/// <returns></returns>
|
|
public bool UpdatePricesBarChartSweepOpenPositions(List<String> symbolsToFetch, DateTime? startDate = null)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "[UpdatePricesBarChartSweepOpenPositions]Enter");
|
|
DateGenerator dateGenerator = new DateGenerator();
|
|
if(null == symbolsToFetch || 0==symbolsToFetch.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweepOpenPositions] No symbols provided, nothing to do");
|
|
return false;
|
|
}
|
|
if (null == startDate) startDate = DateTime.Now;
|
|
symbols = symbolsToFetch;
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"[UpdatePricesBarChartSweepOpenPositions] Preparing to supplement {symbolsToFetch.Count} prices for pricing date {startDate.Value.ToShortDateString()} .");
|
|
currentIndex = 0;
|
|
while (true)
|
|
{
|
|
List<String> queueSymbols = GetQueueSymbols();
|
|
if (null == queueSymbols || 0 == queueSymbols.Count) break;
|
|
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
|
|
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
|
|
{
|
|
resetEvents[eventIndex] = new ManualResetEvent(false);
|
|
}
|
|
for (int index = 0; index < queueSymbols.Count; index++)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], startDate.Value, resetEvents[index]);
|
|
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesBarChartSweep, pricingThreadHelper);
|
|
try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; }
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesBarChartSweepOpenPositions, waiting for queued items to complete.");
|
|
WaitHandle.WaitAll(resetEvents);
|
|
}
|
|
return true;
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesBarChartSweepOpenPositions]End, total took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
|
|
// ********************************************************************************************************************************************
|
|
// ********************************************************************** E N D B A R C H A R T ***************************************************
|
|
// ********************************************************************************************************************************************
|
|
|
|
public bool UpdatePriceAsOfSymbolYahoo(String symbol, DateTime asOf) // get prices from Yahoo
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
symbols = new List<String>();
|
|
symbols.Add(symbol);
|
|
currentIndex = 0;
|
|
while (true)
|
|
{
|
|
List<String> queueSymbols = GetQueueSymbols();
|
|
if (null == queueSymbols || 0 == queueSymbols.Count) break;
|
|
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
|
|
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
|
|
{
|
|
resetEvents[eventIndex] = new ManualResetEvent(false);
|
|
}
|
|
for (int index = 0; index < queueSymbols.Count; index++)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], asOf, resetEvents[index]);
|
|
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesYahoo, pricingThreadHelper);
|
|
try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; }
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "UpdatePricesAsOfSymbol, waiting for queued items to complete.");
|
|
WaitHandle.WaitAll(resetEvents);
|
|
}
|
|
return true;
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdatePricesAsOfSymbol]End, total took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
|
|
public bool GetMissingPricesSymbol(String symbol, DateTime? startDate = null)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
List<String> symbols = new List<String>();
|
|
symbols.Add(symbol);
|
|
this.symbols = symbols;
|
|
currentIndex = 0;
|
|
while (true)
|
|
{
|
|
List<String> queueSymbols = GetQueueSymbols();
|
|
if (null == queueSymbols || 0 == queueSymbols.Count) break;
|
|
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
|
|
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
|
|
{
|
|
resetEvents[eventIndex] = new ManualResetEvent(false);
|
|
}
|
|
for (int index = 0; index < queueSymbols.Count; index++)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index], startDate);
|
|
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackGetMissingPrice, pricingThreadHelper);
|
|
try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; }
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Get missing prices, waiting for queued items to complete.");
|
|
WaitHandle.WaitAll(resetEvents);
|
|
}
|
|
return true;
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetMissingPrices]End, total took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
|
|
public bool GetMissingPrices(DateTime? startDate = null)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
this.symbols = PricingDA.GetSymbols();
|
|
currentIndex = 0;
|
|
while (true)
|
|
{
|
|
List<String> queueSymbols = GetQueueSymbols();
|
|
if (null == queueSymbols || 0 == queueSymbols.Count) break;
|
|
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
|
|
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
|
|
{
|
|
resetEvents[eventIndex] = new ManualResetEvent(false);
|
|
}
|
|
for (int index = 0; index < queueSymbols.Count; index++)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index], startDate);
|
|
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackGetMissingPrice, pricingThreadHelper);
|
|
try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; }
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Get missing prices, waiting for queued items to complete.");
|
|
WaitHandle.WaitAll(resetEvents);
|
|
}
|
|
return true;
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetMissingPrices]End, total took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
|
|
public bool UpdateLatestPriceSymbol(String symbol)
|
|
{
|
|
List<String> symbols = new List<String>();
|
|
symbols.Add(symbol);
|
|
return UpdateLatestPrices(symbols);
|
|
}
|
|
|
|
public bool UpdateLatestPrices(List<String> symbols)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
DateTime pricingDate = DateTime.Now;
|
|
this.symbols = symbols;
|
|
currentIndex = 0;
|
|
while (true)
|
|
{
|
|
List<String> queueSymbols = GetQueueSymbols();
|
|
if (null == queueSymbols || 0 == queueSymbols.Count) break;
|
|
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
|
|
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
|
|
{
|
|
resetEvents[eventIndex] = new ManualResetEvent(false);
|
|
}
|
|
for (int index = 0; index < queueSymbols.Count; index++)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], pricingDate, resetEvents[index]);
|
|
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdateLatestPrice, pricingThreadHelper);
|
|
try { Thread.Sleep(WaitBetweenRequests); } catch (Exception) {; }
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Load prices, waiting for queued items to complete.");
|
|
WaitHandle.WaitAll(resetEvents);
|
|
}
|
|
return true;
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[UpdateLatestPrices]End, total took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
|
|
// ************************************************************************************************************************************
|
|
// *************************************************** T H R E A D P O O L C A L L B A C K ******************************************
|
|
// ************************************************************************************************************************************
|
|
public void ThreadPoolCallbackUpdateLatestPriceAll(Object pricingThreadHelperContext)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("UpdateLatestPriceAll, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
|
|
UpdateLatestPriceAllEx(pricingThreadHelper.Symbol);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("UpdateLatestPriceAll, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
|
|
}
|
|
finally
|
|
{
|
|
pricingThreadHelper.ResetEvent.Set();
|
|
}
|
|
}
|
|
|
|
public void ThreadPoolCallbackUpdatePricesRobinhoodSweep(Object pricingThreadHelperContext)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Robinhood), Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
|
|
UpdatePriceRobinhoodSweepEx(pricingThreadHelper.Symbol, pricingThreadHelper.PricingDate);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Robinhood), Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
|
|
}
|
|
finally
|
|
{
|
|
pricingThreadHelper.ResetEvent.Set();
|
|
}
|
|
}
|
|
|
|
public void ThreadPoolCallbackUpdatePricesYahoo(Object pricingThreadHelperContext)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Yahoo), Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
|
|
UpdatePriceYahoo(pricingThreadHelper.Symbol, pricingThreadHelper.PricingDate);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Yahoo), Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
|
|
}
|
|
finally
|
|
{
|
|
pricingThreadHelper.ResetEvent.Set();
|
|
}
|
|
}
|
|
|
|
public void ThreadPoolCallbackUpdatePricesYahooSweep(Object pricingThreadHelperContext)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Yahoo), Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
|
|
UpdatePriceYahooSweep(pricingThreadHelper.Symbol, pricingThreadHelper.PricingDate);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (Yahoo), Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
|
|
}
|
|
finally
|
|
{
|
|
pricingThreadHelper.ResetEvent.Set();
|
|
}
|
|
}
|
|
|
|
public void ThreadPoolCallbackUpdatePricesBarChartSweep(Object pricingThreadHelperContext)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (BarChart), Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
|
|
UpdatePriceBarChartSweep(pricingThreadHelper.Symbol, pricingThreadHelper.PricingDate);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Load price (BarChart), Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
|
|
}
|
|
finally
|
|
{
|
|
pricingThreadHelper.ResetEvent.Set();
|
|
}
|
|
}
|
|
|
|
public void ThreadPoolCallbackGetMissingPrice(Object pricingThreadHelperContext)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
|
|
try
|
|
{
|
|
GetMissingPricesEx(pricingThreadHelper.Symbol, pricingThreadHelper.StartDate == null ? new DateTime() : pricingThreadHelper.StartDate.Value);
|
|
}
|
|
finally
|
|
{
|
|
pricingThreadHelper.ResetEvent.Set();
|
|
}
|
|
}
|
|
|
|
public void ThreadPoolCallbackUpdateLatestPrice(Object pricingThreadHelperContext)
|
|
{
|
|
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
|
|
try
|
|
{
|
|
Price latestPrice = MarketDataHelper.GetLatestPrice(pricingThreadHelper.Symbol);
|
|
if (null == latestPrice)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("***** Error retrieving latest price for {0} *****", pricingThreadHelper.Symbol));
|
|
return;
|
|
}
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG, Price.Header);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, latestPrice.ToString());
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Removing price for '" + pricingThreadHelper.Symbol + "' on " + Utility.DateTimeToStringMMSDDSYYYY(latestPrice.Date));
|
|
PricingDA.DeletePrice(pricingThreadHelper.Symbol, latestPrice.Date);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Adding price for '" + pricingThreadHelper.Symbol + "' on " + Utility.DateTimeToStringMMSDDSYYYY(latestPrice.Date));
|
|
PricingDA.InsertPrice(latestPrice);
|
|
}
|
|
finally
|
|
{
|
|
pricingThreadHelper.ResetEvent.Set();
|
|
}
|
|
}
|
|
|
|
// ****************************************************************************************************************************************
|
|
// ****************************************************************************************************************************************
|
|
// ****************************************************************************************************************************************
|
|
private List<String> GetQueueSymbols()
|
|
{
|
|
List<String> queueSymbols = new List<String>();
|
|
int index = currentIndex;
|
|
for (; index < currentIndex + MaxThreads && index < symbols.Count; index++)
|
|
{
|
|
queueSymbols.Add(symbols[index]);
|
|
}
|
|
currentIndex = index;
|
|
return queueSymbols;
|
|
}
|
|
|
|
private List<String> GetQueueSymbols(int maxThreads)
|
|
{
|
|
List<String> queueSymbols = new List<String>();
|
|
int index = currentIndex;
|
|
for (; index < currentIndex + maxThreads && index < symbols.Count; index++)
|
|
{
|
|
queueSymbols.Add(symbols[index]);
|
|
}
|
|
currentIndex = index;
|
|
return queueSymbols;
|
|
}
|
|
|
|
// ********************************************************************************************************************************************
|
|
// ************************************************************ W O R K E R S *****************************************************************
|
|
// ********************************************************************************************************************************************
|
|
public static void GetMissingPricesEx(String symbol, DateTime startDate)
|
|
{
|
|
try
|
|
{
|
|
DateGenerator dateGenerator = new DateGenerator();
|
|
TimeSpan oneDay = new TimeSpan(1, 0, 0, 0);
|
|
DateTime? historicalDate = null;
|
|
DateTime minDate = Constants.MIN_PRICING_DATE;
|
|
|
|
List<DateTime> pricingDatesList = PricingDA.GetPricingDatesForSymbol(symbol);
|
|
if (null == pricingDatesList || 0 == pricingDatesList.Count) return;
|
|
pricingDatesList = pricingDatesList.Where(x => x.Date >= minDate.Date).ToList();
|
|
if (null == pricingDatesList || 0 == pricingDatesList.Count) return;
|
|
DateTime beginDate = pricingDatesList[0];
|
|
DateTime endDate = pricingDatesList[pricingDatesList.Count - 1];
|
|
Dictionary<DateTime, DateTime> pricingDates = new Dictionary<DateTime, DateTime>();
|
|
for (int listIndex = 0; listIndex < pricingDatesList.Count; listIndex++)
|
|
{
|
|
DateTime pricingDate = pricingDatesList[listIndex];
|
|
pricingDates.Add(pricingDate.Date, pricingDate.Date);
|
|
}
|
|
Dictionary<DateTime, Price> marketPrices = new Dictionary<DateTime, Price>();
|
|
if (Utility.IsEpoch(startDate)) historicalDate = beginDate;
|
|
else historicalDate = startDate;
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Working on '" + symbol + "' from " + Utility.DateTimeToStringMMHDDHYYYY(historicalDate.Value) + " to " + Utility.DateTimeToStringMMHDDHYYYY(endDate));
|
|
Prices prices = MarketDataHelper.GetDailyPrices(symbol, historicalDate.Value, endDate); // yahoo finance
|
|
// if (null == prices || 0 == prices.Count) prices = MarketDataHelper.GetPricesAsOf(symbol, historicalDate.Value, endDate); // try big charts
|
|
if (null == prices || 0 == prices.Count) return;
|
|
for (int listIndex = 0; listIndex < prices.Count; listIndex++)
|
|
{
|
|
Price price = prices[listIndex];
|
|
if (!marketPrices.ContainsKey(price.Date.Date)) marketPrices.Add(price.Date, price);
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Retrieved {0} prices from feed for {1}", prices.Count, symbol));
|
|
bool added = false;
|
|
for (DateTime date = historicalDate.Value.Date; date < endDate.Date; date += oneDay)
|
|
{
|
|
if (pricingDates.ContainsKey(date.Date)) continue;
|
|
if (dateGenerator.IsWeekend(date)) continue;
|
|
Price price = null;
|
|
if (!marketPrices.ContainsKey(date.Date)) continue;
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting price {0}...", price.ToString()));
|
|
price = marketPrices[date.Date];
|
|
MDTrace.WriteLine(LogLevel.DEBUG, price.ToString());
|
|
PricingDA.InsertPrice(price);
|
|
added = true;
|
|
}
|
|
if (!added)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "There were no new prices to add for '" + symbol);
|
|
}
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception {0}", exception.ToString()));
|
|
}
|
|
}
|
|
|
|
public static void UpdateLatestPriceAllEx(String symbol)
|
|
{
|
|
try
|
|
{
|
|
symbol = symbol.ToUpper();
|
|
Price latestPrice = MarketDataHelper.GetLatestPrice(symbol);
|
|
if (null == latestPrice)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Error retrieving latest price.");
|
|
return;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, Price.Header);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, latestPrice.ToString());
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Removing price for '" + symbol + "' on " + Utility.DateTimeToStringMMSDDSYYYY(latestPrice.Date));
|
|
PricingDA.DeletePrice(symbol, latestPrice.Date);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting price {0}...", latestPrice.ToString()));
|
|
PricingDA.InsertPrice(latestPrice);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, exception.ToString());
|
|
}
|
|
}
|
|
|
|
public static void UpdatePriceRobinhoodSweepEx(String symbol, DateTime pricingDate)
|
|
{
|
|
try
|
|
{
|
|
Price price = PricingDA.GetPrice(symbol, pricingDate);
|
|
if (null != price)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Already have latest price for '" + symbol + "'");
|
|
return;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Requesting price for {0} for start date:{1}", symbol, pricingDate));
|
|
price = MarketDataHelper.GetPriceRobinHood(symbol, pricingDate);
|
|
if (null == price)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "No price (GetPriceRobinHood) for '" + symbol + "'");
|
|
return;
|
|
}
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, price.ToString());
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Inserting....");
|
|
PricingDA.InsertPrice(price);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Done.");
|
|
}
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception {0}", exception.ToString()));
|
|
}
|
|
}
|
|
|
|
public static void UpdatePriceYahoo(String symbol, DateTime pricingDate)
|
|
{
|
|
try
|
|
{
|
|
Price price = PricingDA.GetPrice(symbol, pricingDate);
|
|
if (null != price)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Already have latest price for '" + symbol + "'");
|
|
return;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Requesting price for {0} for start date:{1} end date:{2}", symbol, pricingDate, pricingDate));
|
|
price = MarketDataHelper.GetDailyPrice(symbol, pricingDate);
|
|
if (null == price)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "No price (UpdatePriceYahoo) for '" + symbol + "'");
|
|
return;
|
|
}
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting price {0}...", price.ToString()));
|
|
PricingDA.InsertPrice(price);
|
|
}
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception {0}", exception.ToString()));
|
|
}
|
|
}
|
|
|
|
public static void UpdatePriceYahooSweep(String symbol, DateTime pricingDate)
|
|
{
|
|
try
|
|
{
|
|
CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol);
|
|
Price price = PricingDA.GetPrice(symbol, pricingDate);
|
|
if (null != price)
|
|
{
|
|
if (companyProfile == null || !companyProfile.PricingSourceEnum.Equals(CompanyProfile.EnumPricingSource.YAHOO))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Already have latest price for '" + symbol + "'");
|
|
return;
|
|
}
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("UpdatePriceYahooSweep: Requesting price for {0} for start date:{1} end date:{2}", symbol, pricingDate, pricingDate));
|
|
price = MarketDataHelper.GetDailyPrice(symbol, pricingDate);
|
|
if (null == price)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "No price (UpdatePriceYahooSweep) for '" + symbol + "'");
|
|
return;
|
|
}
|
|
else if (!price.IsValid)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Invalid price (UpdatePriceYahooSweep) for '" + symbol + "'");
|
|
return;
|
|
}
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting price {0}...", price.ToString()));
|
|
PricingDA.InsertPrice(price);
|
|
}
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception {0}", exception.ToString()));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// This feed is intended to supplement BigCharts and Yahoo feed.
|
|
/// It should run in the UPDATEDAILY2 chain after Yahoo during the nightly price capture.
|
|
/// It will only retrieve price for the current date AND it expects us to have a price for the previous business date.
|
|
/// In other words it will not fill pricing gaps, it will only add "tonights" price to an already established pricing history
|
|
/// and only if we are missing a price. See the method UpdatePricesBarChartSweep to understand the context.
|
|
/// </summary>
|
|
/// <param name="symbol"></param>
|
|
/// <param name="pricingDate"></param>
|
|
public static void UpdatePriceBarChartSweep(String symbol, DateTime pricingDate)
|
|
{
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("UpdatePriceBarChartSweep: Requesting latest price for {0} ", symbol));
|
|
Price price = MarketDataHelper.GetLatestPriceBarChart(symbol);
|
|
if (null == price)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "(UpdatePriceBarChartSweep) No price for '" + symbol + "'");
|
|
return;
|
|
}
|
|
else if (!price.IsValid)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "(UpdatePriceBarChartSweep) Invalid price for '" + symbol + "'");
|
|
return;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("(UpdatePriceBarChartSweep) Inserting price {0}...", price.ToString()));
|
|
PricingDA.InsertPrice(price);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("(UpdatePriceBarChartSweep) Exception {0}", exception.ToString()));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|