Sean/ARM64
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Sean
2025-03-25 21:42:32 -04:00
parent c266eecfeb
commit 30c33d3cfd
247 changed files with 60107 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
MarketData/MarketDataLib/Helper/AnalystPriceTargetMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,126 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class AnalystPriceTargetMarketDataHelper : MarketDataHelperBase<String>
{
private static int MaxThreads = 10; //(int)ThreadHelperEnum.MaxThreads;
private static int SLEEP_TIME_MS = 500;
public AnalystPriceTargetMarketDataHelper()
{
}
public bool LoadAnalystPriceTarget(String symbol=null)
{
Profiler profiler=new Profiler();
try
{
if(symbol==null)Queue=PricingDA.GetSymbols();
else (Queue=new List<String>()).Add(symbol);
ShuffleQueue(); // shuffle up the symbols
Index=-1;
ManualResetEvent[] resetEvents = new ManualResetEvent[MaxThreads];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)resetEvents[eventIndex] = new ManualResetEvent(true);
MDTrace.WriteLine(String.Format("Queuing analyst price target fetches ..."));
DateTime modified=DateTime.Now;
while (true)
{
ManualResetEvent[] availableEvents=GetAvailableEvents(resetEvents);
ManualResetEvent[] busyEvents=GetBusyEvents(resetEvents);
if (null == PeekQueueItem() && 0==busyEvents.Length)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Analyst price target queue contains {0} items, busy events {1}, all done.",0,busyEvents.Length));
break;
}
for (int index = 0; index < availableEvents.Length; index++)
{
symbol=GetQueueItem();
if (null != symbol)
{
availableEvents[index].Reset();
ThreadHelper analystPriceTargetThreadHelper = new ThreadHelper(symbol,modified,availableEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadAnalystPriceTarget, analystPriceTargetThreadHelper);
try { Thread.Sleep(SLEEP_TIME_MS); }catch (Exception) { ;}
}
else
{
busyEvents=GetBusyEvents(resetEvents);
if(busyEvents.Length!=availableEvents.Length)
{
ManualResetEvent[] resizedEvents=new ManualResetEvent[busyEvents.Length];
Array.Copy(busyEvents, resizedEvents,busyEvents.Length);
resetEvents = resizedEvents;
}
break;
}
} // for
MDTrace.WriteLine(LogLevel.DEBUG,"Analyst price target waiting for free slots...");
if(resetEvents.Length>0)WaitHandle.WaitAny(resetEvents);
if(null==PeekQueueItem())resetEvents=ResizeEvents(resetEvents);
} // while
MDTrace.WriteLine(LogLevel.DEBUG,"Analyst price target completed.");
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadAnalystPriceTarget]End, total took {0}(ms)",profiler.End()));
}
}
public void ThreadPoolCallbackLoadAnalystPriceTarget(Object analystPriceTargetThreadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)analystPriceTargetThreadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load analyst price target, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
LoadAnalystPriceTargetEx(threadHelper.Symbol,threadHelper.Modified);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load analyst price target, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
}
finally
{
threadHelper.ResetEvent.Set();
}
}
// Single threaded method, use for debugging a single symbol fetch without going through the queue
public static void LoadAnalystPriceTargetEx(String symbol,DateTime modified)
{
if (!AnalystPriceTargetDA.CheckAnalystPriceTargetModifiedOn(symbol, modified))
{
LoadAnalystPriceTargetEx(symbol);
}
else
{
MDTrace.WriteLine(LogLevel.DEBUG,"Already have analyst price target for '"+symbol+"' on "+Utility.DateTimeToStringMMHDDHYYYY(modified));
}
}
public static void LoadAnalystPriceTargetEx(String symbol)
{
try
{
MDTrace.WriteLine(LogLevel.DEBUG,"Loading analyst price target for for '" + symbol + "'");
AnalystPriceTarget analystPriceTarget = MarketDataHelper.GetAnalystPriceTarget(symbol);
if (null != analystPriceTarget)
{
AnalystPriceTargetDA.InsertAnalystPriceTarget(analystPriceTarget);
MDTrace.WriteLine(LogLevel.DEBUG,"");
MDTrace.WriteLine(LogLevel.DEBUG,AnalystPriceTarget.Header);
MDTrace.WriteLine(LogLevel.DEBUG,analystPriceTarget.ToString());
MDTrace.WriteLine(LogLevel.DEBUG,"");
}
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Failed to load analyst price target for '" + symbol + "'");
MDTrace.WriteLine(LogLevel.DEBUG,"Exception was " + exception.ToString());
}
}
}
}

144
MarketData/MarketDataLib/Helper/BollingerBandHelperSheet.cs Executable file
View File

@@ -0,0 +1,144 @@
// using System;
// using System.IO;
// using Microsoft.Office.Interop.Excel;
// using MarketData.DataAccess;
// using MarketData.MarketDataModel;
// using MarketData.Generator;
// using MarketData.Utils;
// namespace MarketData.Helper
// {
// public class BollingerBandSheetHelper
// {
// public BollingerBandSheetHelper()
// {
// }
// public static bool GenerateBollingerBandSheet(String strPathTemplateFile,String symbol,int movingAverageDays,int historicalDays)
// {
// Microsoft.Office.Interop.Excel.Application excelApp = null;
// Microsoft.Office.Interop.Excel.Workbook workbook = null;
// Microsoft.Office.Interop.Excel.Worksheet worksheet = null;
// Microsoft.Office.Interop.Excel.Sheets worksheets = null;
// int rowOffset = 1;
// try
// {
// String currentWorkingDirectory = Directory.GetCurrentDirectory();
// if (!File.Exists(strPathTemplateFile))
// {
// MDTrace.WriteLine(LogLevel.DEBUG,"Cannot locate "+strPathTemplateFile);
// return false;
// }
// DateGenerator dateGenerator = new DateGenerator();
// DateTime startDate = dateGenerator.GetPrevBusinessDay(DateTime.Now);
// Prices prices = PricingDA.GetPrices(symbol,startDate,historicalDays);
// if (null == prices || 0 == prices.Count)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,"No prices for "+symbol);
// return false;
// }
// Price price = prices[0];
// String pathOutputFile = currentWorkingDirectory + "\\" + symbol + "-BB-" + Utility.DateTimeToStringMMHDDHYYYY(price.Date);
// MDTrace.WriteLine(LogLevel.DEBUG,"Generating "+pathOutputFile);
// String companyName = PricingDA.GetNameForSymbol(symbol);
// File.Delete(pathOutputFile+".xlsx");
// excelApp = new Microsoft.Office.Interop.Excel.Application();
// excelApp.ScreenUpdating=false;
// workbook = excelApp.Workbooks.Open(strPathTemplateFile, 0, false, 5, "", "", false, Microsoft.Office.Interop.Excel.XlPlatform.xlWindows, "", true, false, 0, true, false, false);
// worksheets = workbook.Worksheets;
// worksheet = (Microsoft.Office.Interop.Excel.Worksheet)worksheets.get_Item("Sheet1");
// Microsoft.Office.Interop.Excel.ChartObjects chartObjects = worksheet.ChartObjects();
// Microsoft.Office.Interop.Excel.ChartObject chartObject = chartObjects.Item(1);
// chartObject.Chart.ChartTitle.Text = companyName + " (" + symbol + ") "+Utility.DateTimeToStringMMHDDHYYYY(prices[prices.Count-1].Date)+" Thru "+Utility.DateTimeToStringMMHDDHYYYY(price.Date);
// BollingerBands bollingerBands = BollingerBandGenerator.GenerateBollingerBands(prices, movingAverageDays);
// Axis vertAxis = (Axis)chartObject.Chart.Axes(XlAxisType.xlValue, XlAxisGroup.xlPrimary);
// vertAxis.MaximumScaleIsAuto = false;
// vertAxis.MaximumScale = GetMaxData(bollingerBands);
// vertAxis.MinimumScaleIsAuto = false;
// vertAxis.MinimumScale = GetMinData(bollingerBands);
// vertAxis.HasMajorGridlines = true;
// Axis horzAxis = (Axis)chartObject.Chart.Axes(XlAxisType.xlCategory, XlAxisGroup.xlPrimary);
// horzAxis.HasMajorGridlines = true;
// for (int index = 0; index < bollingerBands.Count; index++)
// {
// BollingerBandElement bollingerBandElement = bollingerBands[index];
// worksheet.Cells[index + 1 + rowOffset, 1] = bollingerBandElement.Date;
// worksheet.Cells[index + 1 + rowOffset, 2] = bollingerBandElement.Symbol;
// worksheet.Cells[index + 1 + rowOffset, 3] = bollingerBandElement.Open;
// worksheet.Cells[index + 1 + rowOffset, 4] = bollingerBandElement.High;
// worksheet.Cells[index + 1 + rowOffset, 5] = bollingerBandElement.Low;
// worksheet.Cells[index + 1 + rowOffset, 6] = bollingerBandElement.Close;
// worksheet.Cells[index + 1 + rowOffset, 7] = bollingerBandElement.Volume;
// worksheet.Cells[index + 1 + rowOffset, 8] = bollingerBandElement.SMAN;
// worksheet.Cells[index + 1 + rowOffset, 9] = bollingerBandElement.StDevN;
// worksheet.Cells[index + 1 + rowOffset, 10] = bollingerBandElement.K;
// worksheet.Cells[index + 1 + rowOffset, 11] = bollingerBandElement.L;
// worksheet.Cells[index + 1 + rowOffset, 12] = bollingerBandElement.KL1;
// worksheet.Cells[index + 1 + rowOffset, 13] = bollingerBandElement.LP1;
// }
// workbook.SaveAs(pathOutputFile, Microsoft.Office.Interop.Excel.XlFileFormat.xlOpenXMLWorkbook, Type.Missing, Type.Missing, false, false, Microsoft.Office.Interop.Excel.XlSaveAsAccessMode.xlNoChange, Microsoft.Office.Interop.Excel.XlSaveConflictResolution.xlUserResolution, true, Type.Missing, Type.Missing, Type.Missing);
// return true;
// }
// catch (Exception exception)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
// return false;
// }
// finally
// {
// if (null != worksheet) ReleaseObject(worksheet);
// if (null != worksheets) ReleaseObject(worksheets);
// if (null != workbook) ReleaseObject(workbook);
// excelApp.Quit();
// }
// }
// private static double GetMinData(BollingerBands bollingerBands)
// {
// double minData = double.MaxValue;
// for (int index = 0; index < bollingerBands.Count; index++)
// {
// BollingerBandElement bollingerBandElement = bollingerBands[index];
// if(bollingerBandElement.Open<minData)minData=bollingerBandElement.Open;
// if(bollingerBandElement.High<minData)minData=bollingerBandElement.High;
// if(bollingerBandElement.Low<minData)minData=bollingerBandElement.Low;
// if (bollingerBandElement.K < minData) minData = bollingerBandElement.K;
// if (bollingerBandElement.L < minData) minData = bollingerBandElement.L;
// if (bollingerBandElement.KL1 < minData) minData = bollingerBandElement.KL1;
// if (bollingerBandElement.LP1 < minData) minData = bollingerBandElement.LP1;
// }
// return minData-(minData*.05);
// }
// private static double GetMaxData(BollingerBands bollingerBands)
// {
// double maxData = 0;
// for (int index = 0; index < bollingerBands.Count; index++)
// {
// BollingerBandElement bollingerBandElement = bollingerBands[index];
// if (bollingerBandElement.Open > maxData) maxData = bollingerBandElement.Open;
// if (bollingerBandElement.High > maxData) maxData = bollingerBandElement.High;
// if (bollingerBandElement.Low > maxData) maxData = bollingerBandElement.Low;
// if (bollingerBandElement.K > maxData) maxData = bollingerBandElement.K;
// if (bollingerBandElement.L > maxData) maxData = bollingerBandElement.L;
// if (bollingerBandElement.KL1 > maxData) maxData = bollingerBandElement.KL1;
// if (bollingerBandElement.LP1 > maxData) maxData = bollingerBandElement.LP1;
// }
// return maxData+(maxData*.05);
// }
// private static void ReleaseObject(object obj)
// {
// try
// {
// System.Runtime.InteropServices.Marshal.FinalReleaseComObject(obj);
// obj = null;
// }
// catch (Exception ex)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,ex.ToString());
// obj = null;
// }
// finally
// {
// GC.Collect();
// }
// }
// }
// }

91
MarketData/MarketDataLib/Helper/CIKMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,91 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class CIKMarketDataHelper
{
private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads;
private List<String> symbols;
private int currentIndex = 0;
public CIKMarketDataHelper()
{
}
public bool UpdateAllMissingCIK()
{
Profiler profiler=new Profiler();
try
{
symbols = PricingDA.GetSymbolsForNullCIK();
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper cikThreadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdateMissingCIK, cikThreadHelper);
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load CIK, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateAllMissingCIK]End, total took {0}(ms)",profiler.End()));
}
}
private List<String> GetQueueSymbols()
{
List<String> queueSymbols = new List<String>();
int index = currentIndex;
for (; index < currentIndex + MaxThreads && index < symbols.Count; index++)
{
queueSymbols.Add(symbols[index]);
}
currentIndex = index;
return queueSymbols;
}
public void ThreadPoolCallbackUpdateMissingCIK(Object threadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load CIK, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
UpdateMissingCIKEx(threadHelper.Symbol);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load CIK, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
}
finally
{
threadHelper.ResetEvent.Set();
}
}
public static void UpdateMissingCIKEx(String symbol)
{
DateGenerator dateGenerator = new DateGenerator();
String cik = MarketDataHelper.GetCIK(symbol);
if (null == cik)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No CIK for symbol {0}",symbol));
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CIK for symbol {0}={1}",symbol,cik));
PricingDA.UpdateCIKForSymbol(symbol, cik);
}
}
}

131
MarketData/MarketDataLib/Helper/CompanyProfileMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,131 @@
using System;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class CompanyProfileThreadHelper : ThreadHelper
{
public CompanyProfile CompanyProfile{get;set;}
public CompanyProfileThreadHelper()
{
}
public CompanyProfileThreadHelper(CompanyProfile companyProfile, ManualResetEvent resetEvent)
{
CompanyProfile=companyProfile;
ResetEvent=resetEvent;
}
}
public class CompanyProfileMarketDataHelper
{
private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads;
private CompanyProfiles companyProfiles;
private int currentIndex = 0;
public CompanyProfileMarketDataHelper()
{
}
public bool UpdateCompanyProfile(String symbol)
{
Profiler profiler=new Profiler();
try
{
companyProfiles=new CompanyProfiles();
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
if(null==companyProfile)return false;
companyProfiles.Add(companyProfile);
currentIndex = 0;
while (true)
{
CompanyProfiles queueProfiles = GetQueueProfiles();
if (null == queueProfiles || 0 == queueProfiles.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueProfiles.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueProfiles.Count; index++)
{
CompanyProfileThreadHelper companyProfileThreadHelper = new CompanyProfileThreadHelper(queueProfiles[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallback, companyProfileThreadHelper);
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load company profile, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateCompanyProfiles]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdateCompanyProfiles()
{
Profiler profiler=new Profiler();
try
{
companyProfiles=CompanyProfileDA.GetCompanyProfiles();
currentIndex = 0;
while (true)
{
CompanyProfiles queueProfiles = GetQueueProfiles();
if (null == queueProfiles || 0 == queueProfiles.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueProfiles.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueProfiles.Count; index++)
{
CompanyProfileThreadHelper companyProfileThreadHelper = new CompanyProfileThreadHelper(queueProfiles[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallback, companyProfileThreadHelper);
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load company profile, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateCompanyProfiles]End, total took {0}(ms)",profiler.End()));
}
}
private CompanyProfiles GetQueueProfiles()
{
CompanyProfiles queueProfiles = new CompanyProfiles();
int index = currentIndex;
for (; index < currentIndex + MaxThreads && index < companyProfiles.Count; index++)
{
queueProfiles.Add(companyProfiles[index]);
}
currentIndex = index;
return queueProfiles;
}
public void ThreadPoolCallback(Object companyProfileThreadHelperContext)
{
CompanyProfileThreadHelper companyProfileThreadHelper = (CompanyProfileThreadHelper)companyProfileThreadHelperContext;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load profile, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, companyProfileThreadHelper.CompanyProfile.Symbol));
LoadCompanyProfileEx(companyProfileThreadHelper.CompanyProfile);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load profile, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, companyProfileThreadHelper.CompanyProfile.Symbol));
companyProfileThreadHelper.ResetEvent.Set();
}
private static void LoadCompanyProfileEx(CompanyProfile companyProfile)
{
CompanyProfile marketCompanyProfile = MarketDataHelper.GetCompanyProfile(companyProfile.Symbol);
if (null == marketCompanyProfile)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No company profile for symbol '" + companyProfile.Symbol + "'");
return;
}
if(!String.IsNullOrEmpty(marketCompanyProfile.Sector) && !marketCompanyProfile.Sector.Equals("N/A"))companyProfile.Sector=marketCompanyProfile.Sector;
if(!String.IsNullOrEmpty(marketCompanyProfile.Industry) && !marketCompanyProfile.Industry.Equals("N/A"))companyProfile.Industry=marketCompanyProfile.Industry;
if(!String.IsNullOrEmpty(marketCompanyProfile.Description) && !marketCompanyProfile.Description.Equals("N/A"))companyProfile.Description=marketCompanyProfile.Description;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("LoadCompanyProfileEx: Updating company profile.....Symbol:{0} Sector:{1} Industry:{2}",companyProfile.Symbol,companyProfile.Sector, companyProfile.Industry));
CompanyProfileDA.UpdateCompanyProfile(companyProfile);
}
}
}

170
MarketData/MarketDataLib/Helper/DividendHistoryMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,170 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class DividendHistoryThreadHelper : ThreadHelper
{
public DividendHistoryThreadHelper()
{
}
public DividendHistoryThreadHelper(String symbol, ManualResetEvent resetEvent)
{
Symbol=symbol;
ResetEvent=resetEvent;
}
public DividendHistoryThreadHelper(String symbol, DateTime modified,ManualResetEvent resetEvent)
{
Symbol=symbol;
Modified=modified;
ResetEvent=resetEvent;
}
}
public class DividendHistoryMarketDataHelper
{
private static int MaxThreads =10; //2;//(int)ThreadHelperEnum.MaxThreads;
private static int WAIT_TIME_MS=1000; // wait between request
private static Random generator=new Random();
private List<String> symbols;
private int currentIndex = 0;
private static int successHits=0;
public DividendHistoryMarketDataHelper()
{
}
public static int NextRandom(int min,int max)
{
return generator.Next(min,max);
}
public bool UpdateDividendHistory(String symbol)
{
Profiler profiler=new Profiler();
try
{
symbols = new List<String>();
symbols.Add(symbol);
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper threadHelper = new DividendHistoryThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadDividendHistory, threadHelper);
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load dividend history, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateDividendHistory]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdateDividendHistory()
{
Profiler profiler=new Profiler();
try
{
symbols = PricingDA.GetSymbols();
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper threadHelper = new DividendHistoryThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadDividendHistory, threadHelper);
try { Thread.Sleep(WAIT_TIME_MS); }catch(Exception) { ;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load dividend history, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateDividendHistory]End, total took {0}(ms)",profiler.End()));
}
}
private List<String> GetQueueSymbols()
{
List<String> queueSymbols = new List<String>();
int index = currentIndex;
for (; index < currentIndex + MaxThreads && index < symbols.Count; index++)
{
queueSymbols.Add(symbols[index]);
}
currentIndex = index;
return queueSymbols;
}
public void ThreadPoolCallbackLoadDividendHistory(Object threadHelperContext)
{
DividendHistoryThreadHelper threadHelper = (DividendHistoryThreadHelper)threadHelperContext;
try
{ // nasdaq website is very bot sensitive. this is why all the sleeps and delays here in this block
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load dividend history, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
if(UpdateDividendHistoryEx(threadHelper.Symbol))successHits++;
//if(successHits>100)
//{
// successHits=0;
// MDTrace.WriteLine(LogLevel.DEBUG,"Pausing for 60 seconds.");
// Thread.Sleep(60000);
//}
//else if(1==DividendHistoryMarketDataHelper.NextRandom(1,20))
//{
// MDTrace.WriteLine(LogLevel.DEBUG,"Pausing for 10 seconds.");
// Thread.Sleep(10000);
//}
//else if(1==DividendHistoryMarketDataHelper.NextRandom(1,100))
//{
// MDTrace.WriteLine(LogLevel.DEBUG,"Pausing for 30 seconds.");
// Thread.Sleep(30000);
//}
//else Thread.Sleep(50);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load dividend history, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
}
finally
{
threadHelper.ResetEvent.Set();
}
}
public static bool UpdateDividendHistoryEx(String symbol)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving dividend history for " + symbol);
DividendHistory dividendHistory = MarketDataHelper.GetDividendHistory(symbol);
if (null == dividendHistory || 0 == dividendHistory.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Unable to retrieve dividend history for " + symbol);
return false;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} dividend history records for {1}", dividendHistory.Count, symbol));
MDTrace.WriteLine(LogLevel.DEBUG,"Adding...");
if (DividendHistoryDA.InsertOrUpdate(dividendHistory)) MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Added {0} dividend history records for {1}", dividendHistory.Count, symbol));
else MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Failed to update dividend history for {0}", symbol));
return true;
}
}
}

133
MarketData/MarketDataLib/Helper/ETFHoldingsMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,133 @@
using System;
using System.Collections.Generic;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class ETFHoldingsMarketDataHelper
{
private static int MaxThreads = 10; // 5 (int)ThreadHelperEnum.MaxThreads;
private List<String> symbols;
private int currentIndex = 0;
private int WAIT_BETWEEN_REQUESTS=250; // 500
public ETFHoldingsMarketDataHelper()
{
}
public bool LoadETFHoldings(String symbol)
{
Profiler profiler=new Profiler();
try
{
symbols = new List<String>();
symbols.Add(symbol);
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadETFHoldings, threadHelper);
try{Thread.Sleep(WAIT_BETWEEN_REQUESTS);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load ETF Holdings, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadETFHoldings]End, total took {0}(ms)",profiler.End()));
}
}
public bool LoadETFHoldings()
{
Profiler profiler=new Profiler();
try
{
symbols = PricingDA.GetSymbolsETF();
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadETFHoldings, threadHelper);
try{Thread.Sleep(WAIT_BETWEEN_REQUESTS);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load ETF Holdings, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadETFHoldings]End, total took {0}(ms)",profiler.End()));
}
}
private List<String> GetQueueSymbols()
{
List<String> queueSymbols = new List<String>();
int index = currentIndex;
for (; index < currentIndex + MaxThreads && index < symbols.Count; index++)
{
queueSymbols.Add(symbols[index]);
}
currentIndex = index;
return queueSymbols;
}
public void ThreadPoolCallbackLoadETFHoldings(Object threadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load ETF Holdings, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
LoadETFHoldingsEx(threadHelper.Symbol);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load ETF Holdings, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
}
finally
{
threadHelper.ResetEvent.Set();
}
}
public static void LoadETFHoldingsEx(String symbol)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving holdings for " + symbol);
ETFHoldings etfHoldings = MarketDataHelper.GetETFHoldings(symbol);
if (null == etfHoldings || 0 == etfHoldings.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Error retreiving ETF holdings for '"+symbol+"'");
return;
}
if (!ETFHoldingsDA.InsertOrUpdate(etfHoldings))
{
MDTrace.WriteLine(LogLevel.DEBUG,"Error adding ETF holdings for " + symbol);
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,"********* E T F H O L D I N G S ********");
foreach (ETFHolding etfHolding in etfHoldings)
{
MDTrace.WriteLine(LogLevel.DEBUG,etfHolding.ETFSymbol + ","+etfHolding.HoldingSymbol + "," + etfHolding.HoldingSymbolShareClass + "," + etfHolding.HoldingCompanyName);
}
MDTrace.WriteLine(LogLevel.DEBUG,"************************");
}
}
}

103
MarketData/MarketDataLib/Helper/EarningsAnnouncementsMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,103 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class EarningsAnnouncementsMarketDataHelper : MarketDataHelperBase<String>
{
private static int MaxThreads = 7; // (int)ThreadHelperEnum.MaxThreads;
public EarningsAnnouncementsMarketDataHelper()
{
}
public bool UpdateEarningsAnnouncements(String symbol=null)
{
Profiler profiler=new Profiler();
try
{
if(symbol==null)Queue=PricingDA.GetSymbols();
else (Queue=new List<String>()).Add(symbol);
Index=-1;
ManualResetEvent[] resetEvents = new ManualResetEvent[MaxThreads];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)resetEvents[eventIndex] = new ManualResetEvent(true);
MDTrace.WriteLine(String.Format("Queuing earnings announcement fetches ..."));
DateTime modified=DateTime.Now;
while (true)
{
ManualResetEvent[] availableEvents=GetAvailableEvents(resetEvents);
ManualResetEvent[] busyEvents=GetBusyEvents(resetEvents);
if (null == PeekQueueItem() && 0==busyEvents.Length)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Earnings announcement queue contains {0} items, busy events {1}, all done.",0,busyEvents.Length));
break;
}
for (int index = 0; index < availableEvents.Length; index++)
{
symbol=GetQueueItem();
if (null != symbol)
{
availableEvents[index].Reset();
ThreadHelper earningsAnnouncementsThreadHelper = new ThreadHelper(symbol,modified,availableEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackEarningsAnnouncements, earningsAnnouncementsThreadHelper);
}
else
{
busyEvents=GetBusyEvents(resetEvents);
if(busyEvents.Length!=availableEvents.Length)
{
ManualResetEvent[] resizedEvents=new ManualResetEvent[busyEvents.Length];
Array.Copy(busyEvents, resizedEvents,busyEvents.Length);
resetEvents = resizedEvents;
}
break;
}
} // for
MDTrace.WriteLine(LogLevel.DEBUG,"Earnings announcements waiting for free slots...");
if(resetEvents.Length>0)WaitHandle.WaitAny(resetEvents);
if(null==PeekQueueItem())resetEvents=ResizeEvents(resetEvents);
} // while
MDTrace.WriteLine(LogLevel.DEBUG,"Earnings announcements completed.");
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateEarningsAnnouncements]End, total took {0}(ms)",profiler.End()));
}
}
public void ThreadPoolCallbackEarningsAnnouncements(Object earningsAnnouncementsThreadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)earningsAnnouncementsThreadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load earnings announcements, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
LoadEarningsAnnouncementsEx(threadHelper.Symbol,threadHelper.Modified);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load earnings announcements, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
}
finally
{
threadHelper.ResetEvent.Set();
}
}
// Single threaded method, use for debugging a single symbol fetch without going through the queue
public static void LoadEarningsAnnouncementsEx(String symbol,DateTime modified)
{
EarningsAnnouncements earningsAnnouncements=MarketDataHelper.GetEarningsAnnouncements(symbol);
if(null==earningsAnnouncements||0==earningsAnnouncements.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Error loading earnings announcements for '"+symbol+"'");
return;
}
MDTrace.WriteLine(String.Format("[UpdateEarningsAnnouncements] Got {0} for {1}",earningsAnnouncements.Count,symbol));
if(!EarningsAnnouncementsDA.InsertEarningsAnnouncements(earningsAnnouncements))MDTrace.WriteLine(String.Format("[UpdateEarningsAnnouncements] Insert failed for {0}",symbol));
else MDTrace.WriteLine(String.Format("[UpdateEarningsAnnouncements] Insert succeeded for {0}",symbol));
}
}
}

249
MarketData/MarketDataLib/Helper/FinancialStatementsMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,249 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
using System.IO;
namespace MarketData.Helper
{
public class FinancialStatementsMarketDataHelper : MarketDataHelperBase<String>
{
private static int MaxThreads = 10; // (int)ThreadHelperEnum.MaxThreads;
private static int PAUSE_BETWEEN_STATEMENTS=125;
public FinancialStatementsMarketDataHelper()
{
}
public bool LoadFinancialStatements(String symbol=null)
{
Profiler profiler=new Profiler();
try
{
if(symbol==null)Queue=PricingDA.GetSymbols();
else (Queue=new List<String>()).Add(symbol);
Index=-1;
ManualResetEvent[] resetEvents = new ManualResetEvent[MaxThreads];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)resetEvents[eventIndex] = new ManualResetEvent(true);
MDTrace.WriteLine(String.Format("Queuing FinancialStatements ..."));
DateTime modified=DateTime.Now;
while (true)
{
ManualResetEvent[] availableEvents=GetAvailableEvents(resetEvents);
ManualResetEvent[] busyEvents=GetBusyEvents(resetEvents);
if (null == PeekQueueItem() && 0==busyEvents.Length)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("FinancialStatements queue contains {0} items, busy events {1}, all done.",0,busyEvents.Length));
break;
}
for (int index = 0; index < availableEvents.Length; index++)
{
symbol=GetQueueItem();
if (null != symbol)
{
availableEvents[index].Reset();
ThreadHelper threadHelper = new ThreadHelper(symbol,modified,availableEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadFinancialStatements, threadHelper);
try{Thread.Sleep(250);}catch(Exception){;} // adding a short pause between requests
}
else
{
busyEvents=GetBusyEvents(resetEvents);
if(busyEvents.Length!=availableEvents.Length)
{
ManualResetEvent[] resizedEvents=new ManualResetEvent[busyEvents.Length];
Array.Copy(busyEvents, resizedEvents,busyEvents.Length);
resetEvents = resizedEvents;
}
break;
}
} // for
MDTrace.WriteLine(LogLevel.DEBUG,"FinancialStatements waiting for free slots...");
if(resetEvents.Length>0)WaitHandle.WaitAny(resetEvents);
if(null==PeekQueueItem())resetEvents=ResizeEvents(resetEvents);
} // while
MDTrace.WriteLine(LogLevel.DEBUG,"FinancialStatements completed.");
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadFinancialStatements]End, total took {0}(ms)",profiler.End()));
}
}
public void ThreadPoolCallbackLoadFinancialStatements(Object threadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Financial Statements, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
LoadFinancialStatementsEx(threadHelper.Symbol,threadHelper.Modified);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Financial Statements, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
}
finally
{
threadHelper.ResetEvent.Set();
}
}
// Single threaded method, use for debugging a single symbol fetch without going through the queue
public static void LoadFinancialStatementsEx(String symbol,DateTime modified)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving income statements for '" + symbol + "'");
// download available annual income statement data for whatever years are available
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
if (!IncomeStatementDA.CheckIncomeStatementModifiedOn(symbol,modified,IncomeStatement.PeriodType.Annual))
{
List<IncomeStatement> incomeStatements = MarketDataHelper.GetIncomeStatement(symbol,IncomeStatement.PeriodType.Annual);
if (null == incomeStatements || 0 == incomeStatements.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No Annual income statement for " + symbol);
}
else
{
foreach(IncomeStatement incomeStatement in incomeStatements)
{
IncomeStatement previousIncomeStatement=IncomeStatementDA.GetIncomeStatement(incomeStatement.Symbol,incomeStatement.AsOf,incomeStatement.Period);
incomeStatement.MergeFrom(previousIncomeStatement);
}
IncomeStatementDA.InsertIncomeStatements(incomeStatements);
MDTrace.WriteLine(LogLevel.DEBUG,IncomeStatement.Heading);
for (int pindex = 0; pindex < incomeStatements.Count; pindex++)
{
MDTrace.WriteLine(LogLevel.DEBUG,incomeStatements[pindex].ToString());
}
}
}
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded annual income statement for '"+symbol+"' on "+Utility.DateTimeToStringMMHDDHYYYY(modified));
// download quarterly income statement data for whatever quarters are available
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
if (!IncomeStatementDA.CheckIncomeStatementModifiedOn(symbol, modified, IncomeStatement.PeriodType.Quarterly))
{
List<IncomeStatement> incomeStatements = MarketDataHelper.GetIncomeStatement(symbol, IncomeStatement.PeriodType.Quarterly);
if (null == incomeStatements || 0 == incomeStatements.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No Quarterly income statement for " + symbol);
}
else
{
foreach(IncomeStatement incomeStatement in incomeStatements)
{
IncomeStatement previousIncomeStatement=IncomeStatementDA.GetIncomeStatement(incomeStatement.Symbol,incomeStatement.AsOf,incomeStatement.Period);
incomeStatement.MergeFrom(previousIncomeStatement);
}
IncomeStatementDA.InsertIncomeStatements(incomeStatements);
MDTrace.WriteLine(LogLevel.DEBUG,IncomeStatement.Heading);
for (int pindex = 0; pindex < incomeStatements.Count; pindex++)
{
MDTrace.WriteLine(LogLevel.DEBUG,incomeStatements[pindex].ToString());
}
}
}
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded quarterly income statement for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving balancesheet for '" + symbol + "'");
// download annual balance sheet data for whatever years are available
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
if (!BalanceSheetDA.CheckBalanceSheetModifiedOn(symbol, modified,BalanceSheet.PeriodType.Annual))
{
List<BalanceSheet> balanceSheets = MarketDataHelper.GetBalanceSheet(symbol,BalanceSheet.PeriodType.Annual);
if (null == balanceSheets || 0 == balanceSheets.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No Annual balance sheet for " + symbol);
}
else
{
foreach(BalanceSheet balanceSheet in balanceSheets)
{
BalanceSheet previousBalanceSheet=BalanceSheetDA.GetBalanceSheet(balanceSheet.Symbol,balanceSheet.AsOf,balanceSheet.Period);
balanceSheet.MergeFrom(previousBalanceSheet);
}
BalanceSheetDA.InsertBalanceSheets(balanceSheets);
MDTrace.WriteLine(LogLevel.DEBUG,BalanceSheet.Heading);
for (int pindex = 0; pindex < balanceSheets.Count; pindex++)
{
MDTrace.WriteLine(LogLevel.DEBUG,balanceSheets[pindex].ToString());
}
}
}
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded annual balance sheet for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
// download quarterly balance sheet data for whatever quarters are available
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
if (!BalanceSheetDA.CheckBalanceSheetModifiedOn(symbol, modified, BalanceSheet.PeriodType.Quarterly))
{
List<BalanceSheet> balanceSheets = MarketDataHelper.GetBalanceSheet(symbol,BalanceSheet.PeriodType.Quarterly);
if (null == balanceSheets || 0 == balanceSheets.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No Quarterly balance sheet for " + symbol);
}
else
{
foreach(BalanceSheet balanceSheet in balanceSheets)
{
BalanceSheet previousBalanceSheet=BalanceSheetDA.GetBalanceSheet(balanceSheet.Symbol,balanceSheet.AsOf,balanceSheet.Period);
balanceSheet.MergeFrom(previousBalanceSheet);
}
BalanceSheetDA.InsertBalanceSheets(balanceSheets);
MDTrace.WriteLine(LogLevel.DEBUG,BalanceSheet.Heading);
for (int pindex = 0; pindex < balanceSheets.Count; pindex++)
{
MDTrace.WriteLine(LogLevel.DEBUG,balanceSheets[pindex].ToString());
}
}
}
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded quarterly balance sheet for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving cashflow statements for '" + symbol + "'");
// download annual cashflow data for whatever years are available
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
if (!CashflowStatementDA.CheckCashflowStatementModifiedOn(symbol, modified,CashflowStatement.PeriodType.Annual))
{
List<CashflowStatement> cashflowStatements = MarketDataHelper.GetCashflowStatement(symbol,CashflowStatement.PeriodType.Annual);
if (null == cashflowStatements || 0 == cashflowStatements.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No Annual cashflow statement for " + symbol);
}
else
{
foreach(CashflowStatement cashflowStatement in cashflowStatements)
{
CashflowStatement previousCashflowStatement=CashflowStatementDA.GetCashflowStatement(cashflowStatement.Symbol,cashflowStatement.AsOf,cashflowStatement.Period);
cashflowStatement.MergeFrom(previousCashflowStatement);
}
CashflowStatementDA.InsertCashflowStatement(cashflowStatements);
MDTrace.WriteLine(LogLevel.DEBUG,CashflowStatement.Heading);
for (int pindex = 0; pindex < cashflowStatements.Count; pindex++)
{
MDTrace.WriteLine(LogLevel.DEBUG,cashflowStatements[pindex].ToString());
}
}
}
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded annual cashflow statement for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
// download quarterly cashflow statement for whatever quarters are available
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
if (!CashflowStatementDA.CheckCashflowStatementModifiedOn(symbol, modified, CashflowStatement.PeriodType.Quarterly))
{
List<CashflowStatement> cashflowStatements = MarketDataHelper.GetCashflowStatement(symbol,CashflowStatement.PeriodType.Quarterly);
if (null == cashflowStatements || 0 == cashflowStatements.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No Quarterly cashflow statement for " + symbol);
}
else
{
foreach(CashflowStatement cashflowStatement in cashflowStatements)
{
CashflowStatement previousCashflowStatement=CashflowStatementDA.GetCashflowStatement(cashflowStatement.Symbol,cashflowStatement.AsOf,cashflowStatement.Period);
cashflowStatement.MergeFrom(previousCashflowStatement);
}
CashflowStatementDA.InsertCashflowStatement(cashflowStatements);
MDTrace.WriteLine(LogLevel.DEBUG,CashflowStatement.Heading);
for (int pindex = 0; pindex < cashflowStatements.Count; pindex++)
{
MDTrace.WriteLine(LogLevel.DEBUG,cashflowStatements[pindex].ToString());
}
}
}
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded quarterly cashflow statement for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
}
}
}

260
MarketData/MarketDataLib/Helper/FundamentalMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,260 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class FundamentalMarketDataHelper : MarketDataHelperBase<String>
{
private static int MaxThreads = 5; //(int)ThreadHelperEnum.MaxThreads;
private UpdateManager updateManager=new UpdateManager();
public FundamentalMarketDataHelper()
{
}
public bool LoadFundamentals(String symbol=null)
{
Profiler profiler=new Profiler();
try
{
if(symbol==null)Queue=PricingDA.GetSymbols();
else (Queue=new List<String>()).Add(symbol);
updateManager.Prepare("fundamentals_session.txt");
Queue=Queue.Except(new List<String>(updateManager.Entries)).ToList();
Index=-1;
ManualResetEvent[] resetEvents = new ManualResetEvent[MaxThreads];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)resetEvents[eventIndex] = new ManualResetEvent(true);
MDTrace.WriteLine(String.Format("Queuing fundamentals ..."));
DateTime modified=DateTime.Now;
while (true)
{
ManualResetEvent[] availableEvents=GetAvailableEvents(resetEvents);
ManualResetEvent[] busyEvents=GetBusyEvents(resetEvents);
if (null == PeekQueueItem() && 0==busyEvents.Length)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Fundamentals queue contains {0} items, busy events {1}, all done.",0,busyEvents.Length));
break;
}
for (int index = 0; index < availableEvents.Length; index++)
{
symbol=GetQueueItem();
if (null != symbol)
{
availableEvents[index].Reset();
ThreadHelper fundamentalThreadHelper = new ThreadHelper(symbol,modified,availableEvents[index]);
fundamentalThreadHelper.UpdateManager=updateManager;
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadFundamental, fundamentalThreadHelper);
// try{Thread.Sleep(2000);}catch(Exception){;} // adding a short pause between requests
try{Thread.Sleep(1000);}catch(Exception){;} // changing this because I am going to run a test against TOR
}
else
{
busyEvents=GetBusyEvents(resetEvents);
if(busyEvents.Length!=availableEvents.Length)
{
ManualResetEvent[] resizedEvents=new ManualResetEvent[busyEvents.Length];
Array.Copy(busyEvents, resizedEvents,busyEvents.Length);
resetEvents = resizedEvents;
}
break;
}
} // for
MDTrace.WriteLine(LogLevel.DEBUG,"Fundamentals waiting for free slots...");
if(resetEvents.Length>0)WaitHandle.WaitAny(resetEvents);
if(null==PeekQueueItem())resetEvents=ResizeEvents(resetEvents);
} // while
MDTrace.WriteLine(LogLevel.DEBUG,"Fundamentals completed.");
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadFundamentals]End, total took {0}(ms)",profiler.End()));
updateManager.Dispose();
}
}
public bool LoadFundamentalsFinViz(String symbol=null)
{
Profiler profiler=new Profiler();
try
{
if(symbol==null)Queue=PricingDA.GetSymbols();
else (Queue=new List<String>()).Add(symbol);
updateManager.Prepare("fundamentals_session.txt");
Queue=Queue.Except(new List<String>(updateManager.Entries)).ToList();
Index=-1;
ManualResetEvent[] resetEvents = new ManualResetEvent[MaxThreads];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)resetEvents[eventIndex] = new ManualResetEvent(true);
MDTrace.WriteLine(String.Format("Queuing fundamentals ..."));
DateTime modified=DateTime.Now;
while (true)
{
ManualResetEvent[] availableEvents=GetAvailableEvents(resetEvents);
ManualResetEvent[] busyEvents=GetBusyEvents(resetEvents);
if (null == PeekQueueItem() && 0==busyEvents.Length)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Fundamentals queue contains {0} items, busy events {1}, all done.",0,busyEvents.Length));
break;
}
for (int index = 0; index < availableEvents.Length; index++)
{
symbol=GetQueueItem();
if (null != symbol)
{
availableEvents[index].Reset();
ThreadHelper fundamentalThreadHelper = new ThreadHelper(symbol,modified,availableEvents[index]);
fundamentalThreadHelper.UpdateManager=updateManager;
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadFundamentalFinViz, fundamentalThreadHelper);
try{Thread.Sleep(250);}catch(Exception){;} // adding a short pause between requests
}
else
{
busyEvents=GetBusyEvents(resetEvents);
if(busyEvents.Length!=availableEvents.Length)
{
ManualResetEvent[] resizedEvents=new ManualResetEvent[busyEvents.Length];
Array.Copy(busyEvents, resizedEvents,busyEvents.Length);
resetEvents = resizedEvents;
}
break;
}
} // for
MDTrace.WriteLine(LogLevel.DEBUG,"Fundamentals waiting for free slots...");
if(resetEvents.Length>0)WaitHandle.WaitAny(resetEvents);
if(null==PeekQueueItem())resetEvents=ResizeEvents(resetEvents);
} // while
MDTrace.WriteLine(LogLevel.DEBUG,"Fundamentals completed.");
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadFundamentals]End, total took {0}(ms)",profiler.End()));
updateManager.Dispose();
}
}
// ******************************************************************************************************************************************************************
// ********************************************************************** T H R E A D C A L L B A C K **************************************************************
// ******************************************************************************************************************************************************************
public void ThreadPoolCallbackLoadFundamental(Object fundamentalThreadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)fundamentalThreadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load fundamental, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
LoadFundamentalEx(threadHelper.Symbol,threadHelper.Modified,threadHelper.updateManager);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load fundamental, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
}
finally
{
threadHelper.ResetEvent.Set();
}
}
public void ThreadPoolCallbackLoadFundamentalFinViz(Object fundamentalThreadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)fundamentalThreadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load fundamental, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
LoadFundamentalEx(threadHelper.Symbol,threadHelper.Modified,threadHelper.updateManager,true);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load fundamental, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
}
finally
{
threadHelper.ResetEvent.Set();
}
}
// *********************************************************************************************************************************************************************
// *********************************************************************************************************************************************************************
// *********************************************************************************************************************************************************************
// Single threaded method, use for debugging a single symbol fetch without going through the queue
public static void LoadFundamentalEx(String symbol,DateTime modified,UpdateManager updateManager,bool useFinViz=false)
{
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
if(null!=companyProfile&&!companyProfile.IsEquity)
{
updateManager.Add(symbol);
return;
}
if (!FundamentalDA.CheckFundamentalModifiedOn(symbol, modified))
{
bool result=false;
result=useFinViz?LoadFundamentalExFinViz(symbol):LoadFundamentalEx(symbol);
if(result)updateManager.Add(symbol);
}
else
{
Fundamental fundamental=FundamentalDA.GetFundamental(symbol,modified);
// Even if we have the record Yahoo may not have given us some values. (i.e) Sometime we get zero back from yahoo in EnterpriseValue. Subsequent runs should re-fetch from Yahoo in this case
// If the existing record contains zero in EnterpriseValue then re-fetch it
if(null==fundamental||0.00==fundamental.EnterpriseValue)
{
bool result=false;
result=useFinViz?LoadFundamentalExFinViz(symbol):LoadFundamentalEx(symbol);
if(result)updateManager.Add(symbol);
}
else
{
MDTrace.WriteLine(LogLevel.DEBUG,"Already have fundamental for '"+symbol+"' on "+Utility.DateTimeToStringMMHDDHYYYY(modified));
updateManager.Add(symbol);
}
}
}
public static bool LoadFundamentalEx(String symbol)
{
try
{
MDTrace.WriteLine(LogLevel.DEBUG,"Loading fundamental data for '"+symbol+"'");
Fundamental fundamental = MarketDataHelper.GetFundamental(symbol);
if (null == fundamental) {MDTrace.WriteLine(LogLevel.DEBUG,"Error loading fundmental for '"+symbol+"'");return false;}
else
{
Fundamental priorFundamental=FundamentalDA.GetFundamental(symbol);
fundamental.MergeFrom(priorFundamental); // if any fields are missing on this run then carry forward fields from previous run
FundamentalDA.InsertFundamental(fundamental);
MDTrace.WriteLine(LogLevel.DEBUG,Fundamental.Header);
MDTrace.WriteLine(LogLevel.DEBUG,fundamental.ToString());
return true;
}
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Failed to load funamental for '"+symbol+"'");
MDTrace.WriteLine(LogLevel.DEBUG,"Exception was "+exception.ToString());
return false;
}
}
public static bool LoadFundamentalExFinViz(String symbol)
{
try
{
MDTrace.WriteLine(LogLevel.DEBUG,"Loading fundamental data for '"+symbol+"'");
Fundamental fundamental = MarketDataHelper.GetFundamentalFinViz(symbol);
if (null == fundamental) {MDTrace.WriteLine(LogLevel.DEBUG,"Error loading fundmental for '"+symbol+"'");return false;}
DateTime? nextEarningsDate=EarningsAnnouncementsDA.GetNextEarningsDate(symbol);
if(null!=nextEarningsDate)fundamental.NextEarningsDate=nextEarningsDate.Value;
Fundamental priorFundamental=FundamentalDA.GetFundamental(symbol);
fundamental.MergeFrom(priorFundamental); // if any fields are missing on this run then carry forward fields from previous run
FundamentalDA.InsertFundamental(fundamental);
MDTrace.WriteLine(LogLevel.DEBUG,Fundamental.Header);
MDTrace.WriteLine(LogLevel.DEBUG,fundamental.ToString());
return true;
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Failed to load funamental for '"+symbol+"'");
MDTrace.WriteLine(LogLevel.DEBUG,"Exception was "+exception.ToString());
return false;
}
}
}
}

191
MarketData/MarketDataLib/Helper/HeadlinesMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,191 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class HeadlinesMarketDataHelper
{
private static int MaxThreads = 5; // (int)ThreadHelperEnum.MaxThreads;
private static int WAIT_BETWEEN_REQUESTS_MS = 2000; // wait ms between requests
private List<String> symbols;
private int currentIndex = 0;
public HeadlinesMarketDataHelper()
{
}
public bool LoadHeadlines()
{
Profiler profiler=new Profiler();
try
{
symbols=PricingDA.GetSymbols();
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadHeadline, threadHelper);
try{Thread.Sleep(WAIT_BETWEEN_REQUESTS_MS);}catch{;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load Headline, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadHeadlines]End, total took {0}(ms)",profiler.End()));
}
}
public bool LoadHeadlines(List<String> requestSymbols)
{
Profiler profiler=new Profiler();
try
{
symbols=requestSymbols;
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadHeadline, threadHelper);
try{Thread.Sleep(WAIT_BETWEEN_REQUESTS_MS);}catch{;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load Headline, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadHeadlines]End, total took {0}(ms)",profiler.End()));
}
}
private List<String> GetQueueSymbols()
{
List<String> queueSymbols = new List<String>();
int index = currentIndex;
for (; index < currentIndex + MaxThreads && index < symbols.Count; index++)
{
queueSymbols.Add(symbols[index]);
}
currentIndex = index;
return queueSymbols;
}
public void ThreadPoolCallbackLoadHeadline(Object threadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Headline, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
LoadHeadlineEx(threadHelper.Symbol);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Headline, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
}
finally
{
threadHelper.ResetEvent.Set();
}
}
// This method is used by unit test
public static Headlines GetHeadlinesEx(String symbol)
{
Headlines headlinesCollection=new Headlines();
Headlines headlines=null;
// SEEKING ALPHA
headlines=MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol);
if(null!=headlines)headlinesCollection.AddRange(headlines);
// NASDAQ
headlines=MarketDataHelper.GetCompanyHeadlinesNASDAQ(symbol);
if(null!=headlines)headlinesCollection.AddRange(headlines);
// MARKETWATCH
headlines=MarketDataHelper.GetCompanyHeadlinesMarketWatch(symbol);
if(null!=headlines)headlinesCollection.AddRange(headlines);
return headlinesCollection;
}
// ***********************************************************************************************************************************************
// ***********************************************************************************************************************************************
// ***********************************************************************************************************************************************
public static void LoadHeadlineEx(String symbol)
{
DateTime marketDate=PremarketDA.GetLatestMarketDate();
marketDate=new DateTime(marketDate.Year,marketDate.Month,marketDate.Day,23,59,59);
Headlines headlines=null;
// NASDAQ
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCompanyHeadlinesNASDAQ {0}",symbol));
headlines=MarketDataHelper.GetCompanyHeadlinesNASDAQ(symbol);
if(headlines.IsNullOrEmpty())
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for {0} from NASDAQ",symbol));
}
else
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} headlines for {1} from NASDAQ",headlines.Count,symbol));
headlines=new Headlines(headlines.Where(x=>x.Date<marketDate).ToList());
HeadlinesDA.InsertHeadlines(headlines);
foreach(Headline headline in headlines)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("NASDAQ: {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry));
}
}
// MARKETWATCH
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCompanyHeadlinesMarketWatch {0}",symbol));
headlines=MarketDataHelper.GetCompanyHeadlinesMarketWatch(symbol);
if(headlines.IsNullOrEmpty())
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for {0} from MARKETWATCH",symbol));
}
else
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} headlines for {1} from MARKETWATCH",headlines.Count,symbol));
headlines=new Headlines(headlines.Where(x=>x.Date<marketDate).ToList());
HeadlinesDA.InsertHeadlines(headlines);
foreach(Headline headline in headlines)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("MARKETWATCH: {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry));
}
}
// SEEKING ALPHA
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCompanyHeadlinesSeekingAlpha {0}",symbol));
headlines=MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol);
if(headlines.IsNullOrEmpty())
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for {0} from Seeking Alpha",symbol));
}
else
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} headlines for {1} from Seeking Alpha",headlines.Count,symbol));
headlines=new Headlines(headlines.Where(x=>x.Date<marketDate).ToList());
HeadlinesDA.InsertHeadlines(headlines);
foreach(Headline headline in headlines)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Seeking Alpha {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry));
}
}
return;
}
}
}

131
MarketData/MarketDataLib/Helper/HistoricalMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,131 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class HistoricalMarketDataHelper
{
private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads;
private List<String> symbols;
private int currentIndex = 0;
public HistoricalMarketDataHelper()
{
}
public bool LoadHistorical(String symbol)
{
Profiler profiler=new Profiler();
try
{
symbols = new List<String>();
symbols.Add(symbol);
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadHistorical, threadHelper);
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load Historical, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadHistorical]End, total took {0}(ms)",profiler.End()));
}
}
public bool LoadHistorical()
{
Profiler profiler=new Profiler();
try
{
symbols = PricingDA.GetSymbols();
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadHistorical, threadHelper);
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load Historical, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadHistorical]End, total took {0}(ms)",profiler.End()));
}
}
private List<String> GetQueueSymbols()
{
List<String> queueSymbols = new List<String>();
int index = currentIndex;
for (; index < currentIndex + MaxThreads && index < symbols.Count; index++)
{
queueSymbols.Add(symbols[index]);
}
currentIndex = index;
return queueSymbols;
}
public void ThreadPoolCallbackLoadHistorical(Object threadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Historical, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
LoadHistoricalEx(threadHelper.Symbol);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Historical, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
}
finally
{
threadHelper.ResetEvent.Set();
}
}
public static void LoadHistoricalEx(String symbol)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Loading time series data values for '" + symbol + "'");
Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> timeSeriesCollection = MarketDataHelper.GetHistoricalValues(symbol);
if (null == timeSeriesCollection || 0 == timeSeriesCollection.Count)
{
return;
}
List<TimeSeriesElement.ElementType> keys=new List<TimeSeriesElement.ElementType>(timeSeriesCollection.Keys);
for (int cIndex = 0; cIndex < keys.Count; cIndex++)
{
TimeSeriesCollection timeSeriesElements = timeSeriesCollection[keys[cIndex]];
if (null == timeSeriesElements || 0 == timeSeriesElements.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Unable to retrieve time series values for '" + symbol + "'");
continue;
}
HistoricalDA.InsertTimeSeries(timeSeriesElements);
}
}
}
}

197
MarketData/MarketDataLib/Helper/InsiderTransactionMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,197 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class InsiderTransactionThreadHelper : ThreadHelper
{
public InsiderTransactionThreadHelper(String symbol, ManualResetEvent resetEvent)
: base(symbol, resetEvent)
{
}
public int YearGreaterEqual { get; set; }
}
public class InsiderTransactionMarketDataHelper
{
private static int MaxThreads = 10; // 10 threads avoids receiving HTTP Response 429 (Too many requests)
private static int WAIT_TIME_MS=500; // wait between request
private List<String> symbols;
private int currentIndex = 0;
private UpdateManager UpdateManager = new UpdateManager();
public InsiderTransactionMarketDataHelper()
{
}
public bool LoadInsiderTransactions(String symbol)
{
Profiler profiler=new Profiler();
try
{
symbols = new List<String>();
symbols.Add(symbol);
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index], resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallback, threadHelper);
}
MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadInsiderTransactions]End, total took {0}(ms)",profiler.End()));
}
}
public bool LoadInsiderTransactions(List<String> symbols)
{
Profiler profiler=new Profiler();
try
{
this.symbols = symbols;
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index], resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallback, threadHelper);
try { Thread.Sleep(WAIT_TIME_MS); } catch(Exception) { ;} // wait
}
MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadInsiderTransactions]End, total took {0}(ms)",profiler.End()));
}
}
public bool LoadInsiderTransactionsYearGreaterEqual(List<String> symbols,int yearGreaterEqual)
{
Profiler profiler=new Profiler();
try
{
this.symbols = symbols;
currentIndex = 0;
UpdateManager.Prepare("load_insider_transactions_year.txt", 7); // use max age 7 days
this.symbols=this.symbols.Except(new List<String>(UpdateManager.Entries)).ToList();
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
InsiderTransactionThreadHelper threadHelper = new InsiderTransactionThreadHelper(queueSymbols[index], resetEvents[index]);
threadHelper.UpdateManager=UpdateManager;
threadHelper.YearGreaterEqual=yearGreaterEqual;
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackYearGreaterEqual, threadHelper);
try { Thread.Sleep(WAIT_TIME_MS); } catch(Exception) { ;} // wait
}
MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadInsiderTransactions]End, total took {0}(ms)",profiler.End()));
}
}
private List<String> GetQueueSymbols()
{
List<String> queueSymbols = new List<String>();
int index = currentIndex;
for (; index < currentIndex + MaxThreads && index < symbols.Count; index++)
{
queueSymbols.Add(symbols[index]);
}
currentIndex = index;
return queueSymbols;
}
public void ThreadPoolCallback(Object threadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
LoadInsiderTransactionsSymbolEx(threadHelper.Symbol);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} ended for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
threadHelper.ResetEvent.Set();
}
public static void LoadInsiderTransactionsSymbolEx(String symbol)
{
symbol = symbol.ToUpper();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load insider transactions for {0}", symbol));
InsiderTransactions insiderTransactions = MarketDataHelper.GetInsiderTransactions(symbol);
if (null == insiderTransactions || 0 == insiderTransactions.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No insider transactions for {0}", symbol));
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions, Saving {0} records for {1}", insiderTransactions.Count, symbol));
InsiderTransactionDA.InsertInsiderTransactions(insiderTransactions);
MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions - Done.");
}
public void ThreadPoolCallbackYearGreaterEqual(Object threadHelperContext)
{
InsiderTransactionThreadHelper threadHelper = (InsiderTransactionThreadHelper)threadHelperContext;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
LoadInsiderTransactionsYearGreaterEqualEx(threadHelper.Symbol,threadHelper.YearGreaterEqual);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} ended for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
threadHelper.UpdateManager.Add(threadHelper.Symbol);
threadHelper.ResetEvent.Set();
}
public static void LoadInsiderTransactionsYearGreaterEqualEx(String symbol, int yearGreaterEqual)
{
symbol = symbol.ToUpper();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load insider transactions for {0} years>={1}", symbol,yearGreaterEqual));
InsiderTransactions insiderTransactions = MarketDataHelper.GetInsiderTransactionsYear(symbol, yearGreaterEqual);
if (null == insiderTransactions || 0 == insiderTransactions.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No insider transactions for {0} years>={1}", symbol,yearGreaterEqual));
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions, Saving {0} records for {1} years>={2}", insiderTransactions.Count, symbol, yearGreaterEqual));
InsiderTransactionDA.DeleteInsiderTransactionsYearsGreaterEqual(symbol, yearGreaterEqual);
InsiderTransactionDA.InsertInsiderTransactions(insiderTransactions);
MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions - Done.");
}
}
}

243
MarketData/MarketDataLib/Helper/InsiderTransactionsParser.cs Executable file
View File

@@ -0,0 +1,243 @@
using System;
using System.Text;
using System.Collections.Generic;
using System.IO;
using HtmlAgilityPack;
using MarketData.Utils;
using MarketData.MarketDataModel;
using MarketDataLib.Utility;
using System.Collections.Concurrent;
namespace MarketData.Helper
{
public class InsiderTransactionsParser
{
private static ConcurrentDictionary<String,String> transactionCodes=new ConcurrentDictionary<String,String>();
private static InsiderTransactionsParser instance=null;
private InsiderTransactionsParser()
{
BuildTransactionCodes();
}
public static InsiderTransactionsParser GetInstance()
{
lock(typeof(InsiderTransactionsParser))
{
if(null==instance)instance=new InsiderTransactionsParser();
return instance;
}
}
/// <summary>
/// Parse the SECFilings. Each SECFiling may contain a list of InsiderTransaction
/// </summary>
/// <param name="secFilings">The SECFilings.</param>
public InsiderTransactions Parse(SECFilings secFilings)
{
try
{
InsiderTransactions insiderTransactions=new InsiderTransactions();
if(null==secFilings || 0==secFilings.Count)return insiderTransactions;
foreach(SECFiling secFiling in secFilings)
{
InsiderTransactions secFilinginsiderTransactions =
Parse(secFiling.FormText, secFiling.Symbol,
secFiling.SECAccessionNumber,
secFiling.Form, secFiling.FilingDate);
if(null!=secFilinginsiderTransactions && secFilinginsiderTransactions.Count>0)
insiderTransactions.AddRange(secFilinginsiderTransactions);
}
return insiderTransactions;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse SECFiling got exception {0}",exception.ToString()));
return new InsiderTransactions();
}
}
public InsiderTransactions Parse(String strHtml,String symbol, String secAccessionNumber, String form, DateTime filingDate)
{
try
{
InsiderTransactions insiderTransactions = new InsiderTransactions();
if(null==strHtml)return null;
List<Dictionary<String,String>> dictionaryList=new List<Dictionary<String,String>>();
String insiderName=null;
byte[] streamBytes = Encoding.ASCII.GetBytes(strHtml);
MemoryStream memoryStream = new MemoryStream(streamBytes);
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.Load(memoryStream);
HtmlNodeCollection tables=htmlDocument.DocumentNode.SelectNodes("//table");
if(null==tables || tables.Count<5)return null;
HtmlNode nameAndAddressTable=FindTable(tables,"1. Name and Address of Reporting Person");
if(null==nameAndAddressTable)return null;
HtmlNodeCollection nameAndAddressRows = nameAndAddressTable.SelectNodes(".//tr");
if(nameAndAddressRows.Count<2)return null;
insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);
for(int index=0;index<tables.Count;index++)
{
HtmlNodeCollection nodeCollection = tables[index].SelectNodes(".//tr");
foreach(HtmlNode node in nodeCollection)
{
HtmlNodeCollection dataCollection = node.SelectNodes(".//td");
if(null==dataCollection || 0==dataCollection.Count)continue;
if(11==dataCollection.Count)
{
Dictionary<String,String> dictionary=new Dictionary<String,String>();
dictionary.Add("Title of Security",dataCollection[0].InnerHtml);
dictionary.Add("Transaction Date",dataCollection[1].InnerHtml);
dictionary.Add("Transaction Code",dataCollection[3].InnerHtml);
dictionary.Add("Securities Acquired or Disposed",dataCollection[5].InnerHtml);
dictionary.Add("Acquired or Disposed",dataCollection[6].InnerHtml);
dictionary.Add("Price",dataCollection[7].InnerHtml);
dictionary.Add("Ownership Form",dataCollection[9].InnerHtml);
dictionaryList.Add(dictionary);
}
}
}
int sequenceNumber=0;
foreach(Dictionary<String,String> dictionary in dictionaryList)
{
String strItem=null;
InsiderTransaction insiderTransaction=new InsiderTransaction();
insiderTransaction.Symbol=symbol;
insiderTransaction.SECAccessionNumber=secAccessionNumber;
insiderTransaction.Form=form;
insiderTransaction.FilingDate=filingDate;
insiderTransaction.InsiderName=insiderName;
insiderTransaction.Securities=GetFirstSection(Sections.GetSections(dictionary["Title of Security"]));
insiderTransaction.OwnershipType=GetOwnershipForm(Sections.GetSections(dictionary["Ownership Form"]));
if(String.IsNullOrEmpty(insiderTransaction.OwnershipType))continue;
strItem=GetFirstSection(Sections.GetSections(dictionary["Transaction Date"]));
if(String.IsNullOrEmpty(strItem))continue;
insiderTransaction.TransactionDate=Utility.ParseDate(strItem);
insiderTransaction.NatureOfTransaction=transactionCodes.ContainsKey(GetFirstSection(Sections.GetSections(dictionary["Transaction Code"])))?transactionCodes[GetFirstSection(Sections.GetSections(dictionary["Transaction Code"]))]:Constants.CONST_QUESTION;
insiderTransaction.NumberOrValueAcquiredDisposed=FeedParser.ParseValue(GetFirstSection(Sections.GetSections(dictionary["Securities Acquired or Disposed"])));
// insiderTransaction.FormRowNumber=((decimal)insiderTransaction.NumberOrValueAcquiredDisposed).ToString();
insiderTransaction.FormRowNumber=(++sequenceNumber).ToString();
insiderTransaction.Price=FeedParser.ParseValue(CombineSections(Sections.GetSections(dictionary["Price"])));
String acquiredOrDisposed=GetAcquiredOrDisposed(dictionary);
if(null==acquiredOrDisposed)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse - Cannot determine acquied or disposed for accession#{0}",secAccessionNumber));
continue;
}
if(acquiredOrDisposed.Equals("D"))
{
insiderTransaction.NumberOrValueAcquiredDisposed*=-1;
}
insiderTransactions.Add(insiderTransaction);
}
return insiderTransactions;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse - Got exception {0}",exception.ToString()));
return null;
}
}
private HtmlNode FindTable(HtmlNodeCollection tables,String startsWith)
{
if(null==tables)return null;
for(int index=0;index<tables.Count;index++)
{
HtmlNodeCollection collection = tables[index].SelectNodes(".//tr");
if(null==collection)continue;
for(int itemIndex=0;itemIndex<collection.Count;itemIndex++)
{
String marker=collection[itemIndex].InnerText;
if(null!=marker && marker.StartsWith(startsWith))
{
return tables[index];
}
}
}
return null;
}
private String ApplyNameCase(String name)
{
StringBuilder sb=new StringBuilder();
name=name.ToLower();
String[] parts=name.Split(' ');
for(int index=0;index<parts.Length;index++)
{
String item=parts[index];
if(item.Length>1)sb.Append(item.Substring(0,1).ToUpper()+item.Substring(1));
else sb.Append(item.ToUpper());
if(index<parts.Length-1)sb.Append(" ");
}
return sb.ToString();
}
private String GetOwnershipForm(List<String> sections)
{
String item=GetFirstSection(sections);
if(null==item)item="";
item=item.ToUpper();
if(item.Equals("D"))return "Direct Ownership";
else if(item.Equals("I"))return "Indirect Ownership";
return "";
}
private String GetAcquiredOrDisposed(Dictionary<String,String> dictionary)
{
if(!dictionary.ContainsKey("Acquired or Disposed"))return null;
List<String> section=Sections.GetSections(dictionary["Acquired or Disposed"]);
if(null==section||0==section.Count)return null;
String acquiredOrDisposed=section[0];
if(null==acquiredOrDisposed)return null;
return acquiredOrDisposed.Trim().ToUpper();
}
private String GetFirstSection(List<String> sections)
{
if(null==sections || 0==sections.Count)return "";
return sections[0];
}
private String CombineSections(List<String> sections)
{
StringBuilder sb=new StringBuilder();
if(null==sections || 0==sections.Count)return "";
foreach(String section in sections)
{
sb.Append(section);
}
return sb.ToString();
}
private void BuildTransactionCodes()
{
transactionCodes.TryAdd("P","Open market or private purchase of non-derivative or derivative security");
transactionCodes.TryAdd("S","Open market or private sale of non-derivative or derivative security");
transactionCodes.TryAdd("V","Transaction voluntarily reported earlier than required");
transactionCodes.TryAdd("A","Grant, award or other acquisition pursuant to Rule 16b-3(d)");
transactionCodes.TryAdd("D","Disposition to the issuer of issuer equity securities pursuant to Rule 16b-3(e)");
transactionCodes.TryAdd("F","Payment of exercise price or tax liability by delivering or withholding securities incident to the receipt, exercise or vesting of a security issued in accordance with Rule 16b-3");
transactionCodes.TryAdd("I","Discretionary transaction in accordance with Rule 16b-3(f) resulting in acquisition or disposition of issuer securities");
transactionCodes.TryAdd("M","Exercise or conversion of derivative security exempted pursuant to Rule 16b-3");
transactionCodes.TryAdd("C","Conversion of derivative security");
transactionCodes.TryAdd("E","Expiration of short derivative position");
transactionCodes.TryAdd("H","Expiration (or cancellation) of long derivative position with value received");
transactionCodes.TryAdd("O","Exercise of out-of-the-money derivative security");
transactionCodes.TryAdd("X","Exercise of in-the-money or at-the-money derivative security");
transactionCodes.TryAdd("G","Bona fide gift");
transactionCodes.TryAdd("L","Small acquisition under Rule 16a-6");
transactionCodes.TryAdd("W","Acquisition or disposition by will or the laws of descent and distribution");
transactionCodes.TryAdd("Z","Deposit into or withdrawal from voting trust");
transactionCodes.TryAdd("J","Other acquisition or disposition (describe transaction)");
transactionCodes.TryAdd("K","Transaction in equity swap or instrument with similar characteristics");
transactionCodes.TryAdd("U","Disposition pursuant to a tender of shares in a change of control transaction");
}
}
}

5527
MarketData/MarketDataLib/Helper/MarketDataHelper.cs Executable file
View File

File diff suppressed because it is too large Load Diff

103
MarketData/MarketDataLib/Helper/MarketDataHelperBase.cs Executable file
View File

@@ -0,0 +1,103 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class MarketDataHelperBase<T>
{
private List<T> queue=null;
private int currentIndex=-1;
public MarketDataHelperBase()
{
}
public List<T> Queue
{
get{return queue;}
set{queue=value;}
}
public void ShuffleQueue()
{
int itemCount = queue.Count;
Random generator = new Random();
for (int index = 0; index < itemCount; index++)
{
int r1;
int r2;
r1 = generator.Next(0, itemCount - 1);
r2 = generator.Next(0, itemCount - 1);
while (r2.Equals(r1)) r2 = generator.Next(0, itemCount - 1);
T temp = queue[r1];
queue[r1] = queue[r2];
queue[r2] = temp;
}
}
public T PeekQueueItem()
{
if(null==queue)return default(T);
if (currentIndex+1 >= queue.Count) return default(T);
return queue[currentIndex+1];
}
public T GetQueueItem()
{
if(null==queue)return default(T);
if (currentIndex+1 >= queue.Count) return default(T);
return queue[++currentIndex];
}
public int Index
{
get{return currentIndex;}
set{currentIndex=value;}
}
public ManualResetEvent[] ResizeEvents(ManualResetEvent[] events)
{
ManualResetEvent[] busyEvents=GetBusyEvents(events);
if(busyEvents.Length!=0)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Resizing event queue to {0}",busyEvents.Length));
ManualResetEvent[] resizedEvents=new ManualResetEvent[busyEvents.Length];
Array.Copy(busyEvents, resizedEvents, busyEvents.Length);
return resizedEvents;
}
return events;
}
public ManualResetEvent[] GetBusyEvents(ManualResetEvent[] resetEvents)
{
List<ManualResetEvent> availableEvents = new List<ManualResetEvent>();
for (int index = 0; index < resetEvents.Length; index++)
{
if (!resetEvents[index].WaitOne(0))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Event {0} is still doing work",resetEvents[index].SafeWaitHandle.DangerousGetHandle().ToString()));
availableEvents.Add(resetEvents[index]);
}
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The event queue has {0} busy slots",availableEvents.Count));
return availableEvents.ToArray();
}
public ManualResetEvent[] GetAvailableEvents(ManualResetEvent[] resetEvents)
{
List<ManualResetEvent> availableEvents = new List<ManualResetEvent>();
for (int index = 0; index < resetEvents.Length; index++)
{
if (resetEvents[index].WaitOne(0))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Event {0} is available for work",resetEvents[index].SafeWaitHandle.DangerousGetHandle().ToString()));
availableEvents.Add(resetEvents[index]);
}
}
return availableEvents.ToArray();
}
}
}

133
MarketData/MarketDataLib/Helper/MovingAverageHelperSheet.cs Executable file
View File

@@ -0,0 +1,133 @@
// using System;
// using System.IO;
// using System.Collections.Generic;
// using Microsoft.Office.Interop.Excel;
// using MarketData.DataAccess;
// using MarketData.MarketDataModel;
// using MarketData.Generator;
// using MarketData.Utils;
// using MarketData.Generator.MovingAverage;
// namespace MarketData.Helper
// {
// public class MovingAverageHelperSheet
// {
// public MovingAverageHelperSheet()
// {
// }
// public static bool GenerateMovingAverageSheet(String strPathTemplateFile, String symbol)
// {
// Microsoft.Office.Interop.Excel.Application excelApp = null;
// Microsoft.Office.Interop.Excel.Workbook workbook = null;
// Microsoft.Office.Interop.Excel.Worksheet worksheet = null;
// Microsoft.Office.Interop.Excel.Sheets worksheets = null;
// int rowOffset = 1;
// try
// {
// String companyName = PricingDA.GetNameForSymbol(symbol);
// DateGenerator dateGenerator = new DateGenerator();
// DateTime startDate = dateGenerator.GetPrevBusinessDay(DateTime.Now);
// String currentWorkingDirectory = Directory.GetCurrentDirectory();
// if (!File.Exists(strPathTemplateFile))
// {
// MDTrace.WriteLine(LogLevel.DEBUG,"Cannot locate " + strPathTemplateFile);
// return false;
// }
// MovingAverages movingAverages = MovingAverageGenerator.GenerateMovingAverages(symbol);
// String pathOutputFile = currentWorkingDirectory + "\\" + symbol + "-MA-" + Utility.DateTimeToStringMMHDDHYYYY(movingAverages.ThruDate);
// MDTrace.WriteLine(LogLevel.DEBUG,"Generating " + pathOutputFile);
// File.Delete(pathOutputFile + ".xlsx");
// excelApp = new Microsoft.Office.Interop.Excel.Application();
// excelApp.ScreenUpdating = false;
// workbook = excelApp.Workbooks.Open(strPathTemplateFile, 0, false, 5, "", "", false, Microsoft.Office.Interop.Excel.XlPlatform.xlWindows, "", true, false, 0, true, false, false);
// worksheets = workbook.Worksheets;
// worksheet = (Microsoft.Office.Interop.Excel.Worksheet)worksheets.get_Item("Sheet1");
// Microsoft.Office.Interop.Excel.ChartObjects chartObjects = worksheet.ChartObjects();
// Microsoft.Office.Interop.Excel.ChartObject movingAverageChartObject = chartObjects.Item(1); // yes, chart 1 is the fast line
// movingAverageChartObject.Chart.ChartTitle.Text = companyName + " (" + symbol + ") " + Utility.DateTimeToStringMMHDDHYYYY(movingAverages.FromDate) + " - " + Utility.DateTimeToStringMMHDDHYYYY(movingAverages.ThruDate) + " MA(55,21,5)";
// Axis vertAxis = (Axis)movingAverageChartObject.Chart.Axes(XlAxisType.xlValue, XlAxisGroup.xlPrimary);
// vertAxis.MaximumScaleIsAuto = false;
// vertAxis.MaximumScale = GetMaxData(movingAverages);
// vertAxis.MinimumScaleIsAuto = false;
// vertAxis.MinimumScale = GetMinData(movingAverages);
// vertAxis.HasMajorGridlines = true;
// Axis horzAxis = (Axis)movingAverageChartObject.Chart.Axes(XlAxisType.xlCategory, XlAxisGroup.xlPrimary);
// horzAxis.HasMajorGridlines = true;
// for (int index = 0; index < movingAverages.Count; index++)
// {
// MovingAverageElement movingAverageElement = movingAverages[index];
// worksheet.Cells[index + 1 + rowOffset, 1] = movingAverageElement.Symbol;
// worksheet.Cells[index + 1 + rowOffset, 2] = movingAverageElement.Date;
// worksheet.Cells[index + 1 + rowOffset, 3] = movingAverageElement.Close;
// worksheet.Cells[index + 1 + rowOffset, 4] = movingAverageElement.High;
// worksheet.Cells[index + 1 + rowOffset, 5] = movingAverageElement.Low;
// worksheet.Cells[index + 1 + rowOffset, 6] = movingAverageElement.MA55;
// worksheet.Cells[index + 1 + rowOffset, 7] = movingAverageElement.MA21;
// worksheet.Cells[index + 1 + rowOffset, 8] = movingAverageElement.MA5;
// }
// workbook.SaveAs(pathOutputFile, Microsoft.Office.Interop.Excel.XlFileFormat.xlOpenXMLWorkbook, Type.Missing, Type.Missing, false, false, Microsoft.Office.Interop.Excel.XlSaveAsAccessMode.xlNoChange, Microsoft.Office.Interop.Excel.XlSaveConflictResolution.xlUserResolution, true, Type.Missing, Type.Missing, Type.Missing);
// return true;
// }
// catch (Exception exception)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
// return false;
// }
// finally
// {
// if (null != worksheet) ReleaseObject(worksheet);
// if (null != worksheets) ReleaseObject(worksheets);
// if (null != workbook) ReleaseObject(workbook);
// excelApp.Quit();
// }
// }
// private static double GetMinData(MovingAverages movingAverages)
// {
// double minData = double.MaxValue;
// for (int index = 0; index < movingAverages.Count; index++)
// {
// MovingAverageElement movingAverageElement = movingAverages[index];
// if (movingAverageElement.Close < minData) minData = movingAverageElement.Close;
// if (movingAverageElement.High < minData) minData = movingAverageElement.High;
// if (movingAverageElement.Low < minData) minData = movingAverageElement.Low;
// if (movingAverageElement.MA55 < minData) minData = movingAverageElement.MA55;
// if (movingAverageElement.MA21 < minData) minData = movingAverageElement.MA21;
// if (movingAverageElement.MA5 < minData) minData = movingAverageElement.MA5;
// }
// return minData - (minData * .05);
// }
// private static double GetMaxData(MovingAverages movingAverages)
// {
// double maxData = 0;
// for (int index = 0; index < movingAverages.Count; index++)
// {
// MovingAverageElement movingAverageElement = movingAverages[index];
// if (movingAverageElement.Close > maxData) maxData = movingAverageElement.Close;
// if (movingAverageElement.High > maxData) maxData = movingAverageElement.High;
// if (movingAverageElement.Low > maxData) maxData = movingAverageElement.Low;
// if (movingAverageElement.MA55 > maxData) maxData = movingAverageElement.MA55;
// if (movingAverageElement.MA21 > maxData) maxData = movingAverageElement.MA21;
// if (movingAverageElement.MA5 > maxData) maxData = movingAverageElement.MA5;
// }
// return maxData + (maxData * .05);
// }
// private static void ReleaseObject(object obj)
// {
// try
// {
// System.Runtime.InteropServices.Marshal.FinalReleaseComObject(obj);
// obj = null;
// }
// catch (Exception ex)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,ex.ToString());
// obj = null;
// }
// finally
// {
// GC.Collect();
// }
// }
// }
// }

106
MarketData/MarketDataLib/Helper/OptionsChainsMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,106 @@
// using System;
// using System.Collections.Generic;
// using System.Linq;
// using System.Text;
// using System.Threading;
// using MarketData.MarketDataModel;
// using MarketData.DataAccess;
// using MarketData.Utils;
// namespace MarketData.Helper
// {
// public class OptionsChainsMarketDataHelper : MarketDataHelperBase<String>
// {
// private static int MaxThreads = 10;
// public OptionsChainsMarketDataHelper()
// {
// }
// // ******************************************************************************************************************************************************
// // ********************************************************** O P T I M I Z E D E V E N T H A N D L E R S T A R T ***********************************
// // ******************************************************************************************************************************************************
// public bool LoadOptionsChains(String symbol)
// {
// List<String> symbols=new List<String>();
// symbols.Add(symbol);
// return LoadOptionsChains(symbols);
// }
// public bool LoadOptionsChains(List<String> symbols)
// {
// Profiler profiler=new Profiler();
// try
// {
// Queue=symbols;
// Index=-1;
// ManualResetEvent[] resetEvents = new ManualResetEvent[MaxThreads];
// for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)resetEvents[eventIndex] = new ManualResetEvent(true);
// MDTrace.WriteLine(String.Format("Queuing options chains ..."));
// while (true)
// {
// ManualResetEvent[] availableEvents=GetAvailableEvents(resetEvents);
// ManualResetEvent[] busyEvents=GetBusyEvents(resetEvents);
// if (null == PeekQueueItem() && 0==busyEvents.Length)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Queue contains {0} items, busy events {1}, all done.",0,busyEvents.Length));
// break;
// }
// for (int index = 0; index < availableEvents.Length; index++)
// {
// String symbol=GetQueueItem();
// if (null != symbol)
// {
// availableEvents[index].Reset();
// ThreadHelper optionsChainsThreadHelper = new ThreadHelper(symbol, availableEvents[index]);
// ThreadPool.QueueUserWorkItem(ThreadPoolCallback, optionsChainsThreadHelper);
// }
// else
// {
// busyEvents=GetBusyEvents(resetEvents);
// if(busyEvents.Length!=availableEvents.Length)
// {
// ManualResetEvent[] resizedEvents=new ManualResetEvent[busyEvents.Length];
// Array.Copy(busyEvents, resizedEvents,busyEvents.Length);
// resetEvents = resizedEvents;
// }
// break;
// }
// } // for
// MDTrace.WriteLine(LogLevel.DEBUG,"Waiting for free slots...");
// if(resetEvents.Length>0)WaitHandle.WaitAny(resetEvents);
// if(null==PeekQueueItem())resetEvents=ResizeEvents(resetEvents);
// } // while
// MDTrace.WriteLine(LogLevel.DEBUG,"Options chains handler completed.");
// return true;
// }
// finally
// {
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadOptionsChains]End, total took {0}(ms)",profiler.End()));
// }
// }
// public void ThreadPoolCallback(Object optionsChainsThreadHelperContext)
// {
// ThreadHelper optionsChainsThreadHelper = (ThreadHelper)optionsChainsThreadHelperContext;
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Options Chains, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, optionsChainsThreadHelper.Symbol));
// LoadOptionsChainsSymbolEx(optionsChainsThreadHelper.Symbol);
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Options Chains, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, optionsChainsThreadHelper.Symbol));
// optionsChainsThreadHelper.ResetEvent.Set();
// }
// public static void LoadOptionsChainsSymbolEx(String symbol)
// {
// if(null==symbol)return;
// symbol = symbol.ToUpper();
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Options Chains for {0}", symbol));
// Options options = MarketDataHelper.GetOptions(symbol);
// if (null == options || 0 == options.Count)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,"Unable to retrieve option chain for " + symbol);
// return;
// }
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Options Chains, saving {0} records for {1}", options.Count, symbol));
// if (OptionsDA.AddOptions(options)) MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Options Chains, Added {0} for {1}", options.Count, symbol));
// else MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Options Chains, Failed to add options chain for {0}",symbol));
// }
// }
// }

831
MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,831 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
using System.Net;
namespace MarketData.Helper
{
public class PricingThreadHelper : ThreadHelper
{
private DateTime pricingDate=DateTime.Parse("12-31-9999");
private DateTime? startDate;
private TimeZoneHelper timeZoneInfo;
public PricingThreadHelper(String symbol,ManualResetEvent resetEvent)
{
Symbol=symbol;
ResetEvent=resetEvent;
}
public PricingThreadHelper(String symbol, DateTime pricingDate,ManualResetEvent resetEvent)
{
Symbol = symbol;
this.pricingDate=pricingDate;
ResetEvent = resetEvent;
timeZoneInfo=new TimeZoneHelper(pricingDate);
}
public PricingThreadHelper(String symbol, ManualResetEvent resetEvent,DateTime? startDate=null)
{
Symbol = symbol;
this.pricingDate=DateTime.Now;
this.startDate=startDate;
ResetEvent = resetEvent;
}
public TimeZoneHelper TimeZoneInfo
{
get{return timeZoneInfo;}
}
public DateTime PricingDate
{
get{return pricingDate;}
}
public DateTime? StartDate
{
get{return startDate;}
}
}
public class PricingMarketDataHelper
{
private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads;
private static int WaitBetweenRequests=125;
private List<String> symbols;
private int currentIndex = 0;
public PricingMarketDataHelper()
{
}
public static Price RollPriceForward(String symbol)
{
try
{
DateGenerator dateGenerator=new DateGenerator();
DateTime latestDate=PremarketDA.GetLatestMarketDate();
if(Utility.IsEpoch(latestDate))return null;
Price latestPrice=PricingDA.GetPrice(symbol);
if(null==latestPrice)return null;
DateTime latestPriceDateSymbol=latestPrice.Date;
latestPrice.Date=latestDate;
PricingDA.InsertPrice(latestPrice);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} price rolled forward from {1} to {2}",symbol,latestPriceDateSymbol.ToShortDateString(),latestDate.ToShortDateString()));
return latestPrice;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception rolling price forward for {0}. Exception was {1}",symbol,exception.ToString()));
return null;
}
}
public bool UpdateLatestPriceAll()
{
Profiler profiler=new Profiler();
try
{
symbols = PricingDA.GetSymbols();
currentIndex=0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdateLatestPriceAll, pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"UpdateLatestPriceAll, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateLatestPriceAll]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdatePricesBigCharts(DateTime? startDate=null) // get prices from BigCharts
{
Profiler profiler=new Profiler();
try
{
symbols = PricingDA.GetSymbols();
currentIndex=0;
if(null==startDate)startDate=DateTime.Now;
TimeZoneHelper timeZoneHelper=new TimeZoneHelper(startDate.Value);
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index],startDate.Value,resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesBigCharts, pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"UpdatePricesBigCharts, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesBigCharts]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdatePricesYahoo(DateTime? startDate=null) // get prices from Yahoo
{
Profiler profiler=new Profiler();
try
{
symbols=PricingDA.GetSymbols();
currentIndex=0;
if(null==startDate) startDate=DateTime.Now;
TimeZoneHelper timeZoneHelper=new TimeZoneHelper(startDate.Value);
while(true)
{
List<String> queueSymbols=GetQueueSymbols();
if(null==queueSymbols||0==queueSymbols.Count) break;
ManualResetEvent[] resetEvents=new ManualResetEvent[queueSymbols.Count];
for(int eventIndex=0;eventIndex<resetEvents.Length;eventIndex++)
{
resetEvents[eventIndex]=new ManualResetEvent(false);
}
for(int index=0;index<queueSymbols.Count;index++)
{
PricingThreadHelper pricingThreadHelper=new PricingThreadHelper(queueSymbols[index],startDate.Value,resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesYahoo,pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"UpdatePricesYahoo, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesBigCharts]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdatePricesYahooSweep(DateTime? startDate=null) // get prices from Yahoo
{
Profiler profiler=new Profiler();
try
{
symbols=PricingDA.GetSymbols();
currentIndex=0;
if(null==startDate) startDate=DateTime.Now;
TimeZoneHelper timeZoneHelper=new TimeZoneHelper(startDate.Value);
while(true)
{
List<String> queueSymbols=GetQueueSymbols();
if(null==queueSymbols||0==queueSymbols.Count) break;
ManualResetEvent[] resetEvents=new ManualResetEvent[queueSymbols.Count];
for(int eventIndex=0;eventIndex<resetEvents.Length;eventIndex++)
{
resetEvents[eventIndex]=new ManualResetEvent(false);
}
for(int index=0;index<queueSymbols.Count;index++)
{
PricingThreadHelper pricingThreadHelper=new PricingThreadHelper(queueSymbols[index],startDate.Value,resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesYahooSweep,pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"UpdatePricesYahooSweep, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesBigCharts]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdatePricesAsOfSymbol(String symbol,DateTime asOf) // get prices from BigCharts
{
Profiler profiler=new Profiler();
try
{
symbols=new List<String>();
symbols.Add(symbol);
currentIndex=0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index],asOf);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesAsOf, pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"UpdatePricesAsOfSymbol, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesAsOfSymbol]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdatePriceAsOfSymbolYahoo(String symbol,DateTime asOf) // get prices from Yahoo
{
Profiler profiler=new Profiler();
try
{
symbols=new List<String>();
symbols.Add(symbol);
currentIndex=0;
while(true)
{
List<String> queueSymbols=GetQueueSymbols();
if(null==queueSymbols||0==queueSymbols.Count) break;
ManualResetEvent[] resetEvents=new ManualResetEvent[queueSymbols.Count];
for(int eventIndex=0;eventIndex<resetEvents.Length;eventIndex++)
{
resetEvents[eventIndex]=new ManualResetEvent(false);
}
for(int index=0;index<queueSymbols.Count;index++)
{
PricingThreadHelper pricingThreadHelper=new PricingThreadHelper(queueSymbols[index],asOf,resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesYahoo,pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"UpdatePricesAsOfSymbol, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesAsOfSymbol]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdatePricesAsOf(DateTime asOf)
{
Profiler profiler=new Profiler();
try
{
symbols=PricingDA.GetSymbols();
currentIndex=0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index],asOf);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdatePricesAsOf, pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"UpdatePricesAsOf, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesAsOf]End, total took {0}(ms)",profiler.End()));
}
}
public bool GetMissingPricesSymbol(String symbol,DateTime? startDate=null)
{
Profiler profiler=new Profiler();
try
{
List<String> symbols=new List<String>();
symbols.Add(symbol);
this.symbols=symbols;
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index],startDate);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackGetMissingPrice, pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"Get missing prices, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetMissingPrices]End, total took {0}(ms)",profiler.End()));
}
}
public bool GetMissingPrices(DateTime? startDate=null)
{
Profiler profiler=new Profiler();
try
{
this.symbols=PricingDA.GetSymbols();
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], resetEvents[index],startDate);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackGetMissingPrice, pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"Get missing prices, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetMissingPrices]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdateLatestPriceSymbol(String symbol)
{
List<String> symbols=new List<String>();
symbols.Add(symbol);
return UpdateLatestPrices(symbols);
}
public bool UpdateLatestPrices(List<String> symbols)
{
Profiler profiler=new Profiler();
try
{
DateTime pricingDate=DateTime.Now;
this.symbols = symbols;
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], pricingDate,resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdateLatestPrice, pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load prices, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateLatestPrices]End, total took {0}(ms)",profiler.End()));
}
}
public bool LoadPrices(List<String> symbols)
{
Profiler profiler=new Profiler();
try
{
DateTime pricingDate=DateTime.Now;
this.symbols = symbols;
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
PricingThreadHelper pricingThreadHelper = new PricingThreadHelper(queueSymbols[index], pricingDate,resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadPrices, pricingThreadHelper);
try{Thread.Sleep(WaitBetweenRequests);}catch(Exception){;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load prices, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadPrices]End, total took {0}(ms)",profiler.End()));
}
}
// ************************************************************************************************************************************
// *************************************************** T H R E A D P O O L C A L L B A C K ******************************************
// ************************************************************************************************************************************
public void ThreadPoolCallbackUpdateLatestPriceAll(Object pricingThreadHelperContext)
{
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("UpdateLatestPriceAll, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
UpdateLatestPriceAllEx(pricingThreadHelper.Symbol);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("UpdateLatestPriceAll, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
}
finally
{
pricingThreadHelper.ResetEvent.Set();
}
}
public void ThreadPoolCallbackUpdatePricesBigCharts(Object pricingThreadHelperContext)
{
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (BigCharts), Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
UpdatePriceBigChartsEx(pricingThreadHelper.Symbol,pricingThreadHelper.TimeZoneInfo);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (BigCharts), Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
}
finally
{
pricingThreadHelper.ResetEvent.Set();
}
}
public void ThreadPoolCallbackUpdatePricesYahoo(Object pricingThreadHelperContext)
{
PricingThreadHelper pricingThreadHelper=(PricingThreadHelper)pricingThreadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (Yahoo), Thread {0} started for {1}...",Thread.CurrentThread.ManagedThreadId,pricingThreadHelper.Symbol));
UpdatePriceYahoo(pricingThreadHelper.Symbol,pricingThreadHelper.TimeZoneInfo);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (Yahoo), Thread {0} ended for {1}",Thread.CurrentThread.ManagedThreadId,pricingThreadHelper.Symbol));
}
finally
{
pricingThreadHelper.ResetEvent.Set();
}
}
public void ThreadPoolCallbackUpdatePricesYahooSweep(Object pricingThreadHelperContext)
{
PricingThreadHelper pricingThreadHelper=(PricingThreadHelper)pricingThreadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (Yahoo), Thread {0} started for {1}...",Thread.CurrentThread.ManagedThreadId,pricingThreadHelper.Symbol));
UpdatePriceYahooSweep(pricingThreadHelper.Symbol,pricingThreadHelper.TimeZoneInfo);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price (Yahoo), Thread {0} ended for {1}",Thread.CurrentThread.ManagedThreadId,pricingThreadHelper.Symbol));
}
finally
{
pricingThreadHelper.ResetEvent.Set();
}
}
public void ThreadPoolCallbackUpdatePricesAsOf(Object pricingThreadHelperContext)
{
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
UpdatePriceAsOfEx(pricingThreadHelper.Symbol,pricingThreadHelper.StartDate.Value);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
}
finally
{
pricingThreadHelper.ResetEvent.Set();
}
}
public void ThreadPoolCallbackGetMissingPrice(Object pricingThreadHelperContext)
{
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
try
{
GetMissingPricesEx(pricingThreadHelper.Symbol,pricingThreadHelper.StartDate==null?new DateTime():pricingThreadHelper.StartDate.Value);
}
finally
{
pricingThreadHelper.ResetEvent.Set();
}
}
public void ThreadPoolCallbackUpdateLatestPrice(Object pricingThreadHelperContext)
{
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
try
{
Price latestPrice = MarketDataHelper.GetLatestPrice(pricingThreadHelper.Symbol);
if (null == latestPrice)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Error retrieving latest price for {0} *****",pricingThreadHelper.Symbol));
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,Price.Header);
MDTrace.WriteLine(LogLevel.DEBUG,latestPrice.ToString());
MDTrace.WriteLine(LogLevel.DEBUG,"Removing price for '"+pricingThreadHelper.Symbol+"' on "+Utility.DateTimeToStringMMSDDSYYYY(latestPrice.Date));
PricingDA.DeletePrice(pricingThreadHelper.Symbol,latestPrice.Date);
MDTrace.WriteLine(LogLevel.DEBUG,"Adding price for '" + pricingThreadHelper.Symbol + "' on " + Utility.DateTimeToStringMMSDDSYYYY(latestPrice.Date));
PricingDA.InsertPrice(latestPrice);
}
finally
{
pricingThreadHelper.ResetEvent.Set();
}
}
public void ThreadPoolCallbackLoadPrices(Object pricingThreadHelperContext)
{
PricingThreadHelper pricingThreadHelper = (PricingThreadHelper)pricingThreadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
LoadPricesSymbolEx(pricingThreadHelper.Symbol,pricingThreadHelper.PricingDate);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load price, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, pricingThreadHelper.Symbol));
}
finally
{
pricingThreadHelper.ResetEvent.Set();
}
}
// ****************************************************************************************************************************************
// ****************************************************************************************************************************************
// ****************************************************************************************************************************************
private List<String> GetQueueSymbols()
{
List<String> queueSymbols = new List<String>();
int index = currentIndex;
for (; index < currentIndex + MaxThreads && index < symbols.Count; index++)
{
queueSymbols.Add(symbols[index]);
}
currentIndex = index;
return queueSymbols;
}
// ********************************************************************************************************************************************
// ************************************************************ W O R K E R S *****************************************************************
// ********************************************************************************************************************************************
public static void GetMissingPricesEx(String symbol,DateTime startDate)
{
try
{
DateGenerator dateGenerator=new DateGenerator();
TimeSpan oneDay=new TimeSpan(1,0,0,0);
DateTime? historicalDate = null;
DateTime minDate=Constants.MIN_PRICING_DATE;
List<DateTime> pricingDatesList = PricingDA.GetPricingDatesForSymbol(symbol);
if (null == pricingDatesList || 0 == pricingDatesList.Count) return;
pricingDatesList=pricingDatesList.Where(x => x.Date>=minDate.Date).ToList();
if(null==pricingDatesList||0==pricingDatesList.Count) return;
DateTime beginDate = pricingDatesList[0];
DateTime endDate = pricingDatesList[pricingDatesList.Count - 1];
Dictionary<DateTime, DateTime> pricingDates = new Dictionary<DateTime, DateTime>();
for (int listIndex = 0; listIndex < pricingDatesList.Count; listIndex++)
{
DateTime pricingDate=pricingDatesList[listIndex];
pricingDates.Add(pricingDate.Date, pricingDate.Date);
}
Dictionary<DateTime, Price> marketPrices = new Dictionary<DateTime, Price>();
if (Utility.IsEpoch(startDate)) historicalDate = beginDate;
else historicalDate = startDate;
MDTrace.WriteLine(LogLevel.DEBUG,"Working on '" + symbol + "' from "+Utility.DateTimeToStringMMHDDHYYYY(historicalDate.Value)+" to "+Utility.DateTimeToStringMMHDDHYYYY(endDate));
Prices prices = MarketDataHelper.GetDailyPrices(symbol, historicalDate.Value, endDate); // yahoo finance
if(null==prices||0==prices.Count)prices=MarketDataHelper.GetPricesAsOf(symbol,historicalDate.Value,endDate); // try big charts
if (null == prices||0==prices.Count)return;
for (int listIndex = 0; listIndex < prices.Count; listIndex++)
{
Price price = prices[listIndex];
if(!marketPrices.ContainsKey(price.Date.Date))marketPrices.Add(price.Date,price);
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Retrieved {0} prices from feed for {1}",prices.Count,symbol));
bool added=false;
for (DateTime date = historicalDate.Value.Date; date < endDate.Date; date += oneDay)
{
if (pricingDates.ContainsKey(date.Date)) continue;
if (dateGenerator.IsWeekend(date)) continue;
Price price = null;
if (!marketPrices.ContainsKey(date.Date))continue;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Inserting price {0}...",price.ToString()));
price = marketPrices[date.Date];
MDTrace.WriteLine(LogLevel.DEBUG,price.ToString());
PricingDA.InsertPrice(price);
added=true;
}
if(!added)
{
MDTrace.WriteLine(LogLevel.DEBUG,"There were no new prices to add for '" + symbol);
}
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception {0}",exception.ToString()));
}
}
public static void UpdateLatestPriceAllEx(String symbol)
{
try
{
symbol = symbol.ToUpper();
Price latestPrice = MarketDataHelper.GetLatestPrice(symbol);
if (null == latestPrice)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Error retrieving latest price.");
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,Price.Header);
MDTrace.WriteLine(LogLevel.DEBUG,latestPrice.ToString());
MDTrace.WriteLine(LogLevel.DEBUG,"Removing price for '" + symbol + "' on " + Utility.DateTimeToStringMMSDDSYYYY(latestPrice.Date));
PricingDA.DeletePrice(symbol, latestPrice.Date);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Inserting price {0}...",latestPrice.ToString()));
PricingDA.InsertPrice(latestPrice);
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
}
}
public static void UpdatePriceBigChartsEx(String symbol,TimeZoneHelper timeZoneInfo)
{
try
{
Price price=PricingDA.GetPrice(symbol,timeZoneInfo.StartDate);
if(null!=price)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Already have latest price for '" + symbol + "'");
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Requesting price for {0} for start date:{1} end date:{2}",symbol,timeZoneInfo.StartDate,timeZoneInfo.StartDate));
Prices prices = MarketDataHelper.GetPricesAsOf(symbol, timeZoneInfo.StartDate,timeZoneInfo.StartDate);
if (null == prices||0==prices.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No prices (GetPricesAsOf) for '" + symbol+ "'");
return;
}
else
{
for (int secIndex = 0; secIndex < prices.Count; secIndex++)
{
MDTrace.WriteLine(LogLevel.DEBUG,prices[secIndex].ToString());
}
MDTrace.WriteLine(LogLevel.DEBUG,"Inserting....");
PricingDA.InsertPrices(prices);
MDTrace.WriteLine(LogLevel.DEBUG,"Done.");
}
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception {0}",exception.ToString()));
}
}
public static void UpdatePriceYahoo(String symbol,TimeZoneHelper timeZoneInfo)
{
try
{
Price price=PricingDA.GetPrice(symbol,timeZoneInfo.StartDate);
if(null!=price)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Already have latest price for '"+symbol+"'");
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Requesting price for {0} for start date:{1} end date:{2}",symbol,timeZoneInfo.StartDate,timeZoneInfo.StartDate));
price=MarketDataHelper.GetDailyPrice(symbol,timeZoneInfo.StartDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No price (UpdatePriceYahoo) for '"+symbol+"'");
return;
}
else
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Inserting price {0}...",price.ToString()));
PricingDA.InsertPrice(price);
}
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception {0}",exception.ToString()));
}
}
public static void UpdatePriceYahooSweep(String symbol,TimeZoneHelper timeZoneInfo)
{
try
{
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
Price price=PricingDA.GetPrice(symbol,timeZoneInfo.StartDate);
if(null!=price)
{
if(companyProfile==null || !companyProfile.PricingSourceEnum.Equals(CompanyProfile.EnumPricingSource.YAHOO))
{
MDTrace.WriteLine(LogLevel.DEBUG,"Already have latest price for '"+symbol+"'");
return;
}
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("UpdatePriceYahooSweep: Requesting price for {0} for start date:{1} end date:{2}",symbol,timeZoneInfo.StartDate,timeZoneInfo.StartDate));
price=MarketDataHelper.GetDailyPrice(symbol,timeZoneInfo.StartDate);
if(null==price)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No price (UpdatePriceYahoo) for '"+symbol+"'");
return;
}
else if(!price.IsValid)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Invalid price (UpdatePriceYahooSweep) for '"+symbol+"'");
return;
}
else
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Inserting price {0}...",price.ToString()));
PricingDA.InsertPrice(price);
}
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception {0}",exception.ToString()));
}
}
public static void UpdatePriceAsOfEx(String symbol,DateTime asOf)
{
try
{
DateTime latestDate = PricingDA.GetLatestDate(symbol);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Latest pricing date for {0} is {1}",symbol,latestDate.ToShortDateString()));
if (latestDate.Equals(asOf))
{
MDTrace.WriteLine(LogLevel.DEBUG,"Already have price for '" + symbol + "' on "+Utility.DateTimeToStringMMHDDHYYYY(asOf));
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Retrieving price for {0} on {1} from BigCharts",symbol,asOf.ToShortDateString()));
Price price = MarketDataHelper.GetPriceAsOf(symbol,asOf);
if (null == price)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Error retrieving price for {0} on {1}",symbol,asOf.ToShortDateString()));
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,Price.Header);
MDTrace.WriteLine(LogLevel.DEBUG,price.ToString());
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Inserting price {0}...",price.ToString()));
PricingDA.InsertPrice(price);
MDTrace.WriteLine(LogLevel.DEBUG,"");
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
}
}
public static void LoadPricesSymbolEx(String symbol,DateTime pricingDate)
{
DateGenerator dateGenerator = new DateGenerator();
DateTime startDate = pricingDate; // the least recent date
DateTime endDate = pricingDate; // the most recent date
endDate = dateGenerator.GetPrevBusinessDay(endDate);
startDate = PricingDA.GetLatestDate(symbol);
startDate = dateGenerator.FindNextBusinessDay(startDate);
if (startDate.Date > endDate.Date)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Already have latest price for '" + symbol + "'");
return;
}
Prices prices=MarketDataHelper.GetPricesAsOf(symbol,startDate,endDate);
if (null == prices||0==prices.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No prices (GetDailyPrices) for '" + symbol + "'");
return;
}
else
{
for (int secIndex = 0; secIndex < prices.Count; secIndex++)
{
MDTrace.WriteLine(LogLevel.DEBUG,prices[secIndex].ToString());
}
PricingDA.InsertPrices(prices);
MDTrace.WriteLine(LogLevel.DEBUG,"Done.");
}
}
}
}

162
MarketData/MarketDataLib/Helper/SECFilingMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,162 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class SECFilingMarketDataHelper
{
private static int MaxThreads = 5; // 10 requests per second is what is allowable under SEC.GOV. We'll request 5 symbols per batch. Note:each request may contain subrequests
private static int WAIT_TIME_MS=1000; // wait between requests
private List<String> symbols;
private int currentIndex = 0;
public SECFilingMarketDataHelper()
{
}
public bool UpdateSECFilings(List<String> symbols)
{
Profiler profiler=new Profiler();
try
{
this.symbols=symbols;
currentIndex=0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdateSECFiling, threadHelper);
try { Thread.Sleep(WAIT_TIME_MS); } catch(Exception) { ;} // SEC has a traffic limit
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load SEC Filings, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateSECFilings]Exception {0}",exception.ToString()));
return false;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateSECFilings]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdateSECFilings()
{
Profiler profiler=new Profiler();
try
{
List<String> watchListSymbols = WatchListDA.GetWatchList("Valuations"); // get the current watchlist symbols
List<String> secFilingSymbols=SECFilingDA.GetDistinctFilingSymbols(); // get the current SEC filing symbols
symbols=watchListSymbols.Union(secFilingSymbols).Distinct().ToList(); // use the union of the two
currentIndex = 0;
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackUpdateSECFiling, threadHelper);
try { Thread.Sleep(WAIT_TIME_MS); }catch(Exception) { ;} // SEC has a traffic limit
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load SEC Filings, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateSECFilings]Exception {0}",exception.ToString()));
return false;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateSECFilings]End, total took {0}(ms)",profiler.End()));
}
}
private List<String> GetQueueSymbols()
{
List<String> queueSymbols = new List<String>();
int index = currentIndex;
for (; index < currentIndex + MaxThreads && index < symbols.Count; index++)
{
queueSymbols.Add(symbols[index]);
}
currentIndex = index;
return queueSymbols;
}
public void ThreadPoolCallbackUpdateSECFiling(Object threadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load SEC Filing, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
UpdateSECFiling(threadHelper.Symbol);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load SEC Filing, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
}
finally
{
threadHelper.ResetEvent.Set();
}
}
public static void UpdateSECFiling(String symbol)
{
try
{
MDTrace.WriteLine(LogLevel.DEBUG,"UpdateSECFiling '"+symbol+"'");
String cik = PricingDA.GetCIKForSymbol(symbol);
if (null == cik)
{
MDTrace.WriteLine(LogLevel.DEBUG,"No CIK for symbol '" + symbol + "'");
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,"UpdateSECFiling for symbol '" + symbol + "'");
SECFilings secFilings = MarketDataHelper.GetSECFilings(symbol, cik);
if (null != secFilings)
{
for (int index = 0; index < secFilings.Count; index++)
{
StringBuilder sb = new StringBuilder();
SECFiling secFiling = secFilings[index];
sb.Append(secFiling.Symbol).Append(",");
sb.Append(Utility.DateTimeToStringYYYYHMMHDD(secFiling.FilingDate)).Append(",");
sb.Append(secFiling.SECAccessionNumber).Append(",");
sb.Append(secFiling.Sequence);
MDTrace.WriteLine(LogLevel.DEBUG,sb.ToString());
}
MDTrace.WriteLine(LogLevel.DEBUG,"Got "+secFilings.Count+" filings for symbol '"+symbol+"'");
SECFilingDA.InsertOrUpdateSECFilings(secFilings);
}
return;
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
return;
}
}
}
}

55
MarketData/MarketDataLib/Helper/SplitHelper.cs Executable file
View File

@@ -0,0 +1,55 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{ public class SplitHelper
{
// I have found that simply reloading prices with splits from Yahoo solves the issue of corporate actions as they are handled on Yahoo Finanace
public static bool ProcessSplits(Splits splits)
{
foreach(Split split in splits)ProcessSplit(split.Symbol);
return true;
}
public static bool ProcessSplit(String symbol)
{
Splits splits=SplitsDA.GetSplits(symbol);
if(null==splits||0==splits.Count)return false;
DateGenerator dateGenerator=new DateGenerator();
for(int index=0;index<splits.Count;index++)
{
Split split=splits[index];
if(!PricingDA.SecurityExists(split.Symbol))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No security detail for {0}",split.Symbol));
continue;
}
Prices prices=PricingDA.GetPrices(split.Symbol);
if(null==prices||0==prices.Count)continue;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Reloading prices for {0}",split.Symbol));
Prices marketPrices=MarketDataHelper.GetDailyPrices(split.Symbol,prices[prices.Count-1].Date,prices[0].Date);
if(null==marketPrices||0==marketPrices.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No prices for {0} from {1} to {2}",split.Symbol,Utility.DateTimeToStringMMHDDHYYYY(prices[prices.Count-1].Date),Utility.DateTimeToStringMMHDDHYYYY(prices[0].Date)));
continue;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Inserting prices for {0}",split.Symbol));
PricingDA.InsertPrices(marketPrices);
split.Applied=true;
split.AppliedLeastRecent=marketPrices[marketPrices.Count-1].Date;
split.AppliedMostRecent=marketPrices[0].Date;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Updating split status for {0}",split.Symbol));
SplitsDA.UpdateSplit(split);
}
return true;
}
}
}

112
MarketData/MarketDataLib/Helper/StochasticsHelperSheet.cs Executable file
View File

@@ -0,0 +1,112 @@
// using System;
// using System.IO;
// using Microsoft.Office.Interop.Excel;
// using MarketData.DataAccess;
// using MarketData.MarketDataModel;
// using MarketData.Generator;
// using MarketData.Utils;
// namespace MarketData.Helper
// {
// public class StochasticsSheetHelper
// {
// public StochasticsSheetHelper()
// {
// }
// public static bool GenerateStochasticsSheet(String strPathTemplateFile, String symbol, int dayCount,int periodN,int periodK)
// {
// Microsoft.Office.Interop.Excel.Application excelApp = null;
// Microsoft.Office.Interop.Excel.Workbook workbook = null;
// Microsoft.Office.Interop.Excel.Worksheet worksheet = null;
// Microsoft.Office.Interop.Excel.Sheets worksheets = null;
// int rowOffset = 1;
// try
// {
// String currentWorkingDirectory = Directory.GetCurrentDirectory();
// if (!File.Exists(strPathTemplateFile))
// {
// MDTrace.WriteLine(LogLevel.DEBUG,"Cannot locate " + strPathTemplateFile);
// return false;
// }
// DateGenerator dateGenerator = new DateGenerator();
// DateTime startDate = dateGenerator.GetPrevBusinessDay(DateTime.Now);
// Prices prices = PricingDA.GetPrices(symbol, startDate, dayCount);
// if (null == prices || 0 == prices.Count)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,"No prices for " + symbol);
// return false;
// }
// Price price = prices[0];
// String pathOutputFile = currentWorkingDirectory + "\\" + symbol + "-ST-" + Utility.DateTimeToStringMMHDDHYYYY(price.Date);
// MDTrace.WriteLine(LogLevel.DEBUG,"Generating " + pathOutputFile);
// String companyName = PricingDA.GetNameForSymbol(symbol);
// File.Delete(pathOutputFile + ".xlsx");
// excelApp = new Microsoft.Office.Interop.Excel.Application();
// excelApp.ScreenUpdating = false;
// workbook = excelApp.Workbooks.Open(strPathTemplateFile, 0, false, 5, "", "", false, Microsoft.Office.Interop.Excel.XlPlatform.xlWindows, "", true, false, 0, true, false, false);
// worksheets = workbook.Worksheets;
// worksheet = (Microsoft.Office.Interop.Excel.Worksheet)worksheets.get_Item("Sheet1");
// Microsoft.Office.Interop.Excel.ChartObjects chartObjects = worksheet.ChartObjects();
// Microsoft.Office.Interop.Excel.ChartObject chartObject = chartObjects.Item(1);
// chartObject.Chart.ChartTitle.Text = companyName + " (" + symbol + ") " + Utility.DateTimeToStringMMHDDHYYYY(prices[prices.Count - 1].Date) + " Thru " + Utility.DateTimeToStringMMHDDHYYYY(price.Date)+" N="+periodN+", K="+periodK;
// Stochastics stochastics = StochasticsGenerator.GenerateStochastics(prices, periodN, periodK);
// if (null == stochastics)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,"Error generating stochastics.");
// return false;
// }
// Axis vertAxis = (Axis)chartObject.Chart.Axes(XlAxisType.xlValue, XlAxisGroup.xlPrimary);
// vertAxis.HasMajorGridlines = true;
// Axis horzAxis = (Axis)chartObject.Chart.Axes(XlAxisType.xlCategory, XlAxisGroup.xlPrimary);
// horzAxis.HasMajorGridlines = true;
// for (int index = 0; index < stochastics.Count; index++)
// {
// StochasticElement stochasticElement = stochastics[index];
// worksheet.Cells[index + 1 + rowOffset, 1] = stochasticElement.Date;
// worksheet.Cells[index + 1 + rowOffset, 2] = stochasticElement.Symbol;
// worksheet.Cells[index + 1 + rowOffset, 3] = stochasticElement.Open;
// worksheet.Cells[index + 1 + rowOffset, 4] = stochasticElement.High;
// worksheet.Cells[index + 1 + rowOffset, 5] = stochasticElement.Low;
// worksheet.Cells[index + 1 + rowOffset, 6] = stochasticElement.Close;
// worksheet.Cells[index + 1 + rowOffset, 7] = stochasticElement.LN;
// worksheet.Cells[index + 1 + rowOffset, 8] = stochasticElement.HN;
// worksheet.Cells[index + 1 + rowOffset, 9] = stochasticElement.HX;
// worksheet.Cells[index + 1 + rowOffset, 10] = stochasticElement.LX;
// worksheet.Cells[index + 1 + rowOffset, 11] = stochasticElement.PK;
// worksheet.Cells[index + 1 + rowOffset, 12] = stochasticElement.PD;
// }
// workbook.SaveAs(pathOutputFile, Microsoft.Office.Interop.Excel.XlFileFormat.xlOpenXMLWorkbook, Type.Missing, Type.Missing, false, false, Microsoft.Office.Interop.Excel.XlSaveAsAccessMode.xlNoChange, Microsoft.Office.Interop.Excel.XlSaveConflictResolution.xlUserResolution, true, Type.Missing, Type.Missing, Type.Missing);
// return true;
// }
// catch (Exception exception)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
// return false;
// }
// finally
// {
// if (null != worksheet) ReleaseObject(worksheet);
// if (null != worksheets) ReleaseObject(worksheets);
// if (null != workbook) ReleaseObject(workbook);
// excelApp.Quit();
// }
// }
// private static void ReleaseObject(object obj)
// {
// try
// {
// System.Runtime.InteropServices.Marshal.FinalReleaseComObject(obj);
// obj = null;
// }
// catch (Exception ex)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,ex.ToString());
// obj = null;
// }
// finally
// {
// GC.Collect();
// }
// }
// }
// }

87
MarketData/MarketDataLib/Helper/SwingTradeHelper.cs Executable file
View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Text;
using MarketData.MarketDataModel;
using MarketData.Utils;
namespace MarketData.Helper
{
public class SwingTradeHelper
{
private SwingTradeHelper()
{
}
public static SwingTrades FindSwingTrades(Prices prices,DMAPricesByDate ma21,BollingerBandElementsByDate bollingerBandElements=null)
{
SwingTrades swingTrades=new SwingTrades();
if (null == prices) return swingTrades;
prices.Sort(new PriceComparerDesc());
for (int index = 0; index < prices.Count; index++)
{
SwingTrade swingTrade = FindSwing(prices, index, ma21, bollingerBandElements);
if (null == swingTrade) continue;
swingTrades.Add(swingTrade);
}
return swingTrades;
}
private static SwingTrade FindSwing(Prices prices, int startIndex,DMAPricesByDate ma21,BollingerBandElementsByDate bollingerBandElements=null)
{
if (startIndex + 4 > prices.Count) return null;
DMAPrice maPrice = null;
Price p = prices[startIndex];
Price p1 = prices[startIndex + 1];
Price p2 = prices[startIndex + 2];
Price p3 = prices[startIndex + 3];
if(!ma21.ContainsKey(p.Date))return null;
maPrice = ma21[p.Date];
if (p1.Low >= maPrice.AVGPrice) return null;
if (!(p1.Low < p.Low && p2.Low < p1.Low && p3.Low > p2.Low)) return null;
SwingTrade swingTrade = new SwingTrade();
swingTrade.Symbol = p.Symbol;
DateGenerator dateGenerator = new DateGenerator();
if (null != bollingerBandElements && bollingerBandElements.ContainsKey(p1.Date) && bollingerBandElements.ContainsKey(p2.Date) && bollingerBandElements.ContainsKey(p3.Date))
{
BollingerBandElement b1=bollingerBandElements[p1.Date];
BollingerBandElement b2=bollingerBandElements[p2.Date];
BollingerBandElement b3 = bollingerBandElements[p3.Date];
if (p1.Low < b1.L && p2.Low < b2.L && p3.Low > b3.L)
{
swingTrade.SwingType = SwingTrade.TypeOfSwing.LongBand;
swingTrade.CandleDate = p1.Date;
swingTrade.CandleHigh = p1.High;
swingTrade.StopPrice = p1.Low;
swingTrade.CloseDate = p3.Date;
swingTrade.EntryDate = dateGenerator.FindNextBusinessDay(swingTrade.CloseDate);
return swingTrade;
}
}
DateTime? closeDate = FindCloseAboveCandle(prices, p1, startIndex + 4);
swingTrade.SwingType = SwingTrade.TypeOfSwing.Long;
swingTrade.CandleDate = p1.Date;
swingTrade.CandleHigh = p1.High;
swingTrade.StopPrice = p1.Low;
if (null != closeDate)
{
swingTrade.CloseDate = closeDate.Value;
swingTrade.EntryDate = dateGenerator.FindNextBusinessDay(swingTrade.CloseDate);
}
else
{
swingTrade.CloseDate = Utility.Epoch;
swingTrade.EntryDate = Utility.Epoch;
}
if (!swingTrade.EntryDate.Equals(Utility.Epoch) && dateGenerator.DaysBetween(swingTrade.CandleDate, swingTrade.EntryDate) > 15) return null;
return swingTrade;
}
private static DateTime? FindCloseAboveCandle(Prices prices, Price priceCandle, int startIndex)
{
if (startIndex > prices.Count) return null;
for (int index = startIndex; index < prices.Count; index++)
{
Price price = prices[index];
if (price.Close > priceCandle.High) return price.Date;
}
return null;
}
}
}

36
MarketData/MarketDataLib/Helper/ThreadHelper.cs Executable file
View File

@@ -0,0 +1,36 @@
using System;
using System.Threading;
using MarketData.Utils;
namespace MarketData.Helper
{
public enum ThreadHelperEnum{MaxThreads=25};
public class ThreadHelper
{
public String Symbol{get;set;}
public DateTime Modified{get;set;}
public ManualResetEvent ResetEvent{get;set;}
public UpdateManager updateManager;
public ThreadHelper()
{
Modified=DateTime.Now;
}
public ThreadHelper(String symbol, ManualResetEvent resetEvent)
{
Symbol=symbol;
ResetEvent=resetEvent;
}
public ThreadHelper(String symbol, DateTime modified,ManualResetEvent resetEvent)
{
Symbol=symbol;
Modified=modified;
ResetEvent=resetEvent;
}
public UpdateManager UpdateManager
{
get{return updateManager;}
set{updateManager=value;}
}
}
}

89
MarketData/MarketDataLib/Helper/TimeZoneHelper.cs Executable file
View File

@@ -0,0 +1,89 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MarketData.Utils;
using System.Collections.ObjectModel;
namespace MarketData.Helper
{
public class TimeZoneHelper
{
private DateGenerator dateGenerator=new DateGenerator();
private DateTime startDate;
private DateTime endDate;
private DateTime prevBusinessDate;
private DateTime easternTime;
private TimeZoneInfo localZone = TimeZoneInfo.Local;
private TimeZone currentTimeZone=TimeZone.CurrentTimeZone;
private static TimeZoneInfo easternTimeZone = GetEasternTimeZone();
public TimeZoneHelper()
{
startDate=DateTime.Now;
endDate = startDate; // the most recent date
prevBusinessDate=startDate;
prevBusinessDate = dateGenerator.GetPrevBusinessDay(endDate);
easternTime=new DateTime(prevBusinessDate.Year,prevBusinessDate.Month,prevBusinessDate.Day,20,30,0);
if(!currentTimeZone.DaylightName.Equals("Eastern Daylight Time")&&!currentTimeZone.DaylightName.Equals("Eastern Time (US & Canada)"))
{
easternTime=TimeZoneInfo.ConvertTimeToUtc(new DateTime(easternTime.Year,easternTime.Month,easternTime.Day,20,30,0),easternTimeZone);
}
if(prevBusinessDate.Equals(endDate)&&DateTime.Now.TimeOfDay<new TimeSpan(easternTime.Hour,easternTime.Minute,easternTime.Second))
{
prevBusinessDate=dateGenerator.FindPrevBusinessDay(prevBusinessDate);
}
}
public TimeZoneHelper(DateTime startDate)
{
this.startDate=startDate;
endDate = startDate; // the most recent date
prevBusinessDate=startDate;
prevBusinessDate = dateGenerator.GetPrevBusinessDay(endDate);
easternTime=new DateTime(prevBusinessDate.Year,prevBusinessDate.Month,prevBusinessDate.Day,20,30,0);
if(!currentTimeZone.DaylightName.Equals("Eastern Daylight Time")&&!currentTimeZone.DaylightName.Equals("Eastern Time (US & Canada)"))
{
easternTime = TimeZoneInfo.ConvertTimeToUtc(new DateTime(easternTime.Year, easternTime.Month, easternTime.Day, 20, 30, 0), easternTimeZone);
}
if(prevBusinessDate.Equals(endDate)&&DateTime.Now.TimeOfDay<new TimeSpan(easternTime.Hour,easternTime.Minute,easternTime.Second))
{
prevBusinessDate=dateGenerator.FindPrevBusinessDay(prevBusinessDate);
}
}
private static TimeZoneInfo GetEasternTimeZone()
{
TimeZoneInfo timeZoneInfo = null;
ReadOnlyCollection<TimeZoneInfo> timeZones;
timeZones= TimeZoneInfo.GetSystemTimeZones();
for (int index = 0; index < timeZones.Count; index++)
{
timeZoneInfo = timeZones[index];
if (timeZoneInfo.DisplayName.Equals("Eastern Daylight Time")||timeZoneInfo.DisplayName.Equals("Eastern Time (US & Canada)")||
timeZoneInfo.DaylightName.Equals("Eastern Daylight Time")||timeZoneInfo.StandardName.Equals("Eastern Standard Time")) break;
}
return timeZoneInfo;
}
public DateTime StartDate
{
get{return startDate;}
}
public DateTime EndDate
{
get{return endDate;}
}
public DateTime PrevBusinessDate
{
get{return prevBusinessDate;}
}
public DateTime EasternTime
{
get{return easternTime;}
}
public DateGenerator DateGenerator
{
get{return dateGenerator;}
}
}
}

114
MarketData/MarketDataLib/Helper/ZacksRankMarketDataHelper.cs Executable file
View File

@@ -0,0 +1,114 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;
namespace MarketData.Helper
{
public class ZacksRankMarketDataHelper : MarketDataHelperBase<String>
{
private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads;
public ZacksRankMarketDataHelper()
{
}
public bool LoadZacksRank(String symbol=null)
{
Profiler profiler=new Profiler();
try
{
if(symbol==null)Queue=PricingDA.GetSymbols();
else (Queue=new List<String>()).Add(symbol);
Index=-1;
ManualResetEvent[] resetEvents = new ManualResetEvent[MaxThreads];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)resetEvents[eventIndex] = new ManualResetEvent(true);
MDTrace.WriteLine(String.Format("Queuing Zacks rank load ..."));
DateTime modified=DateTime.Now;
while (true)
{
ManualResetEvent[] availableEvents=GetAvailableEvents(resetEvents);
ManualResetEvent[] busyEvents=GetBusyEvents(resetEvents);
if (null == PeekQueueItem() && 0==busyEvents.Length)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Zacks Rank queue contains {0} items, busy events {1}, all done.",0,busyEvents.Length));
break;
}
for (int index = 0; index < availableEvents.Length; index++)
{
symbol=GetQueueItem();
if (null != symbol)
{
availableEvents[index].Reset();
ThreadHelper zacksRankThreadHelper = new ThreadHelper(symbol,modified,availableEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallback, zacksRankThreadHelper);
}
else
{
busyEvents=GetBusyEvents(resetEvents);
if(busyEvents.Length!=availableEvents.Length)
{
ManualResetEvent[] resizedEvents=new ManualResetEvent[busyEvents.Length];
Array.Copy(busyEvents, resizedEvents,busyEvents.Length);
resetEvents = resizedEvents;
}
break;
}
} // for
MDTrace.WriteLine(LogLevel.DEBUG,"Zacks Rank waiting for free slots...");
if(resetEvents.Length>0)WaitHandle.WaitAny(resetEvents);
if(null==PeekQueueItem())resetEvents=ResizeEvents(resetEvents);
} // while
MDTrace.WriteLine(LogLevel.DEBUG,"Zacks Rank completed.");
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadZacksRanks]End, total took {0}(ms)",profiler.End()));
}
}
public void ThreadPoolCallback(Object threadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Zacks Rank, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
LoadZacksRankEx(threadHelper.Symbol,threadHelper.Modified);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Zacks Rank, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
}
finally
{
threadHelper.ResetEvent.Set();
}
}
// Single threaded method, use for debugging a single symbol fetch without going through the queue
public static void LoadZacksRankEx(String symbol,DateTime modified)
{
LoadZacksRankEx(symbol);
//if (!ZacksRankDA.CheckZacksRankModifiedOn(symbol, modified)) LoadZacksRankEx(symbol);
//else MDTrace.WriteLine(LogLevel.DEBUG,"Already have Zacks Rank for '"+symbol+"' on "+Utility.DateTimeToStringMMHDDHYYYY(modified));
}
public static void LoadZacksRankEx(String symbol)
{
try
{
MDTrace.WriteLine(LogLevel.DEBUG,"Loading Zacks Rank for '" + symbol + "'");
ZacksRank zacksRank=MarketDataHelper.GetZacksRank(symbol);
if (null != zacksRank)
{
ZacksRankDA.InsertZacksRank(zacksRank);
MDTrace.WriteLine(LogLevel.DEBUG,AnalystPriceTarget.Header);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Zacks rank:{0} {1}",zacksRank.Symbol,zacksRank.Rank));
}
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,"Failed to load Zacks Rank for '" + symbol + "'");
MDTrace.WriteLine(LogLevel.DEBUG,"Exception was " + exception.ToString());
}
}
}
}