Initial Commit
This commit is contained in:
249
MarketData/MarketDataLib/Helper/FinancialStatementsMarketDataHelper.cs
Executable file
249
MarketData/MarketDataLib/Helper/FinancialStatementsMarketDataHelper.cs
Executable file
@@ -0,0 +1,249 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using MarketData.MarketDataModel;
|
||||
using MarketData.DataAccess;
|
||||
using MarketData.Utils;
|
||||
using System.IO;
|
||||
|
||||
namespace MarketData.Helper
|
||||
{
|
||||
public class FinancialStatementsMarketDataHelper : MarketDataHelperBase<String>
|
||||
{
|
||||
private static int MaxThreads = 10; // (int)ThreadHelperEnum.MaxThreads;
|
||||
private static int PAUSE_BETWEEN_STATEMENTS=125;
|
||||
|
||||
public FinancialStatementsMarketDataHelper()
|
||||
{
|
||||
}
|
||||
public bool LoadFinancialStatements(String symbol=null)
|
||||
{
|
||||
Profiler profiler=new Profiler();
|
||||
try
|
||||
{
|
||||
if(symbol==null)Queue=PricingDA.GetSymbols();
|
||||
else (Queue=new List<String>()).Add(symbol);
|
||||
Index=-1;
|
||||
ManualResetEvent[] resetEvents = new ManualResetEvent[MaxThreads];
|
||||
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)resetEvents[eventIndex] = new ManualResetEvent(true);
|
||||
MDTrace.WriteLine(String.Format("Queuing FinancialStatements ..."));
|
||||
DateTime modified=DateTime.Now;
|
||||
while (true)
|
||||
{
|
||||
ManualResetEvent[] availableEvents=GetAvailableEvents(resetEvents);
|
||||
ManualResetEvent[] busyEvents=GetBusyEvents(resetEvents);
|
||||
if (null == PeekQueueItem() && 0==busyEvents.Length)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("FinancialStatements queue contains {0} items, busy events {1}, all done.",0,busyEvents.Length));
|
||||
break;
|
||||
}
|
||||
for (int index = 0; index < availableEvents.Length; index++)
|
||||
{
|
||||
symbol=GetQueueItem();
|
||||
if (null != symbol)
|
||||
{
|
||||
availableEvents[index].Reset();
|
||||
ThreadHelper threadHelper = new ThreadHelper(symbol,modified,availableEvents[index]);
|
||||
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadFinancialStatements, threadHelper);
|
||||
try{Thread.Sleep(250);}catch(Exception){;} // adding a short pause between requests
|
||||
}
|
||||
else
|
||||
{
|
||||
busyEvents=GetBusyEvents(resetEvents);
|
||||
if(busyEvents.Length!=availableEvents.Length)
|
||||
{
|
||||
ManualResetEvent[] resizedEvents=new ManualResetEvent[busyEvents.Length];
|
||||
Array.Copy(busyEvents, resizedEvents,busyEvents.Length);
|
||||
resetEvents = resizedEvents;
|
||||
}
|
||||
break;
|
||||
}
|
||||
} // for
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,"FinancialStatements waiting for free slots...");
|
||||
if(resetEvents.Length>0)WaitHandle.WaitAny(resetEvents);
|
||||
if(null==PeekQueueItem())resetEvents=ResizeEvents(resetEvents);
|
||||
} // while
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,"FinancialStatements completed.");
|
||||
return true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadFinancialStatements]End, total took {0}(ms)",profiler.End()));
|
||||
}
|
||||
}
|
||||
public void ThreadPoolCallbackLoadFinancialStatements(Object threadHelperContext)
|
||||
{
|
||||
ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
|
||||
try
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Financial Statements, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
|
||||
LoadFinancialStatementsEx(threadHelper.Symbol,threadHelper.Modified);
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Financial Statements, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
|
||||
}
|
||||
finally
|
||||
{
|
||||
threadHelper.ResetEvent.Set();
|
||||
}
|
||||
}
|
||||
// Single threaded method, use for debugging a single symbol fetch without going through the queue
|
||||
public static void LoadFinancialStatementsEx(String symbol,DateTime modified)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving income statements for '" + symbol + "'");
|
||||
// download available annual income statement data for whatever years are available
|
||||
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
|
||||
if (!IncomeStatementDA.CheckIncomeStatementModifiedOn(symbol,modified,IncomeStatement.PeriodType.Annual))
|
||||
{
|
||||
List<IncomeStatement> incomeStatements = MarketDataHelper.GetIncomeStatement(symbol,IncomeStatement.PeriodType.Annual);
|
||||
if (null == incomeStatements || 0 == incomeStatements.Count)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,"No Annual income statement for " + symbol);
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach(IncomeStatement incomeStatement in incomeStatements)
|
||||
{
|
||||
IncomeStatement previousIncomeStatement=IncomeStatementDA.GetIncomeStatement(incomeStatement.Symbol,incomeStatement.AsOf,incomeStatement.Period);
|
||||
incomeStatement.MergeFrom(previousIncomeStatement);
|
||||
}
|
||||
IncomeStatementDA.InsertIncomeStatements(incomeStatements);
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,IncomeStatement.Heading);
|
||||
for (int pindex = 0; pindex < incomeStatements.Count; pindex++)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,incomeStatements[pindex].ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded annual income statement for '"+symbol+"' on "+Utility.DateTimeToStringMMHDDHYYYY(modified));
|
||||
// download quarterly income statement data for whatever quarters are available
|
||||
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
|
||||
if (!IncomeStatementDA.CheckIncomeStatementModifiedOn(symbol, modified, IncomeStatement.PeriodType.Quarterly))
|
||||
{
|
||||
List<IncomeStatement> incomeStatements = MarketDataHelper.GetIncomeStatement(symbol, IncomeStatement.PeriodType.Quarterly);
|
||||
if (null == incomeStatements || 0 == incomeStatements.Count)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,"No Quarterly income statement for " + symbol);
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach(IncomeStatement incomeStatement in incomeStatements)
|
||||
{
|
||||
IncomeStatement previousIncomeStatement=IncomeStatementDA.GetIncomeStatement(incomeStatement.Symbol,incomeStatement.AsOf,incomeStatement.Period);
|
||||
incomeStatement.MergeFrom(previousIncomeStatement);
|
||||
}
|
||||
IncomeStatementDA.InsertIncomeStatements(incomeStatements);
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,IncomeStatement.Heading);
|
||||
for (int pindex = 0; pindex < incomeStatements.Count; pindex++)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,incomeStatements[pindex].ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded quarterly income statement for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving balancesheet for '" + symbol + "'");
|
||||
// download annual balance sheet data for whatever years are available
|
||||
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
|
||||
if (!BalanceSheetDA.CheckBalanceSheetModifiedOn(symbol, modified,BalanceSheet.PeriodType.Annual))
|
||||
{
|
||||
List<BalanceSheet> balanceSheets = MarketDataHelper.GetBalanceSheet(symbol,BalanceSheet.PeriodType.Annual);
|
||||
if (null == balanceSheets || 0 == balanceSheets.Count)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,"No Annual balance sheet for " + symbol);
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach(BalanceSheet balanceSheet in balanceSheets)
|
||||
{
|
||||
BalanceSheet previousBalanceSheet=BalanceSheetDA.GetBalanceSheet(balanceSheet.Symbol,balanceSheet.AsOf,balanceSheet.Period);
|
||||
balanceSheet.MergeFrom(previousBalanceSheet);
|
||||
}
|
||||
BalanceSheetDA.InsertBalanceSheets(balanceSheets);
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,BalanceSheet.Heading);
|
||||
for (int pindex = 0; pindex < balanceSheets.Count; pindex++)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,balanceSheets[pindex].ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded annual balance sheet for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
|
||||
// download quarterly balance sheet data for whatever quarters are available
|
||||
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
|
||||
if (!BalanceSheetDA.CheckBalanceSheetModifiedOn(symbol, modified, BalanceSheet.PeriodType.Quarterly))
|
||||
{
|
||||
List<BalanceSheet> balanceSheets = MarketDataHelper.GetBalanceSheet(symbol,BalanceSheet.PeriodType.Quarterly);
|
||||
if (null == balanceSheets || 0 == balanceSheets.Count)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,"No Quarterly balance sheet for " + symbol);
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach(BalanceSheet balanceSheet in balanceSheets)
|
||||
{
|
||||
BalanceSheet previousBalanceSheet=BalanceSheetDA.GetBalanceSheet(balanceSheet.Symbol,balanceSheet.AsOf,balanceSheet.Period);
|
||||
balanceSheet.MergeFrom(previousBalanceSheet);
|
||||
}
|
||||
BalanceSheetDA.InsertBalanceSheets(balanceSheets);
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,BalanceSheet.Heading);
|
||||
for (int pindex = 0; pindex < balanceSheets.Count; pindex++)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,balanceSheets[pindex].ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded quarterly balance sheet for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving cashflow statements for '" + symbol + "'");
|
||||
// download annual cashflow data for whatever years are available
|
||||
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
|
||||
if (!CashflowStatementDA.CheckCashflowStatementModifiedOn(symbol, modified,CashflowStatement.PeriodType.Annual))
|
||||
{
|
||||
List<CashflowStatement> cashflowStatements = MarketDataHelper.GetCashflowStatement(symbol,CashflowStatement.PeriodType.Annual);
|
||||
if (null == cashflowStatements || 0 == cashflowStatements.Count)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,"No Annual cashflow statement for " + symbol);
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach(CashflowStatement cashflowStatement in cashflowStatements)
|
||||
{
|
||||
CashflowStatement previousCashflowStatement=CashflowStatementDA.GetCashflowStatement(cashflowStatement.Symbol,cashflowStatement.AsOf,cashflowStatement.Period);
|
||||
cashflowStatement.MergeFrom(previousCashflowStatement);
|
||||
}
|
||||
CashflowStatementDA.InsertCashflowStatement(cashflowStatements);
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,CashflowStatement.Heading);
|
||||
for (int pindex = 0; pindex < cashflowStatements.Count; pindex++)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,cashflowStatements[pindex].ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded annual cashflow statement for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
|
||||
// download quarterly cashflow statement for whatever quarters are available
|
||||
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
|
||||
if (!CashflowStatementDA.CheckCashflowStatementModifiedOn(symbol, modified, CashflowStatement.PeriodType.Quarterly))
|
||||
{
|
||||
List<CashflowStatement> cashflowStatements = MarketDataHelper.GetCashflowStatement(symbol,CashflowStatement.PeriodType.Quarterly);
|
||||
if (null == cashflowStatements || 0 == cashflowStatements.Count)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,"No Quarterly cashflow statement for " + symbol);
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach(CashflowStatement cashflowStatement in cashflowStatements)
|
||||
{
|
||||
CashflowStatement previousCashflowStatement=CashflowStatementDA.GetCashflowStatement(cashflowStatement.Symbol,cashflowStatement.AsOf,cashflowStatement.Period);
|
||||
cashflowStatement.MergeFrom(previousCashflowStatement);
|
||||
}
|
||||
CashflowStatementDA.InsertCashflowStatement(cashflowStatements);
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,CashflowStatement.Heading);
|
||||
for (int pindex = 0; pindex < cashflowStatements.Count; pindex++)
|
||||
{
|
||||
MDTrace.WriteLine(LogLevel.DEBUG,cashflowStatements[pindex].ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded quarterly cashflow statement for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user