250 lines
12 KiB
C#
250 lines
12 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using System.Threading;
|
|
using MarketData.MarketDataModel;
|
|
using MarketData.DataAccess;
|
|
using MarketData.Utils;
|
|
using System.IO;
|
|
|
|
namespace MarketData.Helper
|
|
{
|
|
public class FinancialStatementsMarketDataHelper : MarketDataHelperBase<String>
|
|
{
|
|
private static int MaxThreads = 10; // (int)ThreadHelperEnum.MaxThreads;
|
|
private static int PAUSE_BETWEEN_STATEMENTS=125;
|
|
|
|
public FinancialStatementsMarketDataHelper()
|
|
{
|
|
}
|
|
public bool LoadFinancialStatements(String symbol=null)
|
|
{
|
|
Profiler profiler=new Profiler();
|
|
try
|
|
{
|
|
if(symbol==null)Queue=PricingDA.GetSymbols();
|
|
else (Queue=new List<String>()).Add(symbol);
|
|
Index=-1;
|
|
ManualResetEvent[] resetEvents = new ManualResetEvent[MaxThreads];
|
|
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)resetEvents[eventIndex] = new ManualResetEvent(true);
|
|
MDTrace.WriteLine(String.Format("Queuing FinancialStatements ..."));
|
|
DateTime modified=DateTime.Now;
|
|
while (true)
|
|
{
|
|
ManualResetEvent[] availableEvents=GetAvailableEvents(resetEvents);
|
|
ManualResetEvent[] busyEvents=GetBusyEvents(resetEvents);
|
|
if (null == PeekQueueItem() && 0==busyEvents.Length)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("FinancialStatements queue contains {0} items, busy events {1}, all done.",0,busyEvents.Length));
|
|
break;
|
|
}
|
|
for (int index = 0; index < availableEvents.Length; index++)
|
|
{
|
|
symbol=GetQueueItem();
|
|
if (null != symbol)
|
|
{
|
|
availableEvents[index].Reset();
|
|
ThreadHelper threadHelper = new ThreadHelper(symbol,modified,availableEvents[index]);
|
|
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadFinancialStatements, threadHelper);
|
|
try{Thread.Sleep(250);}catch(Exception){;} // adding a short pause between requests
|
|
}
|
|
else
|
|
{
|
|
busyEvents=GetBusyEvents(resetEvents);
|
|
if(busyEvents.Length!=availableEvents.Length)
|
|
{
|
|
ManualResetEvent[] resizedEvents=new ManualResetEvent[busyEvents.Length];
|
|
Array.Copy(busyEvents, resizedEvents,busyEvents.Length);
|
|
resetEvents = resizedEvents;
|
|
}
|
|
break;
|
|
}
|
|
} // for
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"FinancialStatements waiting for free slots...");
|
|
if(resetEvents.Length>0)WaitHandle.WaitAny(resetEvents);
|
|
if(null==PeekQueueItem())resetEvents=ResizeEvents(resetEvents);
|
|
} // while
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"FinancialStatements completed.");
|
|
return true;
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadFinancialStatements]End, total took {0}(ms)",profiler.End()));
|
|
}
|
|
}
|
|
public void ThreadPoolCallbackLoadFinancialStatements(Object threadHelperContext)
|
|
{
|
|
ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Financial Statements, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
|
|
LoadFinancialStatementsEx(threadHelper.Symbol,threadHelper.Modified);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Financial Statements, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
|
|
}
|
|
finally
|
|
{
|
|
threadHelper.ResetEvent.Set();
|
|
}
|
|
}
|
|
// Single threaded method, use for debugging a single symbol fetch without going through the queue
|
|
public static void LoadFinancialStatementsEx(String symbol,DateTime modified)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving income statements for '" + symbol + "'");
|
|
// download available annual income statement data for whatever years are available
|
|
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
|
|
if (!IncomeStatementDA.CheckIncomeStatementModifiedOn(symbol,modified,IncomeStatement.PeriodType.Annual))
|
|
{
|
|
List<IncomeStatement> incomeStatements = MarketDataHelper.GetIncomeStatement(symbol,IncomeStatement.PeriodType.Annual);
|
|
if (null == incomeStatements || 0 == incomeStatements.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"No Annual income statement for " + symbol);
|
|
}
|
|
else
|
|
{
|
|
foreach(IncomeStatement incomeStatement in incomeStatements)
|
|
{
|
|
IncomeStatement previousIncomeStatement=IncomeStatementDA.GetIncomeStatement(incomeStatement.Symbol,incomeStatement.AsOf,incomeStatement.Period);
|
|
incomeStatement.MergeFrom(previousIncomeStatement);
|
|
}
|
|
IncomeStatementDA.InsertIncomeStatements(incomeStatements);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,IncomeStatement.Heading);
|
|
for (int pindex = 0; pindex < incomeStatements.Count; pindex++)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,incomeStatements[pindex].ToString());
|
|
}
|
|
}
|
|
}
|
|
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded annual income statement for '"+symbol+"' on "+Utility.DateTimeToStringMMHDDHYYYY(modified));
|
|
// download quarterly income statement data for whatever quarters are available
|
|
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
|
|
if (!IncomeStatementDA.CheckIncomeStatementModifiedOn(symbol, modified, IncomeStatement.PeriodType.Quarterly))
|
|
{
|
|
List<IncomeStatement> incomeStatements = MarketDataHelper.GetIncomeStatement(symbol, IncomeStatement.PeriodType.Quarterly);
|
|
if (null == incomeStatements || 0 == incomeStatements.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"No Quarterly income statement for " + symbol);
|
|
}
|
|
else
|
|
{
|
|
foreach(IncomeStatement incomeStatement in incomeStatements)
|
|
{
|
|
IncomeStatement previousIncomeStatement=IncomeStatementDA.GetIncomeStatement(incomeStatement.Symbol,incomeStatement.AsOf,incomeStatement.Period);
|
|
incomeStatement.MergeFrom(previousIncomeStatement);
|
|
}
|
|
IncomeStatementDA.InsertIncomeStatements(incomeStatements);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,IncomeStatement.Heading);
|
|
for (int pindex = 0; pindex < incomeStatements.Count; pindex++)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,incomeStatements[pindex].ToString());
|
|
}
|
|
}
|
|
}
|
|
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded quarterly income statement for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving balancesheet for '" + symbol + "'");
|
|
// download annual balance sheet data for whatever years are available
|
|
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
|
|
if (!BalanceSheetDA.CheckBalanceSheetModifiedOn(symbol, modified,BalanceSheet.PeriodType.Annual))
|
|
{
|
|
List<BalanceSheet> balanceSheets = MarketDataHelper.GetBalanceSheet(symbol,BalanceSheet.PeriodType.Annual);
|
|
if (null == balanceSheets || 0 == balanceSheets.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"No Annual balance sheet for " + symbol);
|
|
}
|
|
else
|
|
{
|
|
foreach(BalanceSheet balanceSheet in balanceSheets)
|
|
{
|
|
BalanceSheet previousBalanceSheet=BalanceSheetDA.GetBalanceSheet(balanceSheet.Symbol,balanceSheet.AsOf,balanceSheet.Period);
|
|
balanceSheet.MergeFrom(previousBalanceSheet);
|
|
}
|
|
BalanceSheetDA.InsertBalanceSheets(balanceSheets);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,BalanceSheet.Heading);
|
|
for (int pindex = 0; pindex < balanceSheets.Count; pindex++)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,balanceSheets[pindex].ToString());
|
|
}
|
|
}
|
|
}
|
|
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded annual balance sheet for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
|
|
// download quarterly balance sheet data for whatever quarters are available
|
|
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
|
|
if (!BalanceSheetDA.CheckBalanceSheetModifiedOn(symbol, modified, BalanceSheet.PeriodType.Quarterly))
|
|
{
|
|
List<BalanceSheet> balanceSheets = MarketDataHelper.GetBalanceSheet(symbol,BalanceSheet.PeriodType.Quarterly);
|
|
if (null == balanceSheets || 0 == balanceSheets.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"No Quarterly balance sheet for " + symbol);
|
|
}
|
|
else
|
|
{
|
|
foreach(BalanceSheet balanceSheet in balanceSheets)
|
|
{
|
|
BalanceSheet previousBalanceSheet=BalanceSheetDA.GetBalanceSheet(balanceSheet.Symbol,balanceSheet.AsOf,balanceSheet.Period);
|
|
balanceSheet.MergeFrom(previousBalanceSheet);
|
|
}
|
|
BalanceSheetDA.InsertBalanceSheets(balanceSheets);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,BalanceSheet.Heading);
|
|
for (int pindex = 0; pindex < balanceSheets.Count; pindex++)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,balanceSheets[pindex].ToString());
|
|
}
|
|
}
|
|
}
|
|
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded quarterly balance sheet for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving cashflow statements for '" + symbol + "'");
|
|
// download annual cashflow data for whatever years are available
|
|
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
|
|
if (!CashflowStatementDA.CheckCashflowStatementModifiedOn(symbol, modified,CashflowStatement.PeriodType.Annual))
|
|
{
|
|
List<CashflowStatement> cashflowStatements = MarketDataHelper.GetCashflowStatement(symbol,CashflowStatement.PeriodType.Annual);
|
|
if (null == cashflowStatements || 0 == cashflowStatements.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"No Annual cashflow statement for " + symbol);
|
|
}
|
|
else
|
|
{
|
|
foreach(CashflowStatement cashflowStatement in cashflowStatements)
|
|
{
|
|
CashflowStatement previousCashflowStatement=CashflowStatementDA.GetCashflowStatement(cashflowStatement.Symbol,cashflowStatement.AsOf,cashflowStatement.Period);
|
|
cashflowStatement.MergeFrom(previousCashflowStatement);
|
|
}
|
|
CashflowStatementDA.InsertCashflowStatement(cashflowStatements);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,CashflowStatement.Heading);
|
|
for (int pindex = 0; pindex < cashflowStatements.Count; pindex++)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,cashflowStatements[pindex].ToString());
|
|
}
|
|
}
|
|
}
|
|
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded annual cashflow statement for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
|
|
// download quarterly cashflow statement for whatever quarters are available
|
|
try{Thread.Sleep(PAUSE_BETWEEN_STATEMENTS);}catch(Exception){;}
|
|
if (!CashflowStatementDA.CheckCashflowStatementModifiedOn(symbol, modified, CashflowStatement.PeriodType.Quarterly))
|
|
{
|
|
List<CashflowStatement> cashflowStatements = MarketDataHelper.GetCashflowStatement(symbol,CashflowStatement.PeriodType.Quarterly);
|
|
if (null == cashflowStatements || 0 == cashflowStatements.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"No Quarterly cashflow statement for " + symbol);
|
|
}
|
|
else
|
|
{
|
|
foreach(CashflowStatement cashflowStatement in cashflowStatements)
|
|
{
|
|
CashflowStatement previousCashflowStatement=CashflowStatementDA.GetCashflowStatement(cashflowStatement.Symbol,cashflowStatement.AsOf,cashflowStatement.Period);
|
|
cashflowStatement.MergeFrom(previousCashflowStatement);
|
|
}
|
|
CashflowStatementDA.InsertCashflowStatement(cashflowStatements);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,CashflowStatement.Heading);
|
|
for (int pindex = 0; pindex < cashflowStatements.Count; pindex++)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,cashflowStatements[pindex].ToString());
|
|
}
|
|
}
|
|
}
|
|
else MDTrace.WriteLine(LogLevel.DEBUG,"Already downloaded quarterly cashflow statement for '" + symbol + "' on " + Utility.DateTimeToStringMMHDDHYYYY(modified));
|
|
}
|
|
}
|
|
}
|
|
|