1274 lines
53 KiB
C#
Executable File
1274 lines
53 KiB
C#
Executable File
using System.Diagnostics;
|
|
using MarketData.DataAccess;
|
|
using MarketData.Interface;
|
|
using MarketData.Utils;
|
|
using MarketData.Configuration;
|
|
using Microsoft.Extensions.Configuration;
|
|
using MarketData.MarketDataModel;
|
|
using MarketData.Helper;
|
|
using MarketData.Integration;
|
|
using MarketData.Cache;
|
|
using MarketData.Generator;
|
|
using MarketData.ModelHelper;
|
|
using MarketData.Numerical;
|
|
|
|
namespace MarketData.Services
|
|
{
|
|
public class MainService : IMainService
|
|
{
|
|
private Dictionary<String, Func<CommandArgs,Task>> tasks = new Dictionary<String,Func<CommandArgs,Task>>();
|
|
|
|
/// <summary>
|
|
/// DisplayUsage
|
|
/// </summary>
|
|
public static void DisplayUsage()
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"USAGE");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"LOADHEADLINESWATCHLIST");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"LOADPREMARKETDATA");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEDAILY2 /DATE:");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTPRICEOPENPOSITIONS");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTPRICEWATCHLIST /WATCHLIST:");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEINTRADAYPRICING:");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTANALYSTRATINGS");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEANALYSTRATINGS");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATESECFILINGSWATCHLIST /WATCHLIST:");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATECOMPANYPROFILES");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEGDPPERCAPITA");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEETFHOLDINGS");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEFINANCIALSTATEMENTS");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEFUNDAMENTALS");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEHISTORICAL");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"CALCSTICKER /WAITFORCOMPLETION:{argument1,argument2,...} For example CALCSTICKER /WAITFORCOMPLETION:UPDATEFINANCIALSTATEMENTS,UPDATEFUNDAMENTALS,UPDATEHISTORICAL");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"MGSHSESSION /SESSIONFILE:");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"MGSHRUNBACKTEST /USEHEDGING: /HEDGEINITIALCASH: /USESTOPLIMITS: /KEEPSLOTPOSITIONS: /STARTDATE: /MAXPOSITIONS: /INITIALCASH: /HOLDINGPERIOD: /{ENDDATE}: /SESSIONFILE: ");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"MGSHRUNDAILY /SESSIONFILE: /TRADEDATE:");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"RUNCMTREND /MODE:DAILY|BACKTEST|RUNTRENDTEMPLATE|ENTRYTEST /SYMBOL:{for mode ANALYZE,ENTRYTEST} /TRADEDATE:{for mode DAILY,RUNTRENDTEMPLATE) /STARTDATE:(for mode BACKTEST,ENTRYTEST) /ENDDATE:(for mode BACKTEST) /INITIALCASH: /SESSIONFILE: /MAXOPENPOSITIONS: /MAXDAILYPOSITIONS: Runs Mark Minervini trend");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"CMTSESSION /SESSIONFILE:{pathfilename} Runs Mark Minervini trend display session");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"RUNCMBACKTEST /STARTDATE: /MAXPOSITIONS: /INITIALCASH: /HOLDINGPERIOD: /{USEBINBASEDPOSITIONSIZING}: /{USEBINBASEDPOSITIONSIZINGNUMBINS}: /{TARGETBETA}: /{ENDDATE}: /SESSIONFILE: /{USECNN}: /{USECNNHOST}: /{USECNNDAYCOUNT}:");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"CMSESSION /SESSIONFILE:");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"CMCANDIDATELASTRESORT /TRsADEDATE:");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,@"CMGAINLOSS /SESSIONFILE:{PATHSESSIONFILE} (i.e.) CMGAINLOSS /SESSIONFILE:C:\boneyard\marketdata\bin\Debug\saferun\CM20191031.txt");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"MGSESSION /SESSIONFILE:");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,@"MGGAINLOSS /SESSIONFILE:{PATHSESSIONFILE} (i.e.) MGGAINLOSS /SESSIONFILE:C:\boneyard\marketdata\bin\Debug\saferun\MG20180131.txt");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"RUNBACKTEST /STARTDATE: /MAXPOSITIONS: /INITIALCASH: /HOLDINGPERIOD: /{ENDDATE}: /{SESSIONFILE}:");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"CALCBETAS recalculates beta36, beta24, and bet06 for Fundamentals");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"OPTIMIZEDB optimizes the database tables");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"ECHO {param1} {param2} {param(n)");
|
|
}
|
|
|
|
/// <summary>
|
|
/// This is the main entry point
|
|
/// </summary>
|
|
/// <param name="args"></param>
|
|
/// <param name="configuration"></param>
|
|
public void RunService(String[] args,IConfiguration configuration)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
DateTime currentDate=DateTime.Now;
|
|
|
|
tasks.Add("LOADHEADLINESWATCHLIST",TaskLoadHeadlinesWatchList);
|
|
tasks.Add("LOADPREMARKETDATA",TaskLoadPremarketData);
|
|
tasks.Add("UPDATEDAILY2",TaskUpdateDaily2);
|
|
tasks.Add("UPDATELATESTPRICEOPENPOSITIONS",TaskUpdateLatestPriceOpenPositions);
|
|
tasks.Add("UPDATELATESTPRICEWATCHLIST",TaskUpdateLatestPriceWatchList);
|
|
tasks.Add("UPDATEINTRADAYPRICING",TaskUpdateIntradayPricing);
|
|
tasks.Add("UPDATELATESTANALYSTRATINGS",TaskUpdateLatestAnalystRatings);
|
|
tasks.Add("UPDATEANALYSTRATINGS",TaskUpdateAnalystRatings);
|
|
tasks.Add("UPDATESECFILINGSWATCHLIST",TaskUpdateSECFilingsWatchList);
|
|
tasks.Add("UPDATECOMPANYPROFILES",TaskUpdateCompanyProfiles);
|
|
tasks.Add("UPDATEGDPPERCAPITA",TaskUpdateGDPPerCapita);
|
|
tasks.Add("UPDATEETFHOLDINGS",TaskUpdateETFHoldings);
|
|
tasks.Add("UPDATEFINANCIALSTATEMENTS",TaskUpdateFinancialStatements);
|
|
tasks.Add("UPDATEFUNDAMENTALS",TaskUpdateFundamentals);
|
|
tasks.Add("UPDATEHISTORICAL",TaskUpdateHistorical);
|
|
tasks.Add("CALCSTICKER",TaskCalcSticker);
|
|
tasks.Add("MGSHSESSION",TaskMGSHSession);
|
|
tasks.Add("MGSHRUNBACKTEST",TaskMGSHRunBacktest);
|
|
tasks.Add("MGSHRUNDAILY",TaskMGSHRunDaily);
|
|
tasks.Add("RUNCMTREND",TaskRunCMTrend);
|
|
tasks.Add("CMTSESSION",TaskCMTSession);
|
|
tasks.Add("RUNCMBACKTEST",TaskCMMRunCMBacktest);
|
|
tasks.Add("CMSESSION",TaskCMMRunCMSession);
|
|
tasks.Add("CMCANDIDATELASTRESORT",TaskCMMRunCMCandidateLastResort);
|
|
tasks.Add("CMGAINLOSS",TaskCMMRunCMGainLoss);
|
|
tasks.Add("MGSESSION",TaskMGRunMGSession);
|
|
tasks.Add("MGGAINLOSS",TaskMGRunMGGainLoss);
|
|
tasks.Add("RUNBACKTEST",TaskMGRunMGBacktest);
|
|
tasks.Add("CALCBETAS",TaskCalcBetas);
|
|
tasks.Add("OPTIMIZEDB",TaskOptimizeDatabase);
|
|
tasks.Add("ECHO",TaskEcho);
|
|
GlobalConfig.Instance.Configuration = configuration; // This call sets up configuration stuff so it needs to be first.
|
|
|
|
if (args.Length < 1 || String.IsNullOrEmpty(args[0]))
|
|
{
|
|
DisplayUsage();
|
|
return;
|
|
}
|
|
|
|
string arg = args[0].ToUpper();
|
|
|
|
// log files are now of the form market_data+task.log. Also log files will expire when > 1 days old
|
|
if(!CreateLogging(arg))
|
|
{
|
|
Console.WriteLine("CreateLogging returned false.");
|
|
return;
|
|
}
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"[RunService] Started @ {Utility.DateTimeToStringYYYYHMMHDDHHMMSSTT(currentDate)}");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"[RunService] Argument {arg}");
|
|
|
|
DateTime maxHolidayDate =HolidayDA.GetMaxHolidayDate();
|
|
if(currentDate>maxHolidayDate)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("There are no holidays defined in the system. Add holidays for year {0} into marketholidays table",currentDate.Year));
|
|
return;
|
|
}
|
|
|
|
if(!tasks.ContainsKey(arg))
|
|
{
|
|
DisplayUsage();
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
tasks[arg](new CommandArgs(args));
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return;
|
|
}
|
|
finally
|
|
{
|
|
LocalPriceCache.GetInstance().Dispose();
|
|
GBPriceCache.GetInstance().Dispose();
|
|
}
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"[RunService] Done, total took {profiler.End()}(ms)");
|
|
}
|
|
// **********************************************************************************************************************************************************
|
|
// ********************************************************* T A S K S *************************************************************************************
|
|
// **********************************************************************************************************************************************************
|
|
/// <summary>
|
|
/// Performs the CalcSticker (Valuation) run.
|
|
/// There is no need for an initial pause because the crontab job is configured to start this run 15 minutes after the monthly feeds begin
|
|
/// 0 0 15 * * cd $CRON_DIR ; /opt/MarketData/MarketData/mk UPDATEFINANCIALSTATEMENTS > /dev/null 2>&1
|
|
/// 0 0 15 * * cd $CRON_DIR ; /opt/MarketData/MarketData/mk UPDATEFUNDAMENTALS > /dev/null 2>&1
|
|
/// 0 0 15 * * cd $CRON_DIR ; /opt/MarketData/MarketData/mk UPDATEHISTORICAL > /dev/null 2>&1
|
|
///15 0 15 * * cd $CRON_DIR ; /opt/MarketData/MarketData/mk CALCSTICKER /WAITFORCOMPLETION:PDATEFINANCIALSTATEMENTS,UPDATEFUNDAMENTALS,UPDATEHISTORICAL,
|
|
/// </summary>
|
|
/// <param name="commandArgs"></param>
|
|
/// <returns></returns>
|
|
public async Task TaskCalcSticker(CommandArgs commandArgs)
|
|
{
|
|
int WAIT_TIME_INTERVAL = 60000; // 1 minute intervals between checks.
|
|
if(commandArgs.Has("WAITFORCOMPLETION"))
|
|
{
|
|
String waitForCompletion = commandArgs.Get<String>("WAITFORCOMPLETION");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"/WAITFORCOMPLETION:{waitForCompletion}");
|
|
List<String> processArguments = Utility.ToList(waitForCompletion,',');
|
|
while(Utility.IsProcessRunning("mk",processArguments))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"CALCSTICKER is waiting for items in the process completion list {commandArgs.Get<String>("WAITFORCOMPLETION")}");
|
|
try{Thread.Sleep(WAIT_TIME_INTERVAL);}catch(Exception){;}
|
|
}
|
|
}
|
|
CalcSticker();
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskUpdateHistorical(CommandArgs commandArgs)
|
|
{
|
|
UpdateHistorical();
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
|
|
public async Task TaskUpdateFundamentals(CommandArgs commandArgs)
|
|
{
|
|
UpdateFundamentalsFinViz();
|
|
UpdateFundamentalsFinViz();
|
|
UpdateFundamentalsFinViz();
|
|
UpdateFundamentals();
|
|
UpdateFundamentals();
|
|
UpdateFundamentals();
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskUpdateFinancialStatements(CommandArgs commandArgs)
|
|
{
|
|
UpdateFinancialStatements();
|
|
UpdateFinancialStatements();
|
|
UpdateFinancialStatements();
|
|
UpdateFinancialStatements();
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskUpdateETFHoldings(CommandArgs commandArgs)
|
|
{
|
|
GetETFHoldings();
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskUpdateGDPPerCapita(CommandArgs commandArgs)
|
|
{
|
|
LoadGDPPerCapita();
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskUpdateCompanyProfiles(CommandArgs commandArgs)
|
|
{
|
|
UpdateCompanyProfiles();
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskUpdateSECFilingsWatchList(CommandArgs commandArgs)
|
|
{
|
|
if(!commandArgs.Has("WATCHLIST")){MDTrace.WriteLine(LogLevel.DEBUG,"UPDATESECFILINGSWATCHLIST REQUIRES WATCHLIST");return;}
|
|
String watchListName = commandArgs.Coalesce<String>("WATCHLIST");
|
|
List<String> symbols = WatchListDA.GetWatchList(watchListName);
|
|
SECFilingMarketDataHelper secFilingMarketDataHelper=new SECFilingMarketDataHelper();
|
|
secFilingMarketDataHelper.UpdateSECFilings(symbols);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskLoadHeadlinesWatchList(CommandArgs commandArgs)
|
|
{
|
|
if(!commandArgs.Has("WATCHLIST")){MDTrace.WriteLine(LogLevel.DEBUG,"LOADHEADLINESWATCHLIST REQUIRES WATCHLIST");return;}
|
|
else LoadHeadlinesWatchList(commandArgs.Coalesce<String>("WATCHLIST"));
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskLoadPremarketData(CommandArgs commandArgs)
|
|
{
|
|
LoadPremarketData();
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskUpdateDaily2(CommandArgs commandArgs)
|
|
{
|
|
if(!commandArgs.Has("DATE")){MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEDAILY2 MISSING DATE");return;}
|
|
UpdateDaily2(commandArgs.Coalesce<DateTime>("DATE"));
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskUpdateLatestPriceOpenPositions(CommandArgs commandArgs)
|
|
{
|
|
UpdateLatestPriceOpenPositions();
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskUpdateLatestPriceWatchList(CommandArgs commandArgs)
|
|
{
|
|
if(!commandArgs.Has("WATCHLIST")){MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTPRICEWATCHLIST MISSING WATCHLIST");return;}
|
|
UpdateLatestPriceWatchList(commandArgs.Coalesce<String>("WATCHLIST"));
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskUpdateIntradayPricing(CommandArgs commandArgs)
|
|
{
|
|
try
|
|
{
|
|
if(!CheckRunCriteria())return;
|
|
String watchListName="Valuations";
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"UPDATEINTRADAYPRICING");
|
|
List<String> watchlistSymbols = WatchListDA.GetWatchList(watchListName);
|
|
PortfolioTrades portfolioTrades=PortfolioDA.GetOpenTrades();
|
|
List<String> portfolioSymbols=(from PortfolioTrade portfolioTrade in portfolioTrades select portfolioTrade.Symbol).Distinct().ToList();
|
|
if(portfolioSymbols.Any(x=>x.Equals("SPY")))portfolioSymbols.Add("SH");
|
|
List<String> symbols = new List<String>();
|
|
symbols.AddRange(portfolioSymbols);
|
|
symbols.AddRange(watchlistSymbols);
|
|
symbols = symbols.Distinct().ToList();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Updating {symbols.Count} prices. Includes {portfolioSymbols.Count} open positions and {watchlistSymbols.Count} Valuation watchlist.");
|
|
PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper();
|
|
pricingMarketDataHelper.UpdateLatestPrices(symbols);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
}
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskUpdateLatestAnalystRatings(CommandArgs commandArgs)
|
|
{
|
|
if (commandArgs.Has("UPDATESECURITYMASTER")) UpdateLatestAnalystRatings(commandArgs.Coalesce<Boolean>("UPDATESECURITYMASTER"));
|
|
else UpdateLatestAnalystRatings();
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskUpdateAnalystRatings(CommandArgs commandArgs)
|
|
{
|
|
if (commandArgs.Has("SYMBOL") && commandArgs.Has("MINDATE")) UpdateAnalystRatings(commandArgs.Coalesce<String>("SYMBOL"), commandArgs.Coalesce<DateTime>("MINDATE"));
|
|
else if (commandArgs.Has("SYMBOL") ) UpdateAnalystRatings(commandArgs.Coalesce<String>("SYMBOL"));
|
|
else if (commandArgs.Has("MINDATE")) UpdateAnalystRatings(null,commandArgs.Coalesce<DateTime>("MINDATE"));
|
|
else UpdateAnalystRatings();
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskEcho(CommandArgs commandArgs)
|
|
{
|
|
String[] args = commandArgs.GetArguments();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"ECHO");
|
|
for(int index =0; index<args.Count();index++)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"ARG[{index}]:{args[index]}");
|
|
}
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskCalcBetas(CommandArgs commandArgs)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
List<DateTime> dates = FundamentalDA.GetDistinctAsOf();
|
|
|
|
foreach(DateTime date in dates)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Working on {date.ToShortDateString()}");
|
|
List<String> symbols = FundamentalDA.GetSymbolsAsOf(date);
|
|
foreach(String symbol in symbols)
|
|
{
|
|
double betaCalc36=BetaGenerator.Beta(symbol,date,36);
|
|
double betaCalc24=BetaGenerator.Beta(symbol,date,24);
|
|
double betaCalc06=BetaGenerator.Beta(symbol,date,6);
|
|
if(double.IsNaN(betaCalc36) && double.IsNaN(betaCalc24) && double.IsNaN(betaCalc06))continue;
|
|
FundamentalDA.UpdateBeta(symbol, date, betaCalc36, betaCalc24, betaCalc06);
|
|
}
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Total took {profiler.End()} (ms)");
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Performs optimization on the database tables. Since we don't want to run this if any of the monthly updates is running we will check for those tasks and wait for them
|
|
/// to complete (if running).
|
|
/// </summary>
|
|
/// <param name="commandArgs"></param>
|
|
/// <returns></returns>
|
|
public async Task TaskOptimizeDatabase(CommandArgs commandArgs)
|
|
{
|
|
int WAIT_TIME_INTERVAL = 120000; // 2 minute intervals between checks.
|
|
Profiler profiler = new Profiler();
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"Starting...");
|
|
List<String> monthlyUpdateProcessList = new List<String> { "UPDATEFINANCIALSTATEMENTS", "UPDATEFUNDAMENTALS", "UPDATEHISTORICAL", "CALCSTICKER" };
|
|
while (Utility.IsProcessRunning("mk", monthlyUpdateProcessList))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"OPTIMIZEDB is waiting for items in the process completion list {Utility.ListToString(monthlyUpdateProcessList)}");
|
|
try { Thread.Sleep(WAIT_TIME_INTERVAL); } catch (Exception) {; }
|
|
}
|
|
MaintenanceDA.OptimizeDatabase();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"Total took {profiler.End()} (ms)");
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskMGSHSession(CommandArgs commandArgs)
|
|
{
|
|
MGSHMomentumHelper.HandleMGSHSession(commandArgs);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskMGSHRunBacktest(CommandArgs commandArgs)
|
|
{
|
|
MGSHMomentumHelper.HandleMGSHRunBacktest(commandArgs);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskMGSHRunDaily(CommandArgs commandArgs)
|
|
{
|
|
MGSHMomentumHelper.HandleMGSHRunDaily(commandArgs);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskRunCMTrend(CommandArgs commandArgs)
|
|
{
|
|
CMTrendHelper.HandleRunCMTrend(commandArgs);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskCMTSession(CommandArgs commandArgs)
|
|
{
|
|
CMTrendHelper.HandleCMTSession(commandArgs);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskCMMRunCMBacktest(CommandArgs commandArgs)
|
|
{
|
|
CMMomentumHelper.RunCMMomentum(commandArgs);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskCMMRunCMSession(CommandArgs commandArgs)
|
|
{
|
|
CMMomentumHelper.RunCMSession(commandArgs);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskCMMRunCMCandidateLastResort(CommandArgs commandArgs)
|
|
{
|
|
CMMomentumHelper.RunCMCandidateLastResort(commandArgs);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskCMMRunCMGainLoss(CommandArgs commandArgs)
|
|
{
|
|
CMMomentumHelper.RunCMGainLoss(commandArgs);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskMGRunMGSession(CommandArgs commandArgs)
|
|
{
|
|
MGMomentumHelper.RunMGSession(commandArgs);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskMGRunMGGainLoss(CommandArgs commandArgs)
|
|
{
|
|
MGMomentumHelper.RunMGGainLoss(commandArgs);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
public async Task TaskMGRunMGBacktest(CommandArgs commandArgs)
|
|
{
|
|
MGMomentumHelper.RunBacktest(commandArgs);
|
|
await Task.FromResult(true);
|
|
}
|
|
|
|
// *********************************************************************************************************************************************************
|
|
// ******************************************************************* E N D T A S K S ********************************************************************
|
|
// *********************************************************************************************************************************************************
|
|
|
|
private static bool CreateLogging(String task)
|
|
{
|
|
if(String.IsNullOrEmpty(task))return false;
|
|
task=task.ToLower();
|
|
MDTrace.LogLevel = LogLevel.DEBUG;
|
|
String logFolder = "/logs";
|
|
DateTime currentDate=DateTime.Now;
|
|
String strLogFile = "marketdata_" + task + ".log";
|
|
String currentWorkingDirectory = Directory.GetCurrentDirectory();
|
|
Console.WriteLine($"Current directory is {currentWorkingDirectory}");
|
|
Utility.EnsureLogFolder(currentWorkingDirectory+logFolder);
|
|
Utility.ExpireLogs(currentWorkingDirectory+logFolder,1);
|
|
Trace.Listeners.Remove("Default");
|
|
Console.WriteLine($"Adding Trace Listener :{currentWorkingDirectory+logFolder+"/"+strLogFile}");
|
|
Trace.Listeners.Add(new TextWriterTraceListener(currentWorkingDirectory+logFolder+"/"+strLogFile));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Trace Listener added.");
|
|
Utility.ShowLogs(currentWorkingDirectory + logFolder);
|
|
return true;
|
|
}
|
|
|
|
// **********************************************************************************************************************************************
|
|
// ************************************************* U P D A T E D A I L Y 2 M E T H O D S ***************************************************
|
|
// **********************************************************************************************************************************************
|
|
|
|
public static void UpdateDaily2(DateTime startDate)
|
|
{
|
|
DateGenerator dateGenerator=new DateGenerator();
|
|
IConfiguration configuration = GlobalConfig.Instance.Configuration;
|
|
String smsSMTPAddress = configuration["sms_smtpaddress"];
|
|
String smsUserName = configuration["sms_smsusername"];
|
|
String smsPassword = configuration["sms_smspassword"];
|
|
String[] smsRecipients = configuration["sms_smsrecipients"].Split(',');
|
|
DateTime currentDate=DateTime.Now.Date;
|
|
|
|
// Sanity check. If the given date is not today then ask the user to confirm
|
|
if (currentDate != startDate.Date)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,ConsoleColor.Red,$"Run date is not today: Current Date:{currentDate.ToShortDateString()} Run Date: {startDate.ToShortDateString()}");
|
|
return;
|
|
}
|
|
|
|
if(!CheckRunCriteria())
|
|
{
|
|
return;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"UPDATEDAILY2 DATE:{startDate.ToShortDateString()}");
|
|
int STAGE_1=0,STAGE_2=1,STAGE_3=2,STAGE_4=3,STAGE_5=4,STAGE_6=5,STAGE_7=6,STAGE_8=7,STAGE_9=8,STAGE_10=9,STAGE_11=10,STAGE_FINAL=11;
|
|
DeletePriceWatchList("valuations",startDate.ToShortDateString());
|
|
DeletePriceWatchList("Momentum",startDate.ToShortDateString());
|
|
|
|
ManualResetEvent[] resetEvents = new ManualResetEvent[STAGE_FINAL+1];
|
|
for(int index=0;index<resetEvents.Length;index++)resetEvents[index] = new ManualResetEvent(false);
|
|
|
|
resetEvents[STAGE_1].Reset();
|
|
|
|
// This thread serves as network monitor thread. It will live for the duration of the other threads and exit when they are all completed
|
|
// The monitor thread should always be STAGE_FINAL so make whatever adjustments are necessary to make this hold going forward
|
|
ThreadPool.QueueUserWorkItem(delegate
|
|
{
|
|
List<ManualResetEvent> manualResetEventsList = resetEvents.ToList<ManualResetEvent>();
|
|
|
|
manualResetEventsList.Remove(resetEvents[STAGE_FINAL]); // Remove THIS thread from the events we will wait on..no sense waiting on ourselves
|
|
ManualResetEvent[] allResetEventsSans12 = manualResetEventsList.ToArray();
|
|
bool running = true;
|
|
bool connectionIssue = false;
|
|
|
|
SMSClient.SendSMSEmail("UPDATEDAILY2 Network monitor activated.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
|
|
while (running)
|
|
{
|
|
if (!NetworkStatus.IsInternetConnected())
|
|
{
|
|
connectionIssue = true;
|
|
SMSClient.SendSMSEmail("UPDATEDAILY2 Network is not connected.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
|
|
}
|
|
else if (connectionIssue)
|
|
{
|
|
connectionIssue = false;
|
|
SMSClient.SendSMSEmail("UPDATEDAILY2 There was a network connection issue.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
|
|
}
|
|
|
|
if (WaitHandle.WaitAll(allResetEventsSans12, 30000))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"All events have signaled completion.");
|
|
running = false; // if all worker threads are done then wrap it up here.
|
|
}
|
|
}
|
|
SMSClient.SendSMSEmail("UPDATEDAILY2 Network monitor ended.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
|
|
resetEvents[STAGE_FINAL].Set();
|
|
});
|
|
// Here is the start of the real workers
|
|
ThreadPool.QueueUserWorkItem(delegate
|
|
{
|
|
UpdatePricesBigCharts(startDate); // bigcharts.marketwatch.com
|
|
UpdatePricesYahooSweep(startDate); // The sweep variation of the method is intended to be used after the BigCharts update because the sweep will take pricing_source into consideration when fetching prices.
|
|
resetEvents[STAGE_1].Set();
|
|
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_1 UPDATEPRICESBIGCHARTS/YAHOO done.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_1 complete.");
|
|
});
|
|
resetEvents[STAGE_1].WaitOne(); // wait for pricing to finish
|
|
|
|
ThreadPool.QueueUserWorkItem(delegate
|
|
{
|
|
UpdateCompanyProfiles(); // financials.morningstar.com and finance.yahoo.com
|
|
resetEvents[STAGE_2].Set();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_2 complete.");
|
|
});
|
|
|
|
ThreadPool.QueueUserWorkItem(delegate
|
|
{
|
|
UpdateYieldCurve(); // www.treasury.gov
|
|
LoadGDPPerCapita(); // api.worldbank.org
|
|
LoadConsumerPriceIndex(); // Load consumer price index data from Bureau of Labor Statistics
|
|
resetEvents[STAGE_3].Set();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_3 complete.");
|
|
});
|
|
|
|
ThreadPool.QueueUserWorkItem(delegate
|
|
{
|
|
UpdateLatestAnalystRatings(true); // WWW.BRIEFING.COM
|
|
UpdateMissingAnalystRatings(); // MARKET BEAT
|
|
resetEvents[STAGE_4].Set();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_4 complete");
|
|
});
|
|
|
|
ThreadPool.QueueUserWorkItem(delegate
|
|
{
|
|
UpdateSplits(); // eoddata.com
|
|
ProcessAllSplits(); // non-network operation
|
|
resetEvents[STAGE_5].Set();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_5 complete.");
|
|
});
|
|
|
|
ThreadPool.QueueUserWorkItem(delegate
|
|
{
|
|
LoadInsiderTransactions();
|
|
resetEvents[STAGE_6].Set();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_6 complete.");
|
|
});
|
|
|
|
ThreadPool.QueueUserWorkItem(delegate
|
|
{
|
|
UpdateEarningsAnnouncements(); // www.zacks.com
|
|
LoadZacksRank(); // www.zacks.com
|
|
resetEvents[STAGE_7].Set();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_7 complete.");
|
|
});
|
|
|
|
ThreadPool.QueueUserWorkItem(delegate
|
|
{
|
|
GetSECFilingsWatchList("Valuations");// www.sec.gov
|
|
resetEvents[STAGE_8].Set();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_8 complete");
|
|
});
|
|
|
|
ThreadPool.QueueUserWorkItem(delegate
|
|
{
|
|
UpdateAnalystPriceTarget(); // MarketBeat
|
|
GetETFHoldings(); // finance.yahoo.com
|
|
resetEvents[STAGE_9].Set();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_9 complete.");
|
|
});
|
|
|
|
ThreadPool.QueueUserWorkItem(delegate
|
|
{
|
|
UpdateAllMissingCIK(); // /www.sec.gov
|
|
resetEvents[STAGE_10].Set();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_10 complete.");
|
|
});
|
|
|
|
ThreadPool.QueueUserWorkItem(delegate
|
|
{
|
|
UpdateDividendHistory(); // www.nasdaq.com DIVIDEND HISTORY IS BROKEN.... WORKING ON THIS ONE.
|
|
resetEvents[STAGE_11].Set();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_11 complete.");
|
|
});
|
|
|
|
WaitHandle.WaitAll(resetEvents);
|
|
SMSClient.SendSMSEmail("UPDATEDAILY2 All stages complete.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done.");
|
|
}
|
|
|
|
public static void UpdatePricesBigCharts(DateTime startDate)
|
|
{
|
|
PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper();
|
|
pricingMarketDataHelper.UpdatePricesBigCharts(startDate);
|
|
}
|
|
|
|
public static void UpdatePricesYahooSweep(DateTime startDate)
|
|
{
|
|
PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper();
|
|
pricingMarketDataHelper.UpdatePricesYahooSweep(startDate);
|
|
}
|
|
|
|
public static void DeletePriceWatchList(String watchListName,String strDate)
|
|
{
|
|
try
|
|
{
|
|
List<String> symbols = WatchListDA.GetWatchList(watchListName);
|
|
if(null==symbols||0==symbols.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("There are no symbols for watchlist {0}",watchListName));
|
|
return;
|
|
}
|
|
foreach (String symbol in symbols)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Delete price {0} on {1}, watchlist {2}",symbol,strDate,watchListName));
|
|
DeletePriceSymbol(symbol, strDate);
|
|
}
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
}
|
|
}
|
|
|
|
public static void DeletePriceSymbol(String symbol, String strDate)
|
|
{
|
|
try
|
|
{
|
|
DateTime pricingDate = DateTime.Parse(strDate);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Delete price '" + symbol + "' for " + Utility.DateTimeToStringMMSDDSYYYY(pricingDate));
|
|
if(PricingDA.DeletePrice(symbol, pricingDate)) MDTrace.WriteLine(LogLevel.DEBUG,"Ok");
|
|
else MDTrace.WriteLine(LogLevel.DEBUG,"Failed.");
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
}
|
|
}
|
|
|
|
public static void LoadConsumerPriceIndex()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
|
|
PriceIndices priceIndices=MarketDataHelper.GetConsumerPriceIndices();
|
|
if(null==priceIndices)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Failed, check log file for isssues.");
|
|
return;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got:{0} records from source feed.",priceIndices.Count()));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Saving...");
|
|
ConsumerPriceIndexDA.InsertUpdatePriceIndices(priceIndices);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Save complete...");
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void UpdateCompanyProfiles()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Started.");
|
|
CompanyProfileMarketDataHelper companyProfileMarketDataHelper=new CompanyProfileMarketDataHelper();
|
|
companyProfileMarketDataHelper.UpdateCompanyProfiles();
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms).");
|
|
}
|
|
}
|
|
|
|
public static void LoadGDPPerCapita()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
|
|
EconomicIndicators economicIndicators=MarketDataHelper.GetGDPPerCapita();
|
|
if(null==economicIndicators||0==economicIndicators.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"No data, see log file for potential issues.");
|
|
return;
|
|
}
|
|
List<String> distinctCountry=(from EconomicIndicator economicIndicator in economicIndicators select economicIndicator.CountryCode).Distinct().ToList();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Downloaded {0} countries.",distinctCountry.Count));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Saving...");
|
|
if(EconomicIndicatorDA.InsertUpdateEconomicIndicators(economicIndicators))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Save complete.");
|
|
}
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Failed to save economic indicators.");
|
|
}
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void UpdateYieldCurve() // maintains current year of yield curve data
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
int year = DateTime.Now.Year;
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving yield curve for year "+year);
|
|
YieldCurve yieldCurve=MarketDataHelper.GetYieldCurve(year);
|
|
if (null == yieldCurve)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Failed to get YieldCurve for {0}", year));
|
|
return;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Got "+yieldCurve.Count+" points for "+year);
|
|
YieldCurveDA.InsertOrUpdate(yieldCurve);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void UpdateLatestAnalystRatings(Boolean createSecurityMaster = true)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
if(!CheckRunCriteria())return;
|
|
DateGenerator dateGenerator = new DateGenerator();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Started.");
|
|
AnalystRatings analystRatings = MarketDataHelper.GetLatestAnalystRatings();
|
|
List<String> symbols = PricingDA.GetSymbolsNotIn(analystRatings.Symbols);
|
|
foreach (String symbol in symbols)
|
|
{
|
|
if (false == createSecurityMaster)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Removing " + symbol + ", missing from security master.");
|
|
AnalystRating analystRating = (from item in analystRatings where item.Symbol.Equals(symbol) select item).FirstOrDefault();
|
|
if (null != analystRating) MDTrace.WriteLine(LogLevel.DEBUG, analystRating.ToString());
|
|
}
|
|
}
|
|
if (false == createSecurityMaster) analystRatings.Remove(symbols);
|
|
foreach (AnalystRating analystRating in analystRatings)
|
|
{
|
|
String symbolNotFound = (from symbol in symbols where symbol.Equals(analystRating.Symbol) select symbol).FirstOrDefault();
|
|
if (null != symbolNotFound && createSecurityMaster)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Adding security '" + analystRating.Symbol + "' '" + analystRating.CompanyName + "'");
|
|
if (!PricingDA.AddSecurity(analystRating.Symbol, analystRating.CompanyName)) MDTrace.WriteLine(LogLevel.DEBUG, "Failed to add '" + symbols + "'");
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Added '" + analystRating.Symbol + "'");
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Loading prices for {0}", analystRating.Symbol));
|
|
DateTime pricingDate = dateGenerator.FindPrevBusinessDay(analystRating.Date);
|
|
Prices prices = MarketDataHelper.GetDailyPrices(analystRating.Symbol, Constants.MIN_PRICING_DATE, pricingDate); // use the Yahoo JSON bulk feed
|
|
if (null != prices)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting {0} prices for {1}", prices.Count, analystRating.Symbol));
|
|
PricingDA.InsertPrices(prices);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "Insert done.");
|
|
}
|
|
}
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Rating {analystRating.ToString()}");
|
|
}
|
|
AnalystRatingsDA.InsertAnalystRatings(analystRatings);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception:{exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void UpdateMissingAnalystRatings()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
DateGenerator dateGenerator = new DateGenerator();
|
|
List<String> symbols = PricingDA.GetSymbols();
|
|
DateTime maxDate=AnalystRatingsDA.GetMaxDateNoZacks();
|
|
foreach (String symbol in symbols)
|
|
{
|
|
AnalystRatings analystRatings = MarketDataHelper.GetAnalystRatingsMarketBeat(symbol);
|
|
if (null == analystRatings || 0 == analystRatings.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("No analyst ratings for {0}", symbol));
|
|
continue;
|
|
}
|
|
analystRatings = new AnalystRatings((from AnalystRating analystRating in analystRatings where analystRating.Date >= maxDate.Date select analystRating).ToList());
|
|
AnalystRatings duplicateList = new AnalystRatings();
|
|
foreach (AnalystRating analystRating in analystRatings)
|
|
{
|
|
if (AnalystRatingsDA.ContainsAnalystRating(analystRating))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Already have analyst rating for {0} on {1} from brokerage firm {2}", analystRating.Symbol, analystRating.Date.ToShortDateString(), analystRating.BrokerageFirm));
|
|
duplicateList.Add(analystRating);
|
|
continue;
|
|
}
|
|
}
|
|
analystRatings = new AnalystRatings(analystRatings.Except(duplicateList).ToList());
|
|
foreach (AnalystRating analystRating in analystRatings)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting Analyst Rating for {0} on {1} from brokerage firm {2}", analystRating.Symbol, analystRating.Date.ToShortDateString(), analystRating.BrokerageFirm));
|
|
}
|
|
AnalystRatingsDA.InsertAnalystRatings(analystRatings);
|
|
}
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void UpdateSplits()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Updating splits.");
|
|
Splits splits=MarketDataHelper.GetSplits();
|
|
if(null==splits||0==splits.Count)return;
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} split records.",splits.Count));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Writing to database.",splits.Count));
|
|
SplitsDA.InsertSplits(splits);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Done."));
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void ProcessAllSplits()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Processing all splits");
|
|
Splits splits=SplitsDA.GetUnappliedSplits();
|
|
if(null==splits||0==splits.Count)return;
|
|
SplitHelper.ProcessSplits(splits);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void LoadInsiderTransactions()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
|
|
List<String> symbols = PricingDA.GetSymbols();
|
|
InsiderTransactionMarketDataHelper insiderTransactionMarketDataHelper = new InsiderTransactionMarketDataHelper();
|
|
insiderTransactionMarketDataHelper.LoadInsiderTransactions(symbols);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void UpdateEarningsAnnouncements(String symbol=null)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
|
|
EarningsAnnouncementsMarketDataHelper earningsAnnouncementsMarketDataHelper=new EarningsAnnouncementsMarketDataHelper();
|
|
earningsAnnouncementsMarketDataHelper.UpdateEarningsAnnouncements(symbol);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void LoadZacksRank()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
|
|
ZacksRankMarketDataHelper zacksRankMarketDataHelper=new ZacksRankMarketDataHelper();
|
|
zacksRankMarketDataHelper.LoadZacksRank();
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void GetSECFilingsWatchList(String watchListName)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
|
|
List<String> symbols = WatchListDA.GetWatchList(watchListName);
|
|
SECFilingMarketDataHelper secFilingMarketDataHelper=new SECFilingMarketDataHelper();
|
|
secFilingMarketDataHelper.UpdateSECFilings(symbols);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void UpdateAnalystPriceTarget()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
|
|
AnalystPriceTargetMarketDataHelper analystPriceTargetMarketDataHelper=new AnalystPriceTargetMarketDataHelper();
|
|
analystPriceTargetMarketDataHelper.LoadAnalystPriceTarget();
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void GetETFHoldings()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
|
|
ETFHoldingsMarketDataHelper etfHoldingsMarketDataHelper=new ETFHoldingsMarketDataHelper();
|
|
etfHoldingsMarketDataHelper.LoadETFHoldings();
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void UpdateAllMissingCIK()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
|
|
CIKMarketDataHelper cikMarketDataHelper=new CIKMarketDataHelper();
|
|
cikMarketDataHelper.UpdateAllMissingCIK();
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
public static void UpdateDividendHistory()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
try
|
|
{
|
|
DividendHistoryMarketDataHelper dividendHistoryMarketDataHelper=new DividendHistoryMarketDataHelper();
|
|
dividendHistoryMarketDataHelper.UpdateDividendHistory();
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
|
|
}
|
|
}
|
|
|
|
// **********************************************************************************************************************************************
|
|
// ******************************************* E N D U P D A T E D A I L Y 2 M E T H O D S ***************************************************
|
|
// **********************************************************************************************************************************************
|
|
public static void UpdateAnalystRatings(String paramSymbol = null,DateTime? minDate=null)
|
|
{
|
|
try
|
|
{
|
|
if(!CheckRunCriteria())return;
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "[UPDATEANALYSTRATINGS]");
|
|
List<String> symbols = new List<String>();
|
|
if (paramSymbol == null) symbols = PricingDA.GetSymbols();
|
|
else symbols.Add(paramSymbol);
|
|
foreach (String symbol in symbols)
|
|
{
|
|
AnalystRatings analystRatings = MarketDataHelper.GetAnalystRatingsMarketBeat(symbol);
|
|
if (null == analystRatings || 0 == analystRatings.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("No analyst ratings for {0}", symbol));
|
|
continue;
|
|
}
|
|
AnalystRatings duplicateList = new AnalystRatings();
|
|
if (null != minDate)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Minimum Rating Date of {0} is specified.", minDate.Value.ToShortDateString()));
|
|
analystRatings = new AnalystRatings((from AnalystRating analystRating in analystRatings where analystRating.Date >= minDate.Value select analystRating).ToList());
|
|
}
|
|
foreach (AnalystRating analystRating in analystRatings)
|
|
{
|
|
if (AnalystRatingsDA.ContainsAnalystRating(analystRating))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Already have analyst rating for {0} on {1} from brokerage firm {2}", analystRating.Symbol, analystRating.Date.ToShortDateString(), analystRating.BrokerageFirm));
|
|
duplicateList.Add(analystRating);
|
|
continue;
|
|
}
|
|
}
|
|
analystRatings = new AnalystRatings(analystRatings.Except(duplicateList).ToList());
|
|
foreach (AnalystRating analystRating in analystRatings)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting Analyst Rating for {0} on {1} from brokerage firm {2}", analystRating.Symbol, analystRating.Date.ToShortDateString(), analystRating.BrokerageFirm));
|
|
}
|
|
AnalystRatingsDA.InsertAnalystRatings(analystRatings);
|
|
}
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, exception);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, "[UPDATEANALYSTRATINGS]");
|
|
}
|
|
}
|
|
|
|
public static void UpdateLatestPriceWatchList(String watchListName)
|
|
{
|
|
try
|
|
{
|
|
if(!CheckRunCriteria())return;
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"UPDATELATESTPRICEWATCHLIST WATCHLIST:{watchListName}");
|
|
List<String> symbols = WatchListDA.GetWatchList(watchListName);
|
|
PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper();
|
|
pricingMarketDataHelper.UpdateLatestPrices(symbols);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
}
|
|
}
|
|
|
|
public static void UpdateLatestPriceOpenPositions()
|
|
{
|
|
try
|
|
{
|
|
if(!CheckRunCriteria())return;
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"UPDATELATESTPRICEOPENPOSITIONS");
|
|
PortfolioTrades portfolioTrades=PortfolioDA.GetOpenTrades();
|
|
List<String> symbols=(from PortfolioTrade portfolioTrade in portfolioTrades select portfolioTrade.Symbol).Distinct().ToList();
|
|
if(symbols.Any(x=>x.Equals("SPY")))symbols.Add("SH");
|
|
PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper();
|
|
pricingMarketDataHelper.UpdateLatestPrices(symbols);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
}
|
|
}
|
|
|
|
// **********************************************************************************************************************************************
|
|
|
|
public static void LoadHeadlinesWatchList(String watchList)
|
|
{
|
|
if(!CheckRunCriteria())return;
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("LOADHEADLINESWATCHLIST {0}",watchList));
|
|
List<String> symbols = WatchListDA.GetWatchList(watchList);
|
|
HeadlinesMarketDataHelper headlinesMarketDataHelper= new HeadlinesMarketDataHelper();
|
|
headlinesMarketDataHelper.LoadHeadlines(symbols);
|
|
}
|
|
|
|
private static void LoadPremarketData()
|
|
{
|
|
int retries=3;
|
|
int sleepTime=2000;
|
|
PremarketElements premarketElements=null;
|
|
|
|
if(!CheckRunCriteria())return;
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("LOADPREMARKETDATA"));
|
|
for(int retry=0;retry<retries && (null==premarketElements || 0==premarketElements.Count);retry++,Thread.Sleep(sleepTime))
|
|
{
|
|
premarketElements=MarketDataHelper.GetPremarketData();
|
|
}
|
|
if(null==premarketElements||0==premarketElements.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Failed to retrieve premarket data."));
|
|
return;
|
|
}
|
|
PremarketDA.AddElements(premarketElements);
|
|
foreach(PremarketElement premarketElement in premarketElements)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} {1} {2}",premarketElement.Market,Utility.FormatNumber(premarketElement.ChangeValue),Utility.FormatPercent(premarketElement.ChangePercent/100.00)));
|
|
}
|
|
}
|
|
|
|
public static void CalcSticker()
|
|
{
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Generating valuations..");
|
|
Valuations valuations = ValuationGenerator.GenerateValuations(new ValuationGenerator.OnItemCompleteHandler
|
|
((valuation) =>
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Completed '{valuation?.Symbol}' '{valuation?.Company}'",3);
|
|
return true;
|
|
}
|
|
),
|
|
new ValuationGenerator.OnErrorItemHandler((symbol,message)=>
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Error generating valuation for {symbol} {message}",3);
|
|
}
|
|
));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Updating database.");
|
|
if (!ValuationDA.AddValuations(valuations))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Error adding valuations to database");
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Done.");
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
}
|
|
}
|
|
|
|
public static void UpdateHistorical()
|
|
{
|
|
HistoricalMarketDataHelper historicalMarketDataHelper=new HistoricalMarketDataHelper();
|
|
historicalMarketDataHelper.LoadHistorical();
|
|
}
|
|
|
|
public static void UpdateFundamentalsFinViz()
|
|
{
|
|
FundamentalMarketDataHelper fundamentalMarketDataHelper=new FundamentalMarketDataHelper();
|
|
fundamentalMarketDataHelper.LoadFundamentalsFinViz();
|
|
}
|
|
|
|
public static void UpdateFundamentals()
|
|
{
|
|
FundamentalMarketDataHelper fundamentalMarketDataHelper=new FundamentalMarketDataHelper();
|
|
fundamentalMarketDataHelper.LoadFundamentals();
|
|
}
|
|
|
|
public static void UpdateFinancialStatements()
|
|
{
|
|
FinancialStatementsMarketDataHelper financialStatementsMarketDataHelper=new FinancialStatementsMarketDataHelper();
|
|
financialStatementsMarketDataHelper.LoadFinancialStatements();
|
|
}
|
|
|
|
// ********************************************************************************************************************************
|
|
public static bool CheckRunCriteria()
|
|
{
|
|
DateGenerator dateGenerator = new DateGenerator();
|
|
DateTime currentDate = DateTime.Now;
|
|
|
|
if(!dateGenerator.IsMarketOpen(currentDate))
|
|
{
|
|
String description="";
|
|
if(HolidayDA.IsMarketHoliday(currentDate)) description=HolidayDA.GetHolidayDescription(currentDate);
|
|
else description=Utility.DayOfWeekToString(currentDate.DayOfWeek);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Market is closed today {currentDate.ToShortDateString()} for {description}.");
|
|
return false;
|
|
}
|
|
if(!NetworkStatus.IsInternetConnected())
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"The internet is not connected.");
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
}
|
|
} |