Files
ARM64/MarketData/MarketData/Services/MainService.cs
2025-04-10 21:22:05 -04:00

871 lines
39 KiB
C#
Executable File

using System.Diagnostics;
using MarketData.DataAccess;
using MarketData.Interface;
using MarketData.Utils;
using MarketData.Configuration;
using Microsoft.Extensions.Configuration;
using MarketData.MarketDataModel;
using MarketData.Helper;
using MarketData.Integration;
using MarketData.Cache;
namespace MarketData.Services
{
public class MainService : IMainService
{
private Dictionary<String, Func<CommandArgs,Task>> tasks = new Dictionary<String,Func<CommandArgs,Task>>();
/// <summary>
/// DisplayUsage
/// </summary>
public static void DisplayUsage()
{
MDTrace.WriteLine(LogLevel.DEBUG,"USAGE");
MDTrace.WriteLine(LogLevel.DEBUG,"LOADHEADLINESWATCHLIST");
MDTrace.WriteLine(LogLevel.DEBUG,"LOADPREMARKETDATA");
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEDAILY2 /DATE:");
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTPRICEOPENPOSITIONS");
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTPRICEWATCHLIST /WATCHLIST:");
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTANALYSTRATINGS");
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEANALYSTRATINGS");
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATESECFILINGSWATCHLIST /WATCHLIST:");
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATECOMPANYPROFILES");
MDTrace.WriteLine(LogLevel.DEBUG,"ECHO {param1} {param2} {param(n)");
MDTrace.WriteLine(LogLevel.DEBUG,"MGSHSESSION /SESSIONFILE:");
MDTrace.WriteLine(LogLevel.DEBUG,"MGSHRUNBACKTEST /USEHEDGING: /HEDGEINITIALCASH: /USESTOPLIMITS: /KEEPSLOTPOSITIONS: /STARTDATE: /MAXPOSITIONS: /INITIALCASH: /HOLDINGPERIOD: /{ENDDATE}: /SESSIONFILE: ");
MDTrace.WriteLine(LogLevel.DEBUG,"MGSHRUNDAILY /SESSIONFILE: /TRADEDATE:");
MDTrace.WriteLine(LogLevel.DEBUG,"RUNCMTREND /MODE:DAILY|BACKTEST|RUNTRENDTEMPLATE|ENTRYTEST /SYMBOL:{for mode ANALYZE,ENTRYTEST} /TRADEDATE:{for mode DAILY,RUNTRENDTEMPLATE) /STARTDATE:(for mode BACKTEST,ENTRYTEST) /ENDDATE:(for mode BACKTEST) /INITIALCASH: /SESSIONFILE: /MAXOPENPOSITIONS: /MAXDAILYPOSITIONS: Runs Mark Minervini trend");
MDTrace.WriteLine(LogLevel.DEBUG,"CMTSESSION /SESSIONFILE:{pathfilename} Runs Mark Minervini trend display session");
}
/// <summary>
/// This is the main entry point
/// </summary>
/// <param name="args"></param>
/// <param name="configuration"></param>
public void RunService(String[] args,IConfiguration configuration)
{
Profiler profiler = new Profiler();
DateTime currentDate=DateTime.Now;
tasks.Add("LOADHEADLINESWATCHLIST",TaskLoadHeadlinesWatchList);
tasks.Add("LOADPREMARKETDATA",TaskLoadPremarketData);
tasks.Add("UPDATEDAILY2",TaskUpdateDaily2);
tasks.Add("UPDATELATESTPRICEOPENPOSITIONS",TaskUpdateLatestPriceOpenPositions);
tasks.Add("UPDATELATESTPRICEWATCHLIST",TaskUpdateLatestPriceWatchList);
tasks.Add("UPDATELATESTANALYSTRATINGS",TaskUpdateLatestAnalystRatings);
tasks.Add("UPDATEANALYSTRATINGS",TaskUpdateAnalystRatings);
tasks.Add("UPDATESECFILINGSWATCHLIST",TaskUpdateSECFilingsWatchList);
tasks.Add("UPDATECOMPANYPROFILES",TaskUpdateCompanyProfiles);
tasks.Add("MGSHSESSION",TaskMGSHSession);
tasks.Add("MGSHRUNBACKTEST",TaskMGSHRunBacktest);
tasks.Add("MGSHRUNDAILY",TaskMGSHRunDaily);
tasks.Add("RUNCMTREND",TaskRunCMTrend);
tasks.Add("CMTSESSION",TaskCMTSession);
tasks.Add("ECHO",TaskEcho);
GlobalConfig.Instance.Configuration = configuration; // This call sets up configuration stuff so it needs to be first.
if (args.Length < 1 || String.IsNullOrEmpty(args[0]))
{
DisplayUsage();
return;
}
string arg = args[0].ToUpper();
// log files are now of the form market_data+task.log. Also log files will expire daily
if(!CreateLogging(arg))
{
Console.WriteLine("CreateLogging returned false.");
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,$"[RunService] Started @ {Utility.DateTimeToStringYYYYHMMHDDHHMMSSTT(currentDate)}");
MDTrace.WriteLine(LogLevel.DEBUG,$"[RunService] Argument {arg}");
DateTime maxHolidayDate =HolidayDA.GetMaxHolidayDate();
if(currentDate>maxHolidayDate)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("There are no holidays defined in the system. Add holidays for year {0} into marketholidays table",currentDate.Year));
return;
}
if(!tasks.ContainsKey(arg))
{
DisplayUsage();
return;
}
try
{
tasks[arg](new CommandArgs(args));
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return;
}
finally
{
LocalPriceCache.GetInstance().Dispose();
GBPriceCache.GetInstance().Dispose();
}
MDTrace.WriteLine(LogLevel.DEBUG,$"[RunService] Done, total took {profiler.End()}(ms)");
}
// **********************************************************************************************************************************************************
// ********************************************************* T A S K S *************************************************************************************
// **********************************************************************************************************************************************************
public async Task TaskUpdateCompanyProfiles(CommandArgs commandArgs)
{
UpdateCompanyProfiles();
await Task.FromResult(true);
}
public async Task TaskUpdateSECFilingsWatchList(CommandArgs commandArgs)
{
if(!commandArgs.Has("WATCHLIST")){Console.WriteLine("UPDATESECFILINGSWATCHLIST REQUIRES WATCHLIST");return;}
String watchListName = commandArgs.Coalesce<String>("WATCHLIST");
List<String> symbols = WatchListDA.GetWatchList(watchListName);
SECFilingMarketDataHelper secFilingMarketDataHelper=new SECFilingMarketDataHelper();
secFilingMarketDataHelper.UpdateSECFilings(symbols);
await Task.FromResult(true);
}
public async Task TaskLoadHeadlinesWatchList(CommandArgs commandArgs)
{
if(!commandArgs.Has("WATCHLIST")){Console.WriteLine("LOADHEADLINESWATCHLIST REQUIRES WATCHLIST");return;}
else LoadHeadlinesWatchList(commandArgs.Coalesce<String>("WATCHLIST"));
await Task.FromResult(true);
}
public async Task TaskLoadPremarketData(CommandArgs commandArgs)
{
LoadPremarketData();
await Task.FromResult(true);
}
public async Task TaskUpdateDaily2(CommandArgs commandArgs)
{
if(!commandArgs.Has("DATE")){Console.WriteLine("UPDATEDAILY2 MISSING DATE");return;}
UpdateDaily2(commandArgs.Coalesce<DateTime>("DATE"));
await Task.FromResult(true);
}
public async Task TaskUpdateLatestPriceOpenPositions(CommandArgs commandArgs)
{
UpdateLatestPriceOpenPositions();
await Task.FromResult(true);
}
public async Task TaskUpdateLatestPriceWatchList(CommandArgs commandArgs)
{
if(!commandArgs.Has("WATCHLIST")){Console.WriteLine("UPDATELATESTPRICEWATCHLIST MISSING WATCHLIST");return;}
UpdateLatestPriceWatchList(commandArgs.Coalesce<String>("WATCHLIST"));
await Task.FromResult(true);
}
public async Task TaskUpdateLatestAnalystRatings(CommandArgs commandArgs)
{
if (commandArgs.Has("UPDATESECURITYMASTER")) UpdateLatestAnalystRatings(commandArgs.Coalesce<Boolean>("UPDATESECURITYMASTER"));
else UpdateLatestAnalystRatings();
await Task.FromResult(true);
}
public async Task TaskUpdateAnalystRatings(CommandArgs commandArgs)
{
if (commandArgs.Has("SYMBOL") && commandArgs.Has("MINDATE")) UpdateAnalystRatings(commandArgs.Coalesce<String>("SYMBOL"), commandArgs.Coalesce<DateTime>("MINDATE"));
else if (commandArgs.Has("SYMBOL") ) UpdateAnalystRatings(commandArgs.Coalesce<String>("SYMBOL"));
else if (commandArgs.Has("MINDATE")) UpdateAnalystRatings(null,commandArgs.Coalesce<DateTime>("MINDATE"));
else UpdateAnalystRatings();
await Task.FromResult(true);
}
public async Task TaskEcho(CommandArgs commandArgs)
{
String[] args = commandArgs.GetArguments();
MDTrace.WriteLine(LogLevel.DEBUG,$"ECHO");
for(int index =0; index<args.Count();index++)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"ARG[{index}]:{args[index]}");
}
await Task.FromResult(true);
}
public async Task TaskMGSHSession(CommandArgs commandArgs)
{
MGSHMomentumHelper.HandleMGSHSession(commandArgs);
await Task.FromResult(true);
}
public async Task TaskMGSHRunBacktest(CommandArgs commandArgs)
{
MGSHMomentumHelper.HandleMGSHRunBacktest(commandArgs);
await Task.FromResult(true);
}
public async Task TaskMGSHRunDaily(CommandArgs commandArgs)
{
MGSHMomentumHelper.HandleMGSHRunDaily(commandArgs);
await Task.FromResult(true);
}
public async Task TaskRunCMTrend(CommandArgs commandArgs)
{
CMTrendHelper.HandleRunCMTrend(commandArgs);
await Task.FromResult(true);
}
public async Task TaskCMTSession(CommandArgs commandArgs)
{
CMTrendHelper.HandleCMTSession(commandArgs);
await Task.FromResult(true);
}
// *********************************************************************************************************************************************************
// ******************************************************************* E N D T A S K S ********************************************************************
// *********************************************************************************************************************************************************
private static bool CreateLogging(String task)
{
if(String.IsNullOrEmpty(task))return false;
task=task.ToLower();
MDTrace.LogLevel = LogLevel.DEBUG;
String logFolder = "/logs";
DateTime currentDate=DateTime.Now;
String strLogFile = "marketdata_" + task + ".log";
String currentWorkingDirectory = Directory.GetCurrentDirectory();
Console.WriteLine($"Current directory is {currentWorkingDirectory}");
Utility.EnsureLogFolder(currentWorkingDirectory+logFolder);
Utility.ExpireLogs(currentWorkingDirectory+logFolder,1);
Trace.Listeners.Remove("Default");
Console.WriteLine($"Adding Trace Listener :{currentWorkingDirectory+logFolder+"/"+strLogFile}");
Trace.Listeners.Add(new TextWriterTraceListener(currentWorkingDirectory+logFolder+"/"+strLogFile));
MDTrace.WriteLine($"Trace Listener added.");
Utility.ShowLogs(currentWorkingDirectory + logFolder);
return true;
}
// **********************************************************************************************************************************************
// ************************************************* U P D A T E D A I L Y 2 M E T H O D S ***************************************************
// **********************************************************************************************************************************************
public static void UpdateDaily2(DateTime startDate)
{
DateGenerator dateGenerator=new DateGenerator();
IConfiguration configuration = GlobalConfig.Instance.Configuration;
String smsSMTPAddress = configuration["sms_smtpaddress"];
String smsUserName = configuration["sms_smsusername"];
String smsPassword = configuration["sms_smspassword"];
String[] smsRecipients = configuration["sms_smsrecipients"].Split(',');
DateTime currentDate=DateTime.Now.Date;
// Sanity check. If the given date is not today then ask the user to confirm
if (currentDate != startDate.Date)
{
MDTrace.WriteLine(LogLevel.DEBUG,ConsoleColor.Red,$"Run date is not today: Current Date:{currentDate.ToShortDateString()} Run Date: {startDate.ToShortDateString()}");
return;
}
if(!CheckRunCriteria())
{
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,$"UPDATEDAILY2 DATE:{startDate.ToShortDateString()}");
int STAGE_1=0,STAGE_2=1,STAGE_3=2,STAGE_4=3,STAGE_5=4,STAGE_6=5,STAGE_7=6,STAGE_8=7,STAGE_9=8,STAGE_10=9,STAGE_11=10,STAGE_FINAL=11;
DeletePriceWatchList("valuations",startDate.ToShortDateString());
DeletePriceWatchList("Momentum",startDate.ToShortDateString());
ManualResetEvent[] resetEvents = new ManualResetEvent[STAGE_FINAL+1];
for(int index=0;index<resetEvents.Length;index++)resetEvents[index] = new ManualResetEvent(false);
resetEvents[STAGE_1].Reset();
// This thread serves as network monitor thread. It will live for the duration of the other threads and exit when they are all completed
// The monitor thread should always be STAGE_FINAL so make whatever adjustments are necessary to make this hold going forward
ThreadPool.QueueUserWorkItem(delegate
{
List<ManualResetEvent> manualResetEventsList = resetEvents.ToList<ManualResetEvent>();
manualResetEventsList.Remove(resetEvents[STAGE_FINAL]); // Remove THIS thread from the events we will wait on..no sense waiting on ourselves
ManualResetEvent[] allResetEventsSans12 = manualResetEventsList.ToArray();
bool running = true;
bool connectionIssue = false;
SMSClient.SendSMSEmail("UPDATEDAILY2 Network monitor activated.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
while (running)
{
if (!NetworkStatus.IsInternetConnected())
{
connectionIssue = true;
SMSClient.SendSMSEmail("UPDATEDAILY2 Network is not connected.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
}
else if (connectionIssue)
{
connectionIssue = false;
SMSClient.SendSMSEmail("UPDATEDAILY2 There was a network connection issue.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
}
if (WaitHandle.WaitAll(allResetEventsSans12, 30000))
{
MDTrace.WriteLine(LogLevel.DEBUG,$"All events have signaled completion.");
running = false; // if all worker threads are done then wrap it up here.
}
}
SMSClient.SendSMSEmail("UPDATEDAILY2 Network monitor ended.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
resetEvents[STAGE_FINAL].Set();
});
// Here is the start of the real workers
ThreadPool.QueueUserWorkItem(delegate
{
UpdatePricesBigCharts(startDate); // bigcharts.marketwatch.com
UpdatePricesYahooSweep(startDate); // The sweep variation of the method is intended to be used after the BigCharts update because the sweep will take pricing_source into consideration when fetching prices.
resetEvents[STAGE_1].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_1 UPDATEPRICESBIGCHARTS/YAHOO done.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_1 complete.");
});
resetEvents[STAGE_1].WaitOne(); // wait for pricing to finish
ThreadPool.QueueUserWorkItem(delegate
{
LoadConsumerPriceIndex(); // Load consumer price index data from Bureau of Labor Statistics
UpdateCompanyProfiles(); // financials.morningstar.com and finance.yahoo.com
LoadGDPPerCapita(); // api.worldbank.org
resetEvents[STAGE_2].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_2 complete. LoadConsumerPriceIndex,UpdateCompanyProfiles,LoadGDPPerCapita ", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_2 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
UpdateYieldCurve(); // www.treasury.gov
resetEvents[STAGE_3].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_3 complete. UpdateYieldCurve", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_3 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
UpdateLatestAnalystRatings(true); // WWW.BRIEFING.COM
UpdateMissingAnalystRatings(); // MARKET BEAT
resetEvents[STAGE_4].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_4 complete. UpdateLatestAnalystRatings, UpdateMissingAnalystRatings", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_4 complete");
});
ThreadPool.QueueUserWorkItem(delegate
{
UpdateSplits(); // eoddata.com
ProcessAllSplits(); // non-network operation
resetEvents[STAGE_5].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_5 complete. UpdateSplits, ProcessAllSplits", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_5 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
LoadInsiderTransactions();
resetEvents[STAGE_6].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_6 complete. LoadInsiderTransactions", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_6 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
UpdateEarningsAnnouncements(); // www.zacks.com
LoadZacksRank(); // www.zacks.com
resetEvents[STAGE_7].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_7 complete. UpdateEarningsAnnouncements,LoadZacksRank ", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_7 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
GetSECFilingsWatchList("Valuations");// www.sec.gov
resetEvents[STAGE_8].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_8 complete. GetSECFilingsWatchList", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_8 complete");
});
ThreadPool.QueueUserWorkItem(delegate
{
UpdateAnalystPriceTarget(); // MarketBeat
GetETFHoldings(); // finance.yahoo.com
resetEvents[STAGE_9].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_9 complete. UpdateAnalystPriceTarget, GetETFHoldings", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_9 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
UpdateAllMissingCIK(); // /www.sec.gov
resetEvents[STAGE_10].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_10 complete. UpdateAllMissingCIK", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_10 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
UpdateDividendHistory(); // www.nasdaq.com DIVIDEND HISTORY IS BROKEN.... WORKING ON THIS ONE.
resetEvents[STAGE_11].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_11 complete. UpdateDividendHistory", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_11 complete.");
});
WaitHandle.WaitAll(resetEvents);
SMSClient.SendSMSEmail("UPDATEDAILY2 All stages complete.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"Done.");
}
public static void UpdatePricesBigCharts(DateTime startDate)
{
PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper();
pricingMarketDataHelper.UpdatePricesBigCharts(startDate);
}
public static void UpdatePricesYahooSweep(DateTime startDate)
{
PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper();
pricingMarketDataHelper.UpdatePricesYahooSweep(startDate);
}
public static void DeletePriceWatchList(String watchListName,String strDate)
{
try
{
List<String> symbols = WatchListDA.GetWatchList(watchListName);
if(null==symbols||0==symbols.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("There are no symbols for watchlist {0}",watchListName));
return;
}
foreach (String symbol in symbols)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Delete price {0} on {1}, watchlist {2}",symbol,strDate,watchListName));
DeletePriceSymbol(symbol, strDate);
}
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
}
}
public static void DeletePriceSymbol(String symbol, String strDate)
{
try
{
DateTime pricingDate = DateTime.Parse(strDate);
MDTrace.WriteLine(LogLevel.DEBUG,"Delete price '" + symbol + "' for " + Utility.DateTimeToStringMMSDDSYYYY(pricingDate));
if(PricingDA.DeletePrice(symbol, pricingDate)) MDTrace.WriteLine(LogLevel.DEBUG,"Ok");
else MDTrace.WriteLine(LogLevel.DEBUG,"Failed.");
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
}
}
public static void LoadConsumerPriceIndex()
{
try
{
MDTrace.WriteLine("[LoadConsumerPriceIndex] started.");
PriceIndices priceIndices=MarketDataHelper.GetConsumerPriceIndices();
if(null==priceIndices)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadConsumerPriceIndex] Failed, check log file for isssues.");
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadConsumerPriceIndex] Got:{0} records from source feed.",priceIndices.Count()));
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadConsumerPriceIndex] Saving...");
ConsumerPriceIndexDA.InsertUpdatePriceIndices(priceIndices);
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadConsumerPriceIndex] Save complete...");
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadConsumerPriceIndex] Exception:{0}",exception.ToString()));
}
}
public static void UpdateCompanyProfiles()
{
try
{
CompanyProfileMarketDataHelper companyProfileMarketDataHelper=new CompanyProfileMarketDataHelper();
companyProfileMarketDataHelper.UpdateCompanyProfiles();
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
}
}
public static void LoadGDPPerCapita()
{
try
{
MDTrace.WriteLine("[LoadGSPPerCapita]");
EconomicIndicators economicIndicators=MarketDataHelper.GetGDPPerCapita();
if(null==economicIndicators||0==economicIndicators.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadGDPPerCapita] No data, see log file for potential issues.");
return;
}
List<String> distinctCountry=(from EconomicIndicator economicIndicator in economicIndicators select economicIndicator.CountryCode).Distinct().ToList();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadGDPPerCapita] Downloaded {0} countries.",distinctCountry.Count));
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadGDPPerCapita] Saving...");
if(EconomicIndicatorDA.InsertUpdateEconomicIndicators(economicIndicators))
{
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadGDPPerCapita] Success.");
}
else
{
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadGDPPerCapita] Failed, check log file for isssues.");
}
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadGDPPerCapita] Exception:{0}",exception.ToString()));
}
}
public static void UpdateYieldCurve() // maintains current year of yield curve data
{
Profiler profiler = new Profiler();
try
{
int year = DateTime.Now.Year;
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving yield curve for year "+year);
YieldCurve yieldCurve=MarketDataHelper.GetYieldCurve(year);
if (null == yieldCurve)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Failed to get YieldCurve for {0}", year));
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,"Got "+yieldCurve.Count+" points for "+year);
YieldCurveDA.InsertOrUpdate(yieldCurve);
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
public static void UpdateLatestAnalystRatings(Boolean createSecurityMaster = true)
{
Profiler profiler = new Profiler();
try
{
if(!CheckRunCriteria())return;
DateGenerator dateGenerator = new DateGenerator();
MDTrace.WriteLine(LogLevel.DEBUG, "Enter");
AnalystRatings analystRatings = MarketDataHelper.GetLatestAnalystRatings();
List<String> symbols = PricingDA.GetSymbolsNotIn(analystRatings.Symbols);
foreach (String symbol in symbols)
{
if (false == createSecurityMaster)
{
MDTrace.WriteLine(LogLevel.DEBUG, "Removing " + symbol + ", missing from security master.");
AnalystRating analystRating = (from item in analystRatings where item.Symbol.Equals(symbol) select item).FirstOrDefault();
if (null != analystRating) MDTrace.WriteLine(LogLevel.DEBUG, analystRating.ToString());
}
}
if (false == createSecurityMaster) analystRatings.Remove(symbols);
foreach (AnalystRating analystRating in analystRatings)
{
String symbolNotFound = (from symbol in symbols where symbol.Equals(analystRating.Symbol) select symbol).FirstOrDefault();
if (null != symbolNotFound && createSecurityMaster)
{
MDTrace.WriteLine(LogLevel.DEBUG, "Adding security '" + analystRating.Symbol + "' '" + analystRating.CompanyName + "'");
if (!PricingDA.AddSecurity(analystRating.Symbol, analystRating.CompanyName)) MDTrace.WriteLine(LogLevel.DEBUG, "Failed to add '" + symbols + "'");
else
{
MDTrace.WriteLine(LogLevel.DEBUG, "Added '" + analystRating.Symbol + "'");
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Loading prices for {0}", analystRating.Symbol));
DateTime pricingDate = dateGenerator.FindPrevBusinessDay(analystRating.Date);
Prices prices = MarketDataHelper.GetDailyPrices(analystRating.Symbol, Constants.MIN_PRICING_DATE, pricingDate); // use the Yahoo JSON bulk feed
if (null != prices)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting {0} prices for {1}", prices.Count, analystRating.Symbol));
PricingDA.InsertPrices(prices);
MDTrace.WriteLine(LogLevel.DEBUG, "Insert done.");
}
}
}
MDTrace.WriteLine(LogLevel.DEBUG,$"Rating {analystRating.ToString()}");
}
AnalystRatingsDA.InsertAnalystRatings(analystRatings);
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception:{exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took{profiler.End}(ms)");
}
}
public static void UpdateMissingAnalystRatings()
{
try
{
DateGenerator dateGenerator = new DateGenerator();
List<String> symbols = PricingDA.GetSymbols();
DateTime maxDate=AnalystRatingsDA.GetMaxDateNoZacks();
foreach (String symbol in symbols)
{
AnalystRatings analystRatings = MarketDataHelper.GetAnalystRatingsMarketBeat(symbol);
if (null == analystRatings || 0 == analystRatings.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("No analyst ratings for {0}", symbol));
continue;
}
analystRatings = new AnalystRatings((from AnalystRating analystRating in analystRatings where analystRating.Date >= maxDate.Date select analystRating).ToList());
AnalystRatings duplicateList = new AnalystRatings();
foreach (AnalystRating analystRating in analystRatings)
{
if (AnalystRatingsDA.ContainsAnalystRating(analystRating))
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Already have analyst rating for {0} on {1} from brokerage firm {2}", analystRating.Symbol, analystRating.Date.ToShortDateString(), analystRating.BrokerageFirm));
duplicateList.Add(analystRating);
continue;
}
}
analystRatings = new AnalystRatings(analystRatings.Except(duplicateList).ToList());
foreach (AnalystRating analystRating in analystRatings)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting Analyst Rating for {0} on {1} from brokerage firm {2}", analystRating.Symbol, analystRating.Date.ToShortDateString(), analystRating.BrokerageFirm));
}
AnalystRatingsDA.InsertAnalystRatings(analystRatings);
}
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG, exception);
}
}
public static void UpdateSplits()
{
MDTrace.WriteLine("Updating splits.");
Splits splits=MarketDataHelper.GetSplits();
if(null==splits||0==splits.Count)return;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} split records.",splits.Count));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Writing to database.",splits.Count));
SplitsDA.InsertSplits(splits);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Done."));
}
public static void ProcessAllSplits()
{
MDTrace.WriteLine("Processing all splits");
Splits splits=SplitsDA.GetUnappliedSplits();
if(null==splits||0==splits.Count)return;
SplitHelper.ProcessSplits(splits);
}
public static void LoadInsiderTransactions()
{
List<String> symbols = PricingDA.GetSymbols();
InsiderTransactionMarketDataHelper insiderTransactionMarketDataHelper = new InsiderTransactionMarketDataHelper();
insiderTransactionMarketDataHelper.LoadInsiderTransactions(symbols);
}
public static void UpdateEarningsAnnouncements(String symbol=null)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[UPDATEEARNINGSANNOUNCEMENTS]");
EarningsAnnouncementsMarketDataHelper earningsAnnouncementsMarketDataHelper=new EarningsAnnouncementsMarketDataHelper();
earningsAnnouncementsMarketDataHelper.UpdateEarningsAnnouncements(symbol);
MDTrace.WriteLine(LogLevel.DEBUG,"[UPDATEEARNINGSANNOUNCEMENTS] Done");
}
public static void LoadZacksRank()
{
ZacksRankMarketDataHelper zacksRankMarketDataHelper=new ZacksRankMarketDataHelper();
zacksRankMarketDataHelper.LoadZacksRank();
}
public static void GetSECFilingsWatchList(String watchListName)
{
List<String> symbols = WatchListDA.GetWatchList(watchListName);
SECFilingMarketDataHelper secFilingMarketDataHelper=new SECFilingMarketDataHelper();
secFilingMarketDataHelper.UpdateSECFilings(symbols);
}
public static void UpdateAnalystPriceTarget()
{
MDTrace.WriteLine(LogLevel.DEBUG,"[UPDATEANALYSTPRICETARGET]");
AnalystPriceTargetMarketDataHelper analystPriceTargetMarketDataHelper=new AnalystPriceTargetMarketDataHelper();
analystPriceTargetMarketDataHelper.LoadAnalystPriceTarget();
MDTrace.WriteLine(LogLevel.DEBUG,"[UPDATEANALYSTPRICETARGET] Done");
}
public static void GetETFHoldings()
{
ETFHoldingsMarketDataHelper etfHoldingsMarketDataHelper=new ETFHoldingsMarketDataHelper();
etfHoldingsMarketDataHelper.LoadETFHoldings();
}
public static void UpdateAllMissingCIK()
{
CIKMarketDataHelper cikMarketDataHelper=new CIKMarketDataHelper();
cikMarketDataHelper.UpdateAllMissingCIK();
}
public static void UpdateDividendHistory()
{
DividendHistoryMarketDataHelper dividendHistoryMarketDataHelper=new DividendHistoryMarketDataHelper();
dividendHistoryMarketDataHelper.UpdateDividendHistory();
}
// **********************************************************************************************************************************************
// ******************************************* E N D U P D A T E D A I L Y 2 M E T H O D S ***************************************************
// **********************************************************************************************************************************************
public static void UpdateAnalystRatings(String paramSymbol = null,DateTime? minDate=null)
{
try
{
if(!CheckRunCriteria())return;
MDTrace.WriteLine(LogLevel.DEBUG, "[UPDATEANALYSTRATINGS]");
List<String> symbols = new List<String>();
if (paramSymbol == null) symbols = PricingDA.GetSymbols();
else symbols.Add(paramSymbol);
foreach (String symbol in symbols)
{
AnalystRatings analystRatings = MarketDataHelper.GetAnalystRatingsMarketBeat(symbol);
if (null == analystRatings || 0 == analystRatings.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("No analyst ratings for {0}", symbol));
continue;
}
AnalystRatings duplicateList = new AnalystRatings();
if (null != minDate)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Minimum Rating Date of {0} is specified.", minDate.Value.ToShortDateString()));
analystRatings = new AnalystRatings((from AnalystRating analystRating in analystRatings where analystRating.Date >= minDate.Value select analystRating).ToList());
}
foreach (AnalystRating analystRating in analystRatings)
{
if (AnalystRatingsDA.ContainsAnalystRating(analystRating))
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Already have analyst rating for {0} on {1} from brokerage firm {2}", analystRating.Symbol, analystRating.Date.ToShortDateString(), analystRating.BrokerageFirm));
duplicateList.Add(analystRating);
continue;
}
}
analystRatings = new AnalystRatings(analystRatings.Except(duplicateList).ToList());
foreach (AnalystRating analystRating in analystRatings)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting Analyst Rating for {0} on {1} from brokerage firm {2}", analystRating.Symbol, analystRating.Date.ToShortDateString(), analystRating.BrokerageFirm));
}
AnalystRatingsDA.InsertAnalystRatings(analystRatings);
}
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG, exception);
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG, "[UPDATEANALYSTRATINGS]");
}
}
public static void UpdateLatestPriceWatchList(String watchListName)
{
try
{
if(!CheckRunCriteria())return;
MDTrace.WriteLine(LogLevel.DEBUG,$"UPDATELATESTPRICEWATCHLIST WATCHLIST:{watchListName}");
List<String> symbols = WatchListDA.GetWatchList(watchListName);
PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper();
pricingMarketDataHelper.UpdateLatestPrices(symbols);
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
}
}
public static void UpdateLatestPriceOpenPositions()
{
try
{
if(!CheckRunCriteria())return;
MDTrace.WriteLine(LogLevel.DEBUG,$"UPDATELATESTPRICEOPENPOSITIONS");
PortfolioTrades portfolioTrades=PortfolioDA.GetOpenTrades();
List<String> symbols=(from PortfolioTrade portfolioTrade in portfolioTrades select portfolioTrade.Symbol).Distinct().ToList();
if(symbols.Any(x=>x.Equals("SPY")))symbols.Add("SH");
PricingMarketDataHelper pricingMarketDataHelper=new PricingMarketDataHelper();
pricingMarketDataHelper.UpdateLatestPrices(symbols);
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
}
}
// **********************************************************************************************************************************************
public static void LoadHeadlinesWatchList(String watchList)
{
if(!CheckRunCriteria())return;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("LOADHEADLINESWATCHLIST {0}",watchList));
List<String> symbols = WatchListDA.GetWatchList(watchList);
HeadlinesMarketDataHelper headlinesMarketDataHelper= new HeadlinesMarketDataHelper();
headlinesMarketDataHelper.LoadHeadlines(symbols);
}
private static void LoadPremarketData()
{
int retries=3;
int sleepTime=2000;
PremarketElements premarketElements=null;
if(!CheckRunCriteria())return;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("LOADPREMARKETDATA"));
for(int retry=0;retry<retries && (null==premarketElements || 0==premarketElements.Count);retry++,Thread.Sleep(sleepTime))
{
premarketElements=MarketDataHelper.GetPremarketData();
}
if(null==premarketElements||0==premarketElements.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Failed to retrieve premarket data."));
return;
}
PremarketDA.AddElements(premarketElements);
foreach(PremarketElement premarketElement in premarketElements)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} {1} {2}",premarketElement.Market,Utility.FormatNumber(premarketElement.ChangeValue),Utility.FormatPercent(premarketElement.ChangePercent/100.00)));
}
}
public static bool CheckRunCriteria()
{
DateGenerator dateGenerator = new DateGenerator();
DateTime currentDate = DateTime.Now;
if(!dateGenerator.IsMarketOpen(currentDate))
{
String description="";
if(HolidayDA.IsMarketHoliday(currentDate)) description=HolidayDA.GetHolidayDescription(currentDate);
else description=Utility.DayOfWeekToString(currentDate.DayOfWeek);
MDTrace.WriteLine(LogLevel.DEBUG,$"Market is closed today {currentDate.ToShortDateString()} for {description}.");
return false;
}
if(!NetworkStatus.IsInternetConnected())
{
MDTrace.WriteLine(LogLevel.DEBUG,$"The internet is not connected.");
return false;
}
return true;
}
}
}