From bc117bf2978edc0a37108e98ae60428ed729da70 Mon Sep 17 00:00:00 2001 From: Sean Date: Thu, 10 Apr 2025 18:40:43 -0400 Subject: [PATCH] Add more logging around UPDATEDAILY2 events. --- MarketData/MarketData/Services/MainService.cs | 93 +++++++++++++------ MarketData/MarketDataLib/MDTrace.cs | 16 +--- 2 files changed, 71 insertions(+), 38 deletions(-) diff --git a/MarketData/MarketData/Services/MainService.cs b/MarketData/MarketData/Services/MainService.cs index 6411062..afda3e6 100755 --- a/MarketData/MarketData/Services/MainService.cs +++ b/MarketData/MarketData/Services/MainService.cs @@ -20,9 +20,9 @@ namespace MarketData.Services /// public static void DisplayUsage() { - MDTrace.WriteLine(LogLevel.DEBUG,$"USAGE"); - MDTrace.WriteLine(LogLevel.DEBUG,$"LOADHEADLINESWATCHLIST"); - MDTrace.WriteLine(LogLevel.DEBUG,$"LOADPREMARKETDATA"); + MDTrace.WriteLine(LogLevel.DEBUG,"USAGE"); + MDTrace.WriteLine(LogLevel.DEBUG,"LOADHEADLINESWATCHLIST"); + MDTrace.WriteLine(LogLevel.DEBUG,"LOADPREMARKETDATA"); MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEDAILY2 /DATE:"); MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTPRICEOPENPOSITIONS"); MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTPRICEWATCHLIST /WATCHLIST:"); @@ -264,7 +264,7 @@ namespace MarketData.Services // Sanity check. If the given date is not today then ask the user to confirm if (currentDate != startDate.Date) { - MDTrace.WriteLine(LogLevel.DEBUG,$"Run date is not today: Current Date:{currentDate.ToShortDateString()} Run Date: {startDate.ToShortDateString()}"); + MDTrace.WriteLine(LogLevel.DEBUG,ConsoleColor.Red,$"Run date is not today: Current Date:{currentDate.ToShortDateString()} Run Date: {startDate.ToShortDateString()}"); return; } @@ -273,7 +273,7 @@ namespace MarketData.Services return; } MDTrace.WriteLine(LogLevel.DEBUG,$"UPDATEDAILY2 DATE:{startDate.ToShortDateString()}"); - int STAGE_1=0,STAGE_2=1,STAGE_3=2,STAGE_4=3,STAGE_5=4,STAGE_6=5,STAGE_7=6,STAGE_8=7,STAGE_9=8,STAGE_10=9,STAGE_11=10,STAGE_12=11,STAGE_FINAL=12; + int STAGE_1=0,STAGE_2=1,STAGE_3=2,STAGE_4=3,STAGE_5=4,STAGE_6=5,STAGE_7=6,STAGE_8=7,STAGE_9=8,STAGE_10=9,STAGE_11=10,STAGE_FINAL=11; DeletePriceWatchList("valuations",startDate.ToShortDateString()); DeletePriceWatchList("Momentum",startDate.ToShortDateString()); @@ -293,22 +293,27 @@ namespace MarketData.Services bool running = true; bool connectionIssue = false; - SMSClient.SendSMSEmail("UPDATEDAILY2 Network monitor activated", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + SMSClient.SendSMSEmail("UPDATEDAILY2 Network monitor activated.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); while (running) { if (!NetworkStatus.IsInternetConnected()) { connectionIssue = true; - SMSClient.SendSMSEmail("UPDATEDAILY2 Network is not connected", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + SMSClient.SendSMSEmail("UPDATEDAILY2 Network is not connected.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); } else if (connectionIssue) { connectionIssue = false; SMSClient.SendSMSEmail("UPDATEDAILY2 There was a network connection issue.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); } - if (WaitHandle.WaitAll(allResetEventsSans12, 30000)) running = false; // if all worker threads are done then wrap it up here. + + if (WaitHandle.WaitAll(allResetEventsSans12, 30000)) + { + MDTrace.WriteLine(LogLevel.DEBUG,$"All events have signaled completion."); + running = false; // if all worker threads are done then wrap it up here. + } } - SMSClient.SendSMSEmail("UPDATEDAILY2 Network monitor ended", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + SMSClient.SendSMSEmail("UPDATEDAILY2 Network monitor ended.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); resetEvents[STAGE_FINAL].Set(); }); // Here is the start of the real workers @@ -318,6 +323,7 @@ namespace MarketData.Services UpdatePricesYahooSweep(startDate); // The sweep variation of the method is intended to be used after the BigCharts update because the sweep will take pricing_source into consideration when fetching prices. resetEvents[STAGE_1].Set(); SMSClient.SendSMSEmail("UPDATEDAILY2 UPDATEPRICESBIGCHARTS/YAHOO done.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_1 complete."); }); resetEvents[STAGE_1].WaitOne(); // wait for pricing to finish @@ -327,12 +333,16 @@ namespace MarketData.Services UpdateCompanyProfiles(); // financials.morningstar.com and finance.yahoo.com LoadGDPPerCapita(); // api.worldbank.org resetEvents[STAGE_2].Set(); + SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_2 complete. LoadConsumerPriceIndex,UpdateCompanyProfiles,LoadGDPPerCapita ", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_2 complete."); }); ThreadPool.QueueUserWorkItem(delegate { UpdateYieldCurve(); // www.treasury.gov resetEvents[STAGE_3].Set(); + SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_3 complete. UpdateYieldCurve", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_3 complete."); }); ThreadPool.QueueUserWorkItem(delegate @@ -340,6 +350,8 @@ namespace MarketData.Services UpdateLatestAnalystRatings(true); // WWW.BRIEFING.COM UpdateMissingAnalystRatings(); // MARKET BEAT resetEvents[STAGE_4].Set(); + SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_4 complete. UpdateLatestAnalystRatings, UpdateMissingAnalystRatings", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_4 complete"); }); ThreadPool.QueueUserWorkItem(delegate @@ -347,12 +359,16 @@ namespace MarketData.Services UpdateSplits(); // eoddata.com ProcessAllSplits(); // non-network operation resetEvents[STAGE_5].Set(); + SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_5 complete. UpdateSplits, ProcessAllSplits", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_5 complete."); }); ThreadPool.QueueUserWorkItem(delegate { LoadInsiderTransactions(); resetEvents[STAGE_6].Set(); + SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_6 complete. LoadInsiderTransactions", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_6 complete."); }); ThreadPool.QueueUserWorkItem(delegate @@ -360,12 +376,16 @@ namespace MarketData.Services UpdateEarningsAnnouncements(); // www.zacks.com LoadZacksRank(); // www.zacks.com resetEvents[STAGE_7].Set(); + SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_7 complete. UpdateEarningsAnnouncements,LoadZacksRank ", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_7 complete."); }); ThreadPool.QueueUserWorkItem(delegate { GetSECFilingsWatchList("Valuations");// www.sec.gov resetEvents[STAGE_8].Set(); + SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_8 complete. GetSECFilingsWatchList", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_8 complete"); }); ThreadPool.QueueUserWorkItem(delegate @@ -373,26 +393,29 @@ namespace MarketData.Services UpdateAnalystPriceTarget(); // MarketBeat GetETFHoldings(); // finance.yahoo.com resetEvents[STAGE_9].Set(); + SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_9 complete. UpdateAnalystPriceTarget, GetETFHoldings", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_9 complete."); }); ThreadPool.QueueUserWorkItem(delegate { UpdateAllMissingCIK(); // /www.sec.gov resetEvents[STAGE_10].Set(); + SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_10 complete. UpdateAllMissingCIK", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_10 complete."); }); ThreadPool.QueueUserWorkItem(delegate { UpdateDividendHistory(); // www.nasdaq.com DIVIDEND HISTORY IS BROKEN.... WORKING ON THIS ONE. resetEvents[STAGE_11].Set(); + SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_11 complete. UpdateDividendHistory", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_11 complete."); }); - ThreadPool.QueueUserWorkItem(delegate - { - resetEvents[STAGE_12].Set(); - }); - WaitHandle.WaitAll(resetEvents); + SMSClient.SendSMSEmail("UPDATEDAILY2 All stages complete.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done."); } public static void UpdatePricesBigCharts(DateTime startDate) @@ -510,25 +533,38 @@ namespace MarketData.Services public static void UpdateYieldCurve() // maintains current year of yield curve data { - int year = DateTime.Now.Year; - MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving yield curve for year "+year); - YieldCurve yieldCurve=MarketDataHelper.GetYieldCurve(year); - if (null == yieldCurve) + Profiler profiler = new Profiler(); + try { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Failed to get YieldCurve for {0}", year)); - return; + int year = DateTime.Now.Year; + MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving yield curve for year "+year); + YieldCurve yieldCurve=MarketDataHelper.GetYieldCurve(year); + if (null == yieldCurve) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Failed to get YieldCurve for {0}", year)); + return; + } + MDTrace.WriteLine(LogLevel.DEBUG,"Got "+yieldCurve.Count+" points for "+year); + YieldCurveDA.InsertOrUpdate(yieldCurve); + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}"); + } + finally + { + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)"); } - MDTrace.WriteLine(LogLevel.DEBUG,"got "+yieldCurve.Count+" points for "+year); - YieldCurveDA.InsertOrUpdate(yieldCurve); } public static void UpdateLatestAnalystRatings(Boolean createSecurityMaster = true) { + Profiler profiler = new Profiler(); try { if(!CheckRunCriteria())return; DateGenerator dateGenerator = new DateGenerator(); - MDTrace.WriteLine(LogLevel.DEBUG, "[UPDATELATESTANALYSTRATINGS]"); + MDTrace.WriteLine(LogLevel.DEBUG, "Enter"); AnalystRatings analystRatings = MarketDataHelper.GetLatestAnalystRatings(); List symbols = PricingDA.GetSymbolsNotIn(analystRatings.Symbols); foreach (String symbol in symbols) @@ -540,7 +576,6 @@ namespace MarketData.Services if (null != analystRating) MDTrace.WriteLine(LogLevel.DEBUG, analystRating.ToString()); } } - MDTrace.WriteLine(LogLevel.DEBUG, "*************************"); if (false == createSecurityMaster) analystRatings.Remove(symbols); foreach (AnalystRating analystRating in analystRatings) { @@ -559,17 +594,21 @@ namespace MarketData.Services { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting {0} prices for {1}", prices.Count, analystRating.Symbol)); PricingDA.InsertPrices(prices); - MDTrace.WriteLine(LogLevel.DEBUG, "Done."); + MDTrace.WriteLine(LogLevel.DEBUG, "Insert done."); } } } - MDTrace.WriteLine(LogLevel.DEBUG, analystRating.ToString()); + MDTrace.WriteLine(LogLevel.DEBUG,$"Rating {analystRating.ToString()}"); } AnalystRatingsDA.InsertAnalystRatings(analystRatings); } catch (Exception exception) { - MDTrace.WriteLine(LogLevel.DEBUG, exception); + MDTrace.WriteLine(LogLevel.DEBUG,$"Exception:{exception.ToString()}"); + } + finally + { + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took{profiler.End}(ms)"); } } diff --git a/MarketData/MarketDataLib/MDTrace.cs b/MarketData/MarketDataLib/MDTrace.cs index 81e25f8..4c5c6c4 100755 --- a/MarketData/MarketDataLib/MDTrace.cs +++ b/MarketData/MarketDataLib/MDTrace.cs @@ -40,7 +40,7 @@ namespace MarketData public static void Write(LogLevel logLevel,string message) { if(MDTrace.logLevelvoid public static void WriteLine(string message) { - WriteLine(LogLevel.DEBUG,GetCallerIP()+GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+message); + WriteLine(LogLevel.DEBUG,GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+message); Console.WriteLine(message); } /// WriteLine - Writes a line of text to trace log. @@ -58,7 +58,7 @@ namespace MarketData public static void WriteLine(LogLevel logLevel,Exception exception) { if(MDTrace.logLevelGetCallerIP - Returns the calling methods IP address. -/// String - private static String GetCallerIP() - { - return "[LOCAL]"; - } } }