Sean/ARM64
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Add more logging around UPDATEDAILY2 events.

Browse Source
This commit is contained in:
Sean
2025-04-10 18:40:43 -04:00
parent c3729551a8
commit bc117bf297
2 changed files with 71 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
MarketData/MarketData/Services/MainService.cs
View File

@@ -20,9 +20,9 @@ namespace MarketData.Services
/// </summary>
public static void DisplayUsage()
{
MDTrace.WriteLine(LogLevel.DEBUG,$"USAGE");
MDTrace.WriteLine(LogLevel.DEBUG,$"LOADHEADLINESWATCHLIST");
MDTrace.WriteLine(LogLevel.DEBUG,$"LOADPREMARKETDATA");
MDTrace.WriteLine(LogLevel.DEBUG,"USAGE");
MDTrace.WriteLine(LogLevel.DEBUG,"LOADHEADLINESWATCHLIST");
MDTrace.WriteLine(LogLevel.DEBUG,"LOADPREMARKETDATA");
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEDAILY2 /DATE:");
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTPRICEOPENPOSITIONS");
MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTPRICEWATCHLIST /WATCHLIST:");
@@ -264,7 +264,7 @@ namespace MarketData.Services
// Sanity check. If the given date is not today then ask the user to confirm
if (currentDate != startDate.Date)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Run date is not today: Current Date:{currentDate.ToShortDateString()} Run Date: {startDate.ToShortDateString()}");
MDTrace.WriteLine(LogLevel.DEBUG,ConsoleColor.Red,$"Run date is not today: Current Date:{currentDate.ToShortDateString()} Run Date: {startDate.ToShortDateString()}");
return;
}
@@ -273,7 +273,7 @@ namespace MarketData.Services
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,$"UPDATEDAILY2 DATE:{startDate.ToShortDateString()}");
int STAGE_1=0,STAGE_2=1,STAGE_3=2,STAGE_4=3,STAGE_5=4,STAGE_6=5,STAGE_7=6,STAGE_8=7,STAGE_9=8,STAGE_10=9,STAGE_11=10,STAGE_12=11,STAGE_FINAL=12;
int STAGE_1=0,STAGE_2=1,STAGE_3=2,STAGE_4=3,STAGE_5=4,STAGE_6=5,STAGE_7=6,STAGE_8=7,STAGE_9=8,STAGE_10=9,STAGE_11=10,STAGE_FINAL=11;
DeletePriceWatchList("valuations",startDate.ToShortDateString());
DeletePriceWatchList("Momentum",startDate.ToShortDateString());
@@ -293,22 +293,27 @@ namespace MarketData.Services
bool running = true;
bool connectionIssue = false;
SMSClient.SendSMSEmail("UPDATEDAILY2 Network monitor activated", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
SMSClient.SendSMSEmail("UPDATEDAILY2 Network monitor activated.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
while (running)
{
if (!NetworkStatus.IsInternetConnected())
{
connectionIssue = true;
SMSClient.SendSMSEmail("UPDATEDAILY2 Network is not connected", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
SMSClient.SendSMSEmail("UPDATEDAILY2 Network is not connected.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
}
else if (connectionIssue)
{
connectionIssue = false;
SMSClient.SendSMSEmail("UPDATEDAILY2 There was a network connection issue.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
}
if (WaitHandle.WaitAll(allResetEventsSans12, 30000)) running = false; // if all worker threads are done then wrap it up here.
if (WaitHandle.WaitAll(allResetEventsSans12, 30000))
{
MDTrace.WriteLine(LogLevel.DEBUG,$"All events have signaled completion.");
running = false; // if all worker threads are done then wrap it up here.
}
}
SMSClient.SendSMSEmail("UPDATEDAILY2 Network monitor ended", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
SMSClient.SendSMSEmail("UPDATEDAILY2 Network monitor ended.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
resetEvents[STAGE_FINAL].Set();
});
// Here is the start of the real workers
@@ -318,6 +323,7 @@ namespace MarketData.Services
UpdatePricesYahooSweep(startDate); // The sweep variation of the method is intended to be used after the BigCharts update because the sweep will take pricing_source into consideration when fetching prices.
resetEvents[STAGE_1].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 UPDATEPRICESBIGCHARTS/YAHOO done.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_1 complete.");
});
resetEvents[STAGE_1].WaitOne(); // wait for pricing to finish
@@ -327,12 +333,16 @@ namespace MarketData.Services
UpdateCompanyProfiles(); // financials.morningstar.com and finance.yahoo.com
LoadGDPPerCapita(); // api.worldbank.org
resetEvents[STAGE_2].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_2 complete. LoadConsumerPriceIndex,UpdateCompanyProfiles,LoadGDPPerCapita ", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_2 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
UpdateYieldCurve(); // www.treasury.gov
resetEvents[STAGE_3].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_3 complete. UpdateYieldCurve", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_3 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
@@ -340,6 +350,8 @@ namespace MarketData.Services
UpdateLatestAnalystRatings(true); // WWW.BRIEFING.COM
UpdateMissingAnalystRatings(); // MARKET BEAT
resetEvents[STAGE_4].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_4 complete. UpdateLatestAnalystRatings, UpdateMissingAnalystRatings", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_4 complete");
});
ThreadPool.QueueUserWorkItem(delegate
@@ -347,12 +359,16 @@ namespace MarketData.Services
UpdateSplits(); // eoddata.com
ProcessAllSplits(); // non-network operation
resetEvents[STAGE_5].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_5 complete. UpdateSplits, ProcessAllSplits", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_5 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
LoadInsiderTransactions();
resetEvents[STAGE_6].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_6 complete. LoadInsiderTransactions", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_6 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
@@ -360,12 +376,16 @@ namespace MarketData.Services
UpdateEarningsAnnouncements(); // www.zacks.com
LoadZacksRank(); // www.zacks.com
resetEvents[STAGE_7].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_7 complete. UpdateEarningsAnnouncements,LoadZacksRank ", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_7 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
GetSECFilingsWatchList("Valuations");// www.sec.gov
resetEvents[STAGE_8].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_8 complete. GetSECFilingsWatchList", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_8 complete");
});
ThreadPool.QueueUserWorkItem(delegate
@@ -373,26 +393,29 @@ namespace MarketData.Services
UpdateAnalystPriceTarget(); // MarketBeat
GetETFHoldings(); // finance.yahoo.com
resetEvents[STAGE_9].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_9 complete. UpdateAnalystPriceTarget, GetETFHoldings", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_9 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
UpdateAllMissingCIK(); // /www.sec.gov
resetEvents[STAGE_10].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_10 complete. UpdateAllMissingCIK", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_10 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
UpdateDividendHistory(); // www.nasdaq.com DIVIDEND HISTORY IS BROKEN.... WORKING ON THIS ONE.
resetEvents[STAGE_11].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_11 complete. UpdateDividendHistory", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_11 complete.");
});
ThreadPool.QueueUserWorkItem(delegate
{
resetEvents[STAGE_12].Set();
});
WaitHandle.WaitAll(resetEvents);
SMSClient.SendSMSEmail("UPDATEDAILY2 All stages complete.", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"Done.");
}
public static void UpdatePricesBigCharts(DateTime startDate)
@@ -510,25 +533,38 @@ namespace MarketData.Services
public static void UpdateYieldCurve() // maintains current year of yield curve data
{
int year = DateTime.Now.Year;
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving yield curve for year "+year);
YieldCurve yieldCurve=MarketDataHelper.GetYieldCurve(year);
if (null == yieldCurve)
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Failed to get YieldCurve for {0}", year));
return;
int year = DateTime.Now.Year;
MDTrace.WriteLine(LogLevel.DEBUG,"Retrieving yield curve for year "+year);
YieldCurve yieldCurve=MarketDataHelper.GetYieldCurve(year);
if (null == yieldCurve)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Failed to get YieldCurve for {0}", year));
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,"Got "+yieldCurve.Count+" points for "+year);
YieldCurveDA.InsertOrUpdate(yieldCurve);
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
MDTrace.WriteLine(LogLevel.DEBUG,"got "+yieldCurve.Count+" points for "+year);
YieldCurveDA.InsertOrUpdate(yieldCurve);
}
public static void UpdateLatestAnalystRatings(Boolean createSecurityMaster = true)
{
Profiler profiler = new Profiler();
try
{
if(!CheckRunCriteria())return;
DateGenerator dateGenerator = new DateGenerator();
MDTrace.WriteLine(LogLevel.DEBUG, "[UPDATELATESTANALYSTRATINGS]");
MDTrace.WriteLine(LogLevel.DEBUG, "Enter");
AnalystRatings analystRatings = MarketDataHelper.GetLatestAnalystRatings();
List<String> symbols = PricingDA.GetSymbolsNotIn(analystRatings.Symbols);
foreach (String symbol in symbols)
@@ -540,7 +576,6 @@ namespace MarketData.Services
if (null != analystRating) MDTrace.WriteLine(LogLevel.DEBUG, analystRating.ToString());
}
}
MDTrace.WriteLine(LogLevel.DEBUG, "*************************");
if (false == createSecurityMaster) analystRatings.Remove(symbols);
foreach (AnalystRating analystRating in analystRatings)
{
@@ -559,17 +594,21 @@ namespace MarketData.Services
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Inserting {0} prices for {1}", prices.Count, analystRating.Symbol));
PricingDA.InsertPrices(prices);
MDTrace.WriteLine(LogLevel.DEBUG, "Done.");
MDTrace.WriteLine(LogLevel.DEBUG, "Insert done.");
}
}
}
MDTrace.WriteLine(LogLevel.DEBUG, analystRating.ToString());
MDTrace.WriteLine(LogLevel.DEBUG,$"Rating {analystRating.ToString()}");
}
AnalystRatingsDA.InsertAnalystRatings(analystRatings);
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG, exception);
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception:{exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took{profiler.End}(ms)");
}
}

16
MarketData/MarketDataLib/MDTrace.cs
View File

@@ -40,7 +40,7 @@ namespace MarketData
public static void Write(LogLevel logLevel,string message)
{
if(MDTrace.logLevel<logLevel)return;
Trace.Write(GetCallerIP()+GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+GetMethodInfo()+message);
Trace.Write(GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+GetMethodInfo()+message);
Flush();
}
@@ -49,7 +49,7 @@ namespace MarketData
/// <returns>void</returns>
public static void WriteLine(string message)
{
WriteLine(LogLevel.DEBUG,GetCallerIP()+GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+message);
WriteLine(LogLevel.DEBUG,GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+message);
Console.WriteLine(message);
}
/// <summary>WriteLine - Writes a line of text to trace log.</summary>
@@ -58,7 +58,7 @@ namespace MarketData
public static void WriteLine(LogLevel logLevel,Exception exception)
{
if(MDTrace.logLevel<logLevel)return;
Trace.WriteLine(GetCallerIP()+GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+exception.ToString());
Trace.WriteLine(GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+exception.ToString());
Console.WriteLine(exception);
Flush();
}
@@ -68,7 +68,7 @@ namespace MarketData
public static void WriteLine(LogLevel logLevel,string message)
{
if(MDTrace.logLevel<logLevel)return;
Trace.WriteLine(GetCallerIP()+GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+message);
Trace.WriteLine(GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+message);
Console.WriteLine(message);
Flush();
}
@@ -79,7 +79,7 @@ namespace MarketData
public static void WriteLine(LogLevel logLevel,ConsoleColor consoleColor,string message)
{
if(MDTrace.logLevel<logLevel)return;
Trace.WriteLine(GetCallerIP()+GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+message);
Trace.WriteLine(GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+message);
ConsoleColor currentColor=Console.ForegroundColor;
Console.ForegroundColor = consoleColor;
Console.WriteLine(message);
@@ -147,11 +147,5 @@ namespace MarketData
sb.Append(")]");
return sb.ToString();
}
/// <summary>GetCallerIP - Returns the calling methods IP address.</summary>
/// <returns>String</returns>
private static String GetCallerIP()
{
return "[LOCAL]";
}
}
}