Sean/ARM64
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Logging changes.

Browse Source
This commit is contained in:
Sean
2025-04-10 22:32:10 -04:00
parent f9365049e2
commit 98cab516fb
5 changed files with 235 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
MarketData/MarketData/Services/MainService.cs
View File

@@ -329,9 +329,7 @@ namespace MarketData.Services
ThreadPool.QueueUserWorkItem(delegate
{
LoadConsumerPriceIndex(); // Load consumer price index data from Bureau of Labor Statistics
UpdateCompanyProfiles(); // financials.morningstar.com and finance.yahoo.com
LoadGDPPerCapita(); // api.worldbank.org
resetEvents[STAGE_2].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_2 complete. LoadConsumerPriceIndex,UpdateCompanyProfiles,LoadGDPPerCapita ", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_2 complete.");
@@ -340,6 +338,8 @@ namespace MarketData.Services
ThreadPool.QueueUserWorkItem(delegate
{
UpdateYieldCurve(); // www.treasury.gov
LoadGDPPerCapita(); // api.worldbank.org
LoadConsumerPriceIndex(); // Load consumer price index data from Bureau of Labor Statistics
resetEvents[STAGE_3].Set();
SMSClient.SendSMSEmail("UPDATEDAILY2 STAGE_3 complete. UpdateYieldCurve", smsUserName, smsRecipients, smsSMTPAddress, smsUserName, smsPassword);
MDTrace.WriteLine(LogLevel.DEBUG,$"STAGE_3 complete.");
@@ -469,65 +469,81 @@ namespace MarketData.Services
public static void LoadConsumerPriceIndex()
{
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine("[LoadConsumerPriceIndex] started.");
MDTrace.WriteLine("Started.");
PriceIndices priceIndices=MarketDataHelper.GetConsumerPriceIndices();
if(null==priceIndices)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadConsumerPriceIndex] Failed, check log file for isssues.");
MDTrace.WriteLine(LogLevel.DEBUG,"Failed, check log file for isssues.");
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadConsumerPriceIndex] Got:{0} records from source feed.",priceIndices.Count()));
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadConsumerPriceIndex] Saving...");
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got:{0} records from source feed.",priceIndices.Count()));
MDTrace.WriteLine(LogLevel.DEBUG,"Saving...");
ConsumerPriceIndexDA.InsertUpdatePriceIndices(priceIndices);
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadConsumerPriceIndex] Save complete...");
MDTrace.WriteLine(LogLevel.DEBUG,"Save complete...");
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadConsumerPriceIndex] Exception:{0}",exception.ToString()));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
public static void UpdateCompanyProfiles()
{
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Started.");
CompanyProfileMarketDataHelper companyProfileMarketDataHelper=new CompanyProfileMarketDataHelper();
companyProfileMarketDataHelper.UpdateCompanyProfiles();
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms).");
}
}
public static void LoadGDPPerCapita()
{
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine("[LoadGSPPerCapita]");
MDTrace.WriteLine("Started.");
EconomicIndicators economicIndicators=MarketDataHelper.GetGDPPerCapita();
if(null==economicIndicators||0==economicIndicators.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadGDPPerCapita] No data, see log file for potential issues.");
MDTrace.WriteLine(LogLevel.DEBUG,"No data, see log file for potential issues.");
return;
}
List<String> distinctCountry=(from EconomicIndicator economicIndicator in economicIndicators select economicIndicator.CountryCode).Distinct().ToList();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadGDPPerCapita] Downloaded {0} countries.",distinctCountry.Count));
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadGDPPerCapita] Saving...");
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Downloaded {0} countries.",distinctCountry.Count));
MDTrace.WriteLine(LogLevel.DEBUG,"Saving...");
if(EconomicIndicatorDA.InsertUpdateEconomicIndicators(economicIndicators))
{
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadGDPPerCapita] Success.");
MDTrace.WriteLine(LogLevel.DEBUG,$"Save complete.");
}
else
{
MDTrace.WriteLine(LogLevel.DEBUG,"[LoadGDPPerCapita] Failed, check log file for isssues.");
MDTrace.WriteLine(LogLevel.DEBUG,"Failed to save economic indicators.");
}
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadGDPPerCapita] Exception:{0}",exception.ToString()));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
@@ -564,7 +580,7 @@ namespace MarketData.Services
{
if(!CheckRunCriteria())return;
DateGenerator dateGenerator = new DateGenerator();
MDTrace.WriteLine(LogLevel.DEBUG, "Enter");
MDTrace.WriteLine(LogLevel.DEBUG, "Started.");
AnalystRatings analystRatings = MarketDataHelper.GetLatestAnalystRatings();
List<String> symbols = PricingDA.GetSymbolsNotIn(analystRatings.Symbols);
foreach (String symbol in symbols)
@@ -614,6 +630,7 @@ namespace MarketData.Services
public static void UpdateMissingAnalystRatings()
{
Profiler profiler = new Profiler();
try
{
DateGenerator dateGenerator = new DateGenerator();
@@ -648,11 +665,18 @@ namespace MarketData.Services
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG, exception);
MDTrace.WriteLine(LogLevel.DEBUG, $"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
public static void UpdateSplits()
{
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine("Updating splits.");
Splits splits=MarketDataHelper.GetSplits();
@@ -662,68 +686,188 @@ namespace MarketData.Services
SplitsDA.InsertSplits(splits);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Done."));
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
public static void ProcessAllSplits()
{
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine("Processing all splits");
Splits splits=SplitsDA.GetUnappliedSplits();
if(null==splits||0==splits.Count)return;
SplitHelper.ProcessSplits(splits);
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
public static void LoadInsiderTransactions()
{
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
List<String> symbols = PricingDA.GetSymbols();
InsiderTransactionMarketDataHelper insiderTransactionMarketDataHelper = new InsiderTransactionMarketDataHelper();
insiderTransactionMarketDataHelper.LoadInsiderTransactions(symbols);
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
public static void UpdateEarningsAnnouncements(String symbol=null)
{
MDTrace.WriteLine(LogLevel.DEBUG,"[UPDATEEARNINGSANNOUNCEMENTS]");
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
EarningsAnnouncementsMarketDataHelper earningsAnnouncementsMarketDataHelper=new EarningsAnnouncementsMarketDataHelper();
earningsAnnouncementsMarketDataHelper.UpdateEarningsAnnouncements(symbol);
MDTrace.WriteLine(LogLevel.DEBUG,"[UPDATEEARNINGSANNOUNCEMENTS] Done");
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
public static void LoadZacksRank()
{
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
ZacksRankMarketDataHelper zacksRankMarketDataHelper=new ZacksRankMarketDataHelper();
zacksRankMarketDataHelper.LoadZacksRank();
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
public static void GetSECFilingsWatchList(String watchListName)
{
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
List<String> symbols = WatchListDA.GetWatchList(watchListName);
SECFilingMarketDataHelper secFilingMarketDataHelper=new SECFilingMarketDataHelper();
secFilingMarketDataHelper.UpdateSECFilings(symbols);
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
public static void UpdateAnalystPriceTarget()
{
MDTrace.WriteLine(LogLevel.DEBUG,"[UPDATEANALYSTPRICETARGET]");
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
AnalystPriceTargetMarketDataHelper analystPriceTargetMarketDataHelper=new AnalystPriceTargetMarketDataHelper();
analystPriceTargetMarketDataHelper.LoadAnalystPriceTarget();
MDTrace.WriteLine(LogLevel.DEBUG,"[UPDATEANALYSTPRICETARGET] Done");
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
public static void GetETFHoldings()
{
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
ETFHoldingsMarketDataHelper etfHoldingsMarketDataHelper=new ETFHoldingsMarketDataHelper();
etfHoldingsMarketDataHelper.LoadETFHoldings();
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
public static void UpdateAllMissingCIK()
{
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
CIKMarketDataHelper cikMarketDataHelper=new CIKMarketDataHelper();
cikMarketDataHelper.UpdateAllMissingCIK();
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
public static void UpdateDividendHistory()
{
Profiler profiler = new Profiler();
try
{
DividendHistoryMarketDataHelper dividendHistoryMarketDataHelper=new DividendHistoryMarketDataHelper();
dividendHistoryMarketDataHelper.UpdateDividendHistory();
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Exception: {exception.ToString()}");
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
}
}
// **********************************************************************************************************************************************
// ******************************************* E N D U P D A T E D A I L Y 2 M E T H O D S ***************************************************

4
MarketData/MarketDataLib/Helper/CompanyProfileMarketDataHelper.cs
View File

@@ -50,7 +50,7 @@ namespace MarketData.Helper
Index=-1;
ManualResetEvent[] resetEvents = new ManualResetEvent[MaxThreads];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)resetEvents[eventIndex] = new ManualResetEvent(true);
MDTrace.WriteLine(String.Format("[UpdateCompanyProfile] Queuing company profile fetches ..."));
MDTrace.WriteLine(String.Format("Queuing company profile fetches ..."));
DateTime modified=DateTime.Now;
while (true)
{
@@ -58,7 +58,7 @@ namespace MarketData.Helper
ManualResetEvent[] busyEvents=GetBusyEvents(resetEvents);
if (null == PeekQueueItem() && 0==busyEvents.Length)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdateCompanyProfile] queue contains {0} items, busy events {1}, all done.",0,busyEvents.Length));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Queue contains {0} items, busy events {1}, all done.",0,busyEvents.Length));
break;
}
for (int index = 0; index < availableEvents.Length; index++)

16
MarketData/MarketDataLib/Helper/MarketDataHelperBase.cs
View File

@@ -62,8 +62,8 @@ namespace MarketData.Helper
{
ManualResetEvent[] busyEvents=GetBusyEvents(events);
if(busyEvents.Length!=0)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Resizing event queue to {0}",busyEvents.Length));
{ // The following trace will skip 3 frames to show the caller of this method in the log
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Resizing event queue to {0}",busyEvents.Length),3);
ManualResetEvent[] resizedEvents=new ManualResetEvent[busyEvents.Length];
Array.Copy(busyEvents, resizedEvents, busyEvents.Length);
return resizedEvents;
@@ -77,12 +77,12 @@ namespace MarketData.Helper
for (int index = 0; index < resetEvents.Length; index++)
{
if (!resetEvents[index].WaitOne(0))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Event {0} is still doing work",resetEvents[index].SafeWaitHandle.DangerousGetHandle().ToString()));
{ // Notice the magic number 3 at the end of the trace. The is the skip frames because we want the caller of this method to show up in the log
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Event {0} is still doing work",resetEvents[index].SafeWaitHandle.DangerousGetHandle().ToString()),3);
availableEvents.Add(resetEvents[index]);
}
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The event queue has {0} busy slots",availableEvents.Count));
} // The following trace will skip 3 frames to show the caller of this method in the log
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The event queue has {0} busy slots",availableEvents.Count),3);
return availableEvents.ToArray();
}
@@ -92,8 +92,8 @@ namespace MarketData.Helper
for (int index = 0; index < resetEvents.Length; index++)
{
if (resetEvents[index].WaitOne(0))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Event {0} is available for work",resetEvents[index].SafeWaitHandle.DangerousGetHandle().ToString()));
{ // @ the end of this trace notice the magic number 3. This is the skip frames for the stack trace. We want the caller of this method to show up in the log
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Event {0} is available for work",resetEvents[index].SafeWaitHandle.DangerousGetHandle().ToString()),3);
availableEvents.Add(resetEvents[index]);
}
}

2
MarketData/MarketDataLib/Helper/PricingMarketDataHelper.cs
View File

@@ -205,7 +205,7 @@ namespace MarketData.Helper
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesBigCharts]End, total took {0}(ms)",profiler.End()));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[UpdatePricesYahooSweep]End, total took {0}(ms)",profiler.End()));
}
}
public bool UpdatePricesAsOfSymbol(String symbol,DateTime asOf) // get prices from BigCharts

13
MarketData/MarketDataLib/MDTrace.cs
View File

@@ -65,10 +65,10 @@ namespace MarketData
/// <summary>WriteLine - Writes a line of text to trace log.</summary>
/// <param name="message">string content of message to write.</param>
/// <returns>void</returns>
public static void WriteLine(LogLevel logLevel,string message)
public static void WriteLine(LogLevel logLevel,string message,int skipFrames=2)
{
if(MDTrace.logLevel<logLevel)return;
Trace.WriteLine(GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+message);
Trace.WriteLine(GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo(skipFrames)+message);
Console.WriteLine(message);
Flush();
}
@@ -76,10 +76,10 @@ namespace MarketData
/// <summary>WriteLine - Writes a line of text to trace log.</summary>
/// <param name="message">string content of message to write.</param>
/// <returns>void</returns>
public static void WriteLine(LogLevel logLevel,ConsoleColor consoleColor,string message)
public static void WriteLine(LogLevel logLevel,ConsoleColor consoleColor,string message,int skipFrames=2)
{
if(MDTrace.logLevel<logLevel)return;
Trace.WriteLine(GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+message);
Trace.WriteLine(GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo(skipFrames)+message);
ConsoleColor currentColor=Console.ForegroundColor;
Console.ForegroundColor = consoleColor;
Console.WriteLine(message);
@@ -131,10 +131,11 @@ namespace MarketData
}
/// <summary>GetMethodInfo - Returns information about the calling method 2 frames up.</summary>
/// <returns>String</returns>
private static String GetMethodInfo()
private static String GetMethodInfo(int skipFrames=2)
{
StringBuilder sb=new StringBuilder();
StackFrame frame=new StackFrame(2,true);
StackFrame frame=new StackFrame(skipFrames,true);
if(null==frame)return $"No Stack Frame at:{skipFrames}";
MethodBase methodBase=frame.GetMethod();
ParameterInfo[] parameters=methodBase.GetParameters();
sb.Append("[").Append(methodBase.DeclaringType.FullName).Append("::").Append(methodBase.Name).Append("(");