From c7ef8a04c9c57cc9e3d5124bc31652a8b0e220c6 Mon Sep 17 00:00:00 2001 From: Sean Date: Thu, 27 Mar 2025 18:54:46 -0400 Subject: [PATCH] Code enhancements --- MarketData/MarketData/CommandArgs.cs | 7 + MarketData/MarketData/Executor.cs | 3 +- MarketData/MarketData/Services/MainService.cs | 172 +++++++++++------- MarketData/MarketData/appsettings.json | 2 +- MarketData/MarketData/marketdata.log | 12 -- MarketData/MarketDataLib/MDTrace.cs | 16 -- MarketData/MarketDataLib/Utility/Utility.cs | 101 ++++++---- README.md | 34 ++++ 8 files changed, 216 insertions(+), 131 deletions(-) delete mode 100644 MarketData/MarketData/marketdata.log diff --git a/MarketData/MarketData/CommandArgs.cs b/MarketData/MarketData/CommandArgs.cs index 6fd13c3..0a0b20b 100644 --- a/MarketData/MarketData/CommandArgs.cs +++ b/MarketData/MarketData/CommandArgs.cs @@ -5,9 +5,16 @@ namespace MarketData { public class CommandArgs : Dictionary { + private String[] arguments = default; + + public String[] GetArguments() + { + return arguments; + } // While the commands are converted to uppercase care must be taken to preserve the case of the arguments. public CommandArgs(String[] args) { + arguments = args; for(int index=1;index public void Execute() - { _mainService.RunService(_arguments.GetArguments(), _configuration); + { + _mainService.RunService(_arguments.GetArguments(), _configuration); } } } diff --git a/MarketData/MarketData/Services/MainService.cs b/MarketData/MarketData/Services/MainService.cs index e1af3f1..b6625c4 100755 --- a/MarketData/MarketData/Services/MainService.cs +++ b/MarketData/MarketData/Services/MainService.cs @@ -13,6 +13,8 @@ namespace MarketData.Services { public class MainService : IMainService { + private Dictionary> tasks = new Dictionary>(); + /// /// This is the main entry point /// @@ -21,84 +23,49 @@ namespace MarketData.Services public void RunService(String[] args,IConfiguration configuration) { Profiler profiler = new Profiler(); - MDTrace.LogLevel = LogLevel.DEBUG; - String strLogFile = "marketdata.log"; - Utility.DeleteFile(strLogFile); - Trace.Listeners.Add(new TextWriterTraceListener(strLogFile)); DateTime currentDate=DateTime.Now; - MDTrace.WriteLine(LogLevel.DEBUG,$"[RunService] Started @ {Utility.DateTimeToStringYYYYHMMHDDHHMMSSTT(currentDate)}"); + Trace.Listeners.Remove("Default"); + + tasks.Add("LOADHEADLINESWATCHLIST",TaskLoadHeadlinesWatchList); + tasks.Add("LOADPREMARKETDATA",TaskLoadPremarketData); + tasks.Add("UPDATEDAILY2",TaskUpdateDaily2); + tasks.Add("UPDATELATESTPRICEOPENPOSITIONS",TaskUpdateLatestPriceOpenPositions); + tasks.Add("UPDATELATESTPRICEWATCHLIST",TaskUpdateLatestPriceWatchList); + tasks.Add("UPDATELATESTANALYSTRATINGS",TaskUpdateLatestAnalystRatings); + tasks.Add("UPDATEANALYSTRATINGS",TaskUpdateAnalystRatings); + tasks.Add("ECHO",TaskEcho); + GlobalConfig.Instance.Configuration = configuration; // This call sets up configuration stuff so it needs to be first. + if (args.Length < 1 || String.IsNullOrEmpty(args[0])) + { + DisplayUsage(); + return; + } + + string arg = args[0].ToUpper(); + + CreateLogging(arg); // log files are now of the form market_data+task.log. Also log files will expire daily + + MDTrace.WriteLine(LogLevel.DEBUG,$"[RunService] Started @ {Utility.DateTimeToStringYYYYHMMHDDHHMMSSTT(currentDate)}"); + MDTrace.WriteLine(LogLevel.DEBUG,$"[RunService] Argument {arg}"); + DateTime maxHolidayDate =HolidayDA.GetMaxHolidayDate(); if(currentDate>maxHolidayDate) { - Console.WriteLine(String.Format("There are no holidays defined in the system. Add holidays for year {0} into marketholidays table",currentDate.Year)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("There are no holidays defined in the system. Add holidays for year {0} into marketholidays table",currentDate.Year)); return; } - if (args.Length < 1) + if(!tasks.ContainsKey(arg)) { - DisplayUsage(); + DisplayUsage(); return; } - string arg = args[0].ToUpper(); - MDTrace.WriteLine(LogLevel.DEBUG,$"[RunService] Argument {arg}"); - try { - if(arg.Equals("LOADHEADLINESWATCHLIST")) - { - CommandArgs commandArgs=new CommandArgs(args); - if(!commandArgs.Has("WATCHLIST")){Console.WriteLine("LOADHEADLINESWATCHLIST REQUIRES WATCHLIST");return;} - else LoadHeadlinesWatchList(commandArgs.Coalesce("WATCHLIST")); - } - else if(arg.Equals("LOADPREMARKETDATA")) - { - LoadPremarketData(); - } - else if (arg.Equals("UPDATEDAILY2")) - { - CommandArgs commandArgs=new CommandArgs(args); - if(!commandArgs.Has("DATE")){Console.WriteLine("UPDATEDAILY2 MISSING DATE");return;} - UpdateDaily2(commandArgs.Coalesce("DATE")); - } - else if (arg.Equals("UPDATELATESTPRICEOPENPOSITIONS")) - { - UpdateLatestPriceOpenPositions(); - } - else if (arg.Equals("UPDATELATESTPRICEWATCHLIST")) - { - CommandArgs commandArgs=new CommandArgs(args); - if(!commandArgs.Has("WATCHLIST")){Console.WriteLine("UPDATELATESTPRICEWATCHLIST MISSING WATCHLIST");return;} - UpdateLatestPriceWatchList(commandArgs.Coalesce("WATCHLIST")); - } - else if (arg.Equals("UPDATELATESTANALYSTRATINGS")) - { - CommandArgs commandArgs = new CommandArgs(args); - if (commandArgs.Has("UPDATESECURITYMASTER")) UpdateLatestAnalystRatings(commandArgs.Coalesce("UPDATESECURITYMASTER")); - else UpdateLatestAnalystRatings(); - } - else if (arg.Equals("UPDATEANALYSTRATINGS")) - { - CommandArgs commandArgs = new CommandArgs(args); - if (commandArgs.Has("SYMBOL") && commandArgs.Has("MINDATE")) UpdateAnalystRatings(commandArgs.Coalesce("SYMBOL"), commandArgs.Coalesce("MINDATE")); - else if (commandArgs.Has("SYMBOL") ) UpdateAnalystRatings(commandArgs.Coalesce("SYMBOL")); - else if (commandArgs.Has("MINDATE")) UpdateAnalystRatings(null,commandArgs.Coalesce("MINDATE")); - else UpdateAnalystRatings(); - } - else if(arg.Equals("ECHO")) - { - MDTrace.WriteLine(LogLevel.DEBUG,$"ECHO"); - for(int index =0; index("WATCHLIST")); + await Task.FromResult(true); + } + + public async Task TaskLoadPremarketData(CommandArgs commandArgs) + { + LoadPremarketData(); + await Task.FromResult(true); + } + + public async Task TaskUpdateDaily2(CommandArgs commandArgs) + { + if(!commandArgs.Has("DATE")){Console.WriteLine("UPDATEDAILY2 MISSING DATE");return;} + UpdateDaily2(commandArgs.Coalesce("DATE")); + await Task.FromResult(true); + } + + public async Task TaskUpdateLatestPriceOpenPositions(CommandArgs commandArgs) + { + UpdateLatestPriceOpenPositions(); + await Task.FromResult(true); + } + + public async Task TaskUpdateLatestPriceWatchList(CommandArgs commandArgs) + { + if(!commandArgs.Has("WATCHLIST")){Console.WriteLine("UPDATELATESTPRICEWATCHLIST MISSING WATCHLIST");return;} + UpdateLatestPriceWatchList(commandArgs.Coalesce("WATCHLIST")); + await Task.FromResult(true); + } + + public async Task TaskUpdateLatestAnalystRatings(CommandArgs commandArgs) + { + if (commandArgs.Has("UPDATESECURITYMASTER")) UpdateLatestAnalystRatings(commandArgs.Coalesce("UPDATESECURITYMASTER")); + else UpdateLatestAnalystRatings(); + await Task.FromResult(true); + } + + public async Task TaskUpdateAnalystRatings(CommandArgs commandArgs) + { + if (commandArgs.Has("SYMBOL") && commandArgs.Has("MINDATE")) UpdateAnalystRatings(commandArgs.Coalesce("SYMBOL"), commandArgs.Coalesce("MINDATE")); + else if (commandArgs.Has("SYMBOL") ) UpdateAnalystRatings(commandArgs.Coalesce("SYMBOL")); + else if (commandArgs.Has("MINDATE")) UpdateAnalystRatings(null,commandArgs.Coalesce("MINDATE")); + else UpdateAnalystRatings(); + await Task.FromResult(true); + } + + public async Task TaskEcho(CommandArgs commandArgs) + { + String[] args = commandArgs.GetArguments(); + MDTrace.WriteLine(LogLevel.DEBUG,$"ECHO"); + for(int index =0; indexWriteLine - Writes a line of text to trace log. /// string content of message to write. /// void -// [Conditional("TRACE")] public static void Write(LogLevel logLevel,string message) { if(MDTrace.logLevelWriteLine - Writes a line of text to trace log. /// string content of message to write. /// void -// [Conditional("TRACE")] public static void WriteLine(string message) { WriteLine(LogLevel.DEBUG,GetCallerIP()+GetThreadRep()+GetLogLevelRep()+"["+DateTime.Now.ToString()+"]"+" "+GetMethodInfo()+message); @@ -61,7 +55,6 @@ namespace MarketData /// WriteLine - Writes a line of text to trace log. /// string content of message to write. /// void -// [Conditional("TRACE")] public static void WriteLine(LogLevel logLevel,Exception exception) { if(MDTrace.logLevelWriteLine - Writes a line of text to trace log. /// string content of message to write. /// void -// [Conditional("TRACE")] public static void WriteLine(LogLevel logLevel,string message) { if(MDTrace.logLevelIndent - set trace log indentation. /// void -// [Conditional("TRACE")] public static void Indent() { Trace.Indent(); } /// Unindent - set trace log indentation back. /// void -// [Conditional("TRACE")] public static void Unindent() { Trace.Unindent(); } /// Flush - Flush trace log buffers to disk. /// void -// [Conditional("TRACE")] public static void Flush() { Trace.Flush(); @@ -149,11 +138,6 @@ namespace MarketData private static String GetCallerIP() { return "[LOCAL]"; - // String hostName=(String)CallContext.GetData("HostName"); - // String hostAddress=(String)CallContext.GetData("HostAddress"); - // if(null!=hostName)hostName=hostName.Split('.')[0]; - // if(null==hostName && null==hostAddress)return "[LOCAL]"; - // return "["+hostAddress+"->"+hostName+"]"; } } } diff --git a/MarketData/MarketDataLib/Utility/Utility.cs b/MarketData/MarketDataLib/Utility/Utility.cs index 0358e52..84a9365 100755 --- a/MarketData/MarketDataLib/Utility/Utility.cs +++ b/MarketData/MarketDataLib/Utility/Utility.cs @@ -1,17 +1,11 @@ -using System; -using System.Text; +using System.Text; using System.IO.Compression; using System.IO; using System.Globalization; -using System.Linq; -using System.Collections.Generic; -using Microsoft.Win32; using System.Diagnostics; using MarketData.MarketDataModel; using System.Security.Principal; -using System.Threading; using ThreadState=System.Threading.ThreadState; -using System.Net; namespace MarketData.Utils { @@ -78,44 +72,79 @@ namespace MarketData.Utils return date; } - public static void RemoveLogFiles(String strFolder=null) - { - if(null==strFolder)strFolder=Directory.GetCurrentDirectory(); - String[] logFiles=Directory.GetFiles(strFolder); - if(null==logFiles || 0==logFiles.Length)return; - logFiles=logFiles.Where(x => x.EndsWith(".log",StringComparison.InvariantCultureIgnoreCase)).ToArray(); - if(null==logFiles || 0==logFiles.Length)return; +// This is being called before any trace listeners have been added + public static bool EnsureLogFolder(String strLogFolder) + { + if(Directory.Exists(strLogFolder))return true; + try + { + Directory.CreateDirectory(strLogFolder); + return true; + } + catch(Exception) + { + return false; + } + } + + public static void ExpireLogs(String pathLogFiles,int expiryDays) + { + DateTime currentDate = DateTime.Now; + DateGenerator dateGenerator = new DateGenerator(); + String[] logFiles=Directory.GetFiles(pathLogFiles,"*.log"); foreach(String logFile in logFiles) { try { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Removing {0}",logFile)); - File.Delete(logFile); + DateTime creationTime = File.GetCreationTime(logFile); + int age = dateGenerator.DaysBetweenActual(currentDate, creationTime); + if(age>=expiryDays) + { + File.Delete(logFile); + } } catch(Exception){;} } - } + } - public static void RemoveLogFilesExcept(String exceptFile,String strFolder=null) - { - if(null==strFolder)strFolder=Directory.GetCurrentDirectory(); - String[] logFiles=Directory.GetFiles(strFolder,"*.log"); - if(null==logFiles || 0==logFiles.Length)return; - logFiles=logFiles - .Where(x => x.EndsWith(".log",StringComparison.InvariantCultureIgnoreCase) && - !x.EndsWith(exceptFile,StringComparison.InvariantCultureIgnoreCase)).ToArray(); + // public static void RemoveLogFiles(String strFolder=null) + // { + // if(null==strFolder)strFolder=Directory.GetCurrentDirectory(); + // String[] logFiles=Directory.GetFiles(strFolder); + // if(null==logFiles || 0==logFiles.Length)return; + // logFiles=logFiles.Where(x => x.EndsWith(".log",StringComparison.InvariantCultureIgnoreCase)).ToArray(); + // if(null==logFiles || 0==logFiles.Length)return; + // foreach(String logFile in logFiles) + // { + // try + // { + // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Removing {0}",logFile)); + // File.Delete(logFile); + // } + // catch(Exception){;} + // } + // } - if(null==logFiles || 0==logFiles.Length)return; - foreach(String logFile in logFiles) - { - try - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Removing {0}",logFile)); - File.Delete(logFile); - } - catch(Exception){;} - } - } + // public static void RemoveLogFilesExcept(String exceptFile,String strFolder=null) + // { + // if(null==strFolder)strFolder=Directory.GetCurrentDirectory(); + // String[] logFiles=Directory.GetFiles(strFolder,"*.log"); + // if(null==logFiles || 0==logFiles.Length)return; + // logFiles=logFiles + // .Where(x => x.EndsWith(".log",StringComparison.InvariantCultureIgnoreCase) && + // !x.EndsWith(exceptFile,StringComparison.InvariantCultureIgnoreCase)).ToArray(); + + // if(null==logFiles || 0==logFiles.Length)return; + // foreach(String logFile in logFiles) + // { + // try + // { + // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Removing {0}",logFile)); + // File.Delete(logFile); + // } + // catch(Exception){;} + // } + // } public static String Pad(string str, char filler, int length) { diff --git a/README.md b/README.md index db5d7ce..3fdbbdd 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,23 @@ ARM version of marketdata libraries. sudo truncate syslog -- size 0 +more on syslog and journal +https://ubuntuforums.org/showthread.php?t=2495218 +https://stackoverflow.com/questions/16995425/how-does-cmd-dev-null-21-work +journalctl --disk-usage # See log file disk use + sudo journalctl --vacuum-size=200M # Drop log file size to 200M, if possible. + sudo journalctl --vacuum-time=10d # Drop logs, over 10 days old + +DEV NULL +ping a.b.c > /dev/null 2>&1 + CRON SCHEDULE +** THE LATEST AND GREATEST IS ALWAYS AT THE BOTTOM + +Also note that 1>/dev/null is synonymous to, but more explicit than >/dev/null + # sudo service cron reload , sudo service cron restart # cron helper app here -> https://cron.help/#*/5_9-17_*_*_1-5 DOTNET_ROOT=/opt/dotnet @@ -16,3 +30,23 @@ DOTNET_ROOT=/opt/dotnet 0 19 * * 1-5 /opt/MarketData/MarketData/mk UPDATEDAILY2 /DATE:$(date -d"-0 days" +\%m-\%d-\%Y) 2>&1 | logger -t updatedaily2 +*/15 6-16 * * 1-5 /opt/MarketData/MarketData/mk LOADHEADLINESWATCHLIST /WATCHLIST:Valuations > /dev/null 2>&1 +*/5 5-16 * * 1-5 /opt/MarketData/MarketData/mk LOADPREMARKETDATA > /dev/null 2>&1 +*/5 9-16 * * 1-5 /opt/MarketData/MarketData/mk UPDATELATESTPRICEOPENPOSITIONS > /dev/null 2>&1 +*/5 9-16 * * 1-5 /opt/MarketData/MarketData/mk UPDATELATESTPRICEWATCHLIST /WATCHLIST:Valuations > /dev/null 2>&1 +*/30 9-16 * * 1-5 /opt/MarketData/MarketData/mk UPDATELATESTANALYSTRATINGS 2>&1 | logger -t analystratings +0 19 * * 1-5 /opt/MarketData/MarketData/mk UPDATEDAILY2 /DATE:$(date -d"-0 days" +\%m-\%d-\%Y) > /dev/null 2>&1 + +# m h dom mon dow command +# sudo service cron reload , sudo service cron restart +# cron helper app here -> https://cron.help/#*/5_9-17_*_*_1-5 +DOTNET_ROOT=/opt/dotnet +CRON_DIR=/opt/MarketData/MarketData +*/15 6-16 * * 1-5 cd $CRON_DIR ; /opt/MarketData/MarketData/mk LOADHEADLINESWATCHLIST /WATCHLIST:Valuations > /dev/null 2>&1 +*/5 5-16 * * 1-5 cd $CRON_DIR ; /opt/MarketData/MarketData/mk LOADPREMARKETDATA > /dev/null 2>&1 +*/5 9-16 * * 1-5 cd $CRON_DIR ; /opt/MarketData/MarketData/mk UPDATELATESTPRICEOPENPOSITIONS > /dev/null 2>&1 +*/5 9-16 * * 1-5 cd $CRON_DIR ; /opt/MarketData/MarketData/mk UPDATELATESTPRICEWATCHLIST /WATCHLIST:Valuations > /dev/null 2>&1 +*/30 9-16 * * 1-5 cd $CRON_DIR ; /opt/MarketData/MarketData/mk UPDATELATESTANALYSTRATINGS > /dev/null 2>&1 +0 19 * * 1-5 cd $CRON_DIR ; /opt/MarketData/MarketData/mk UPDATEDAILY2 /DATE:$(date -d"-0 days" +\%m-\%d-\%Y) > /dev/null 2>&1 + +