diff --git a/MarketData/MarketData/Services/MainService.cs b/MarketData/MarketData/Services/MainService.cs index b8e12e3..aac4abb 100755 --- a/MarketData/MarketData/Services/MainService.cs +++ b/MarketData/MarketData/Services/MainService.cs @@ -53,6 +53,7 @@ namespace MarketData.Services MDTrace.WriteLine(LogLevel.DEBUG,@"MGGAINLOSS /SESSIONFILE:{PATHSESSIONFILE} (i.e.) MGGAINLOSS /SESSIONFILE:C:\boneyard\marketdata\bin\Debug\saferun\MG20180131.txt"); MDTrace.WriteLine(LogLevel.DEBUG,"RUNBACKTEST /STARTDATE: /MAXPOSITIONS: /INITIALCASH: /HOLDINGPERIOD: /{ENDDATE}: /{SESSIONFILE}:"); MDTrace.WriteLine(LogLevel.DEBUG,"CALCBETAS recalculates beta36, beta24, and bet06 for Fundamentals"); + MDTrace.WriteLine(LogLevel.DEBUG,"OPTIMIZEDB optimizes the database tables"); MDTrace.WriteLine(LogLevel.DEBUG,"ECHO {param1} {param2} {param(n)"); } @@ -95,6 +96,7 @@ namespace MarketData.Services tasks.Add("MGGAINLOSS",TaskMGRunMGGainLoss); tasks.Add("RUNBACKTEST",TaskMGRunMGBacktest); tasks.Add("CALCBETAS",TaskCalcBetas); + tasks.Add("OPTIMIZEDB",TaskOptimizeDatabase); tasks.Add("ECHO",TaskEcho); GlobalConfig.Instance.Configuration = configuration; // This call sets up configuration stuff so it needs to be first. @@ -224,7 +226,7 @@ namespace MarketData.Services public async Task TaskUpdateSECFilingsWatchList(CommandArgs commandArgs) { - if(!commandArgs.Has("WATCHLIST")){Console.WriteLine("UPDATESECFILINGSWATCHLIST REQUIRES WATCHLIST");return;} + if(!commandArgs.Has("WATCHLIST")){MDTrace.WriteLine(LogLevel.DEBUG,"UPDATESECFILINGSWATCHLIST REQUIRES WATCHLIST");return;} String watchListName = commandArgs.Coalesce("WATCHLIST"); List symbols = WatchListDA.GetWatchList(watchListName); SECFilingMarketDataHelper secFilingMarketDataHelper=new SECFilingMarketDataHelper(); @@ -234,7 +236,7 @@ namespace MarketData.Services public async Task TaskLoadHeadlinesWatchList(CommandArgs commandArgs) { - if(!commandArgs.Has("WATCHLIST")){Console.WriteLine("LOADHEADLINESWATCHLIST REQUIRES WATCHLIST");return;} + if(!commandArgs.Has("WATCHLIST")){MDTrace.WriteLine(LogLevel.DEBUG,"LOADHEADLINESWATCHLIST REQUIRES WATCHLIST");return;} else LoadHeadlinesWatchList(commandArgs.Coalesce("WATCHLIST")); await Task.FromResult(true); } @@ -247,7 +249,7 @@ namespace MarketData.Services public async Task TaskUpdateDaily2(CommandArgs commandArgs) { - if(!commandArgs.Has("DATE")){Console.WriteLine("UPDATEDAILY2 MISSING DATE");return;} + if(!commandArgs.Has("DATE")){MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEDAILY2 MISSING DATE");return;} UpdateDaily2(commandArgs.Coalesce("DATE")); await Task.FromResult(true); } @@ -260,7 +262,7 @@ namespace MarketData.Services public async Task TaskUpdateLatestPriceWatchList(CommandArgs commandArgs) { - if(!commandArgs.Has("WATCHLIST")){Console.WriteLine("UPDATELATESTPRICEWATCHLIST MISSING WATCHLIST");return;} + if(!commandArgs.Has("WATCHLIST")){MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTPRICEWATCHLIST MISSING WATCHLIST");return;} UpdateLatestPriceWatchList(commandArgs.Coalesce("WATCHLIST")); await Task.FromResult(true); } @@ -336,9 +338,32 @@ namespace MarketData.Services FundamentalDA.UpdateBeta(symbol, date, betaCalc36, betaCalc24, betaCalc06); } } - MDTrace.WriteLine($"Total took {profiler.End()} (ms)"); + MDTrace.WriteLine(LogLevel.DEBUG,$"Total took {profiler.End()} (ms)"); await Task.FromResult(true); - } + } + + /// + /// Performs optimization on the database tables. Since we don't want to run this if any of the monthly updates is running we will check for those tasks and wait for them + /// to complete (if running). + /// + /// + /// + public async Task TaskOptimizeDatabase(CommandArgs commandArgs) + { + int WAIT_TIME_INTERVAL = 120000; // 2 minute intervals between checks. + Profiler profiler = new Profiler(); + + MDTrace.WriteLine(LogLevel.DEBUG, $"Starting..."); + List monthlyUpdateProcessList = new List { "UPDATEFINANCIALSTATEMENTS", "UPDATEFUNDAMENTALS", "UPDATEHISTORICAL", "CALCSTICKER" }; + while (Utility.IsProcessRunning("mk", monthlyUpdateProcessList)) + { + MDTrace.WriteLine(LogLevel.DEBUG, $"OPTIMIZEDB is waiting for items in the process completion list {Utility.ListToString(monthlyUpdateProcessList)}"); + try { Thread.Sleep(WAIT_TIME_INTERVAL); } catch (Exception) {; } + } + MaintenanceDA.OptimizeDatabase(); + MDTrace.WriteLine(LogLevel.DEBUG, $"Total took {profiler.End()} (ms)"); + await Task.FromResult(true); + } public async Task TaskMGSHSession(CommandArgs commandArgs) { @@ -431,7 +456,7 @@ namespace MarketData.Services Trace.Listeners.Remove("Default"); Console.WriteLine($"Adding Trace Listener :{currentWorkingDirectory+logFolder+"/"+strLogFile}"); Trace.Listeners.Add(new TextWriterTraceListener(currentWorkingDirectory+logFolder+"/"+strLogFile)); - MDTrace.WriteLine($"Trace Listener added."); + MDTrace.WriteLine(LogLevel.DEBUG,$"Trace Listener added."); Utility.ShowLogs(currentWorkingDirectory + logFolder); return true; } @@ -651,7 +676,7 @@ namespace MarketData.Services Profiler profiler = new Profiler(); try { - MDTrace.WriteLine("Started."); + MDTrace.WriteLine(LogLevel.DEBUG,"Started."); PriceIndices priceIndices=MarketDataHelper.GetConsumerPriceIndices(); if(null==priceIndices) { @@ -697,7 +722,7 @@ namespace MarketData.Services Profiler profiler = new Profiler(); try { - MDTrace.WriteLine("Started."); + MDTrace.WriteLine(LogLevel.DEBUG,"Started."); EconomicIndicators economicIndicators=MarketDataHelper.GetGDPPerCapita(); if(null==economicIndicators||0==economicIndicators.Count) { @@ -857,7 +882,7 @@ namespace MarketData.Services Profiler profiler = new Profiler(); try { - MDTrace.WriteLine("Updating splits."); + MDTrace.WriteLine(LogLevel.DEBUG,"Updating splits."); Splits splits=MarketDataHelper.GetSplits(); if(null==splits||0==splits.Count)return; MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} split records.",splits.Count)); @@ -880,7 +905,7 @@ namespace MarketData.Services Profiler profiler = new Profiler(); try { - MDTrace.WriteLine("Processing all splits"); + MDTrace.WriteLine(LogLevel.DEBUG,"Processing all splits"); Splits splits=SplitsDA.GetUnappliedSplits(); if(null==splits||0==splits.Count)return; SplitHelper.ProcessSplits(splits); diff --git a/MarketData/MarketDataLib/DataAccess/MaintenanceDA.cs b/MarketData/MarketDataLib/DataAccess/MaintenanceDA.cs new file mode 100644 index 0000000..a5952ba --- /dev/null +++ b/MarketData/MarketDataLib/DataAccess/MaintenanceDA.cs @@ -0,0 +1,69 @@ +using System.Text; +using MySql.Data.MySqlClient; +using MarketData.MarketDataModel; +using MarketData.Utils; + +namespace MarketData.DataAccess +{ + public class MaintenanceDA + { + private MaintenanceDA() + { + } + + public static bool OptimizeDatabase() + { + MySqlConnection sqlConnection = null; + String strQuery = null; + int COMMAND_TIMEOUT = 600; // 10 minutes + List tasks = new List + { + "PURGE BINARY LOGS BEFORE NOW();", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE analystpricetarget;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE analystratings;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE balancesheet;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE incomestatement;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE cashflowstatement;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE dividendhistory;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE etfholdings;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE fundamentals;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE historical;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE insidertransaction;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE options;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE prices;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE secfilings;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE valuations;", + "USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE yieldcurve;", + "USE portfolio_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE accounts;", + "USE portfolio_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE cash;", + "USE portfolio_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE trades;", + "USE portfolio_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE watchlist;", + "USE portfolio_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE watchlists;" + }; + + try + { + sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data")); + foreach (String task in tasks) + { + strQuery = task; + MDTrace.WriteLine(LogLevel.DEBUG, $"Executing {task}"); + using MySqlCommand sqlCommand = new MySqlCommand(task, sqlConnection); + sqlCommand.CommandTimeout = COMMAND_TIMEOUT; + sqlCommand.ExecuteNonQuery(); + } + return true; + } + catch (Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG, exception); + MDTrace.WriteLine(LogLevel.DEBUG, "Query was '" + strQuery + "'"); + return false; + } + finally + { + if (null != sqlConnection) sqlConnection.Close(); + } + } + } +} \ No newline at end of file diff --git a/README.md b/README.md index 663e70c..7d65f85 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ CRON_DIR_IPMONITOR=/opt/MarketData/IPMonitor 0 0 15 * * cd $CRON_DIR ; /opt/MarketData/MarketData/mk UPDATEFUNDAMENTALS > /dev/null 2>&1 0 0 15 * * cd $CRON_DIR ; /opt/MarketData/MarketData/mk UPDATEHISTORICAL > /dev/null 2>&1 15 0 15 * * cd $CRON_DIR ; /opt/MarketData/MarketData/mk CALCSTICKER /WAITFORCOMPLETION:UPDATEFINANCIALSTATEMENTS,UPDATEFUNDAMENTALS,UPDATEHISTORICAL > /dev/null 2>&1 +0 18 * * Sun cd $CRON_DIR ; /opt/MarketData/MarketData/mk OPTIMIZEDB > /dev/null 2>&1 0 * * * * cd $CRON_DIR_IPMONITOR ; /opt/MarketData/IPMonitor/ipmonitor IPMONITOR /FORCE:true > /dev/null 2>&1 10,20,30,40,50 * * * * cd $CRON_DIR_IPMONITOR ; /opt/MarketData/IPMonitor/ipmonitor > /dev/null 2>&1