Sean/ARM64
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Add OPTIMIZEDB task and add to CRON

Browse Source
This commit is contained in:
Sean
2025-05-17 12:18:12 -04:00
parent 971e0f96cb
commit a59a6961a0
3 changed files with 106 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
MarketData/MarketData/Services/MainService.cs
View File

@@ -53,6 +53,7 @@ namespace MarketData.Services
MDTrace.WriteLine(LogLevel.DEBUG,@"MGGAINLOSS /SESSIONFILE:{PATHSESSIONFILE} (i.e.) MGGAINLOSS /SESSIONFILE:C:\boneyard\marketdata\bin\Debug\saferun\MG20180131.txt");
MDTrace.WriteLine(LogLevel.DEBUG,"RUNBACKTEST /STARTDATE: /MAXPOSITIONS: /INITIALCASH: /HOLDINGPERIOD: /{ENDDATE}: /{SESSIONFILE}:");
MDTrace.WriteLine(LogLevel.DEBUG,"CALCBETAS recalculates beta36, beta24, and bet06 for Fundamentals");
MDTrace.WriteLine(LogLevel.DEBUG,"OPTIMIZEDB optimizes the database tables");
MDTrace.WriteLine(LogLevel.DEBUG,"ECHO {param1} {param2} {param(n)");
}
@@ -95,6 +96,7 @@ namespace MarketData.Services
tasks.Add("MGGAINLOSS",TaskMGRunMGGainLoss);
tasks.Add("RUNBACKTEST",TaskMGRunMGBacktest);
tasks.Add("CALCBETAS",TaskCalcBetas);
tasks.Add("OPTIMIZEDB",TaskOptimizeDatabase);
tasks.Add("ECHO",TaskEcho);
GlobalConfig.Instance.Configuration = configuration; // This call sets up configuration stuff so it needs to be first.
@@ -224,7 +226,7 @@ namespace MarketData.Services
public async Task TaskUpdateSECFilingsWatchList(CommandArgs commandArgs)
{
if(!commandArgs.Has("WATCHLIST")){Console.WriteLine("UPDATESECFILINGSWATCHLIST REQUIRES WATCHLIST");return;}
if(!commandArgs.Has("WATCHLIST")){MDTrace.WriteLine(LogLevel.DEBUG,"UPDATESECFILINGSWATCHLIST REQUIRES WATCHLIST");return;}
String watchListName = commandArgs.Coalesce<String>("WATCHLIST");
List<String> symbols = WatchListDA.GetWatchList(watchListName);
SECFilingMarketDataHelper secFilingMarketDataHelper=new SECFilingMarketDataHelper();
@@ -234,7 +236,7 @@ namespace MarketData.Services
public async Task TaskLoadHeadlinesWatchList(CommandArgs commandArgs)
{
if(!commandArgs.Has("WATCHLIST")){Console.WriteLine("LOADHEADLINESWATCHLIST REQUIRES WATCHLIST");return;}
if(!commandArgs.Has("WATCHLIST")){MDTrace.WriteLine(LogLevel.DEBUG,"LOADHEADLINESWATCHLIST REQUIRES WATCHLIST");return;}
else LoadHeadlinesWatchList(commandArgs.Coalesce<String>("WATCHLIST"));
await Task.FromResult(true);
}
@@ -247,7 +249,7 @@ namespace MarketData.Services
public async Task TaskUpdateDaily2(CommandArgs commandArgs)
{
if(!commandArgs.Has("DATE")){Console.WriteLine("UPDATEDAILY2 MISSING DATE");return;}
if(!commandArgs.Has("DATE")){MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEDAILY2 MISSING DATE");return;}
UpdateDaily2(commandArgs.Coalesce<DateTime>("DATE"));
await Task.FromResult(true);
}
@@ -260,7 +262,7 @@ namespace MarketData.Services
public async Task TaskUpdateLatestPriceWatchList(CommandArgs commandArgs)
{
if(!commandArgs.Has("WATCHLIST")){Console.WriteLine("UPDATELATESTPRICEWATCHLIST MISSING WATCHLIST");return;}
if(!commandArgs.Has("WATCHLIST")){MDTrace.WriteLine(LogLevel.DEBUG,"UPDATELATESTPRICEWATCHLIST MISSING WATCHLIST");return;}
UpdateLatestPriceWatchList(commandArgs.Coalesce<String>("WATCHLIST"));
await Task.FromResult(true);
}
@@ -336,9 +338,32 @@ namespace MarketData.Services
FundamentalDA.UpdateBeta(symbol, date, betaCalc36, betaCalc24, betaCalc06);
}
}
MDTrace.WriteLine($"Total took {profiler.End()} (ms)");
MDTrace.WriteLine(LogLevel.DEBUG,$"Total took {profiler.End()} (ms)");
await Task.FromResult(true);
}
}
/// <summary>
/// Performs optimization on the database tables. Since we don't want to run this if any of the monthly updates is running we will check for those tasks and wait for them
/// to complete (if running).
/// </summary>
/// <param name="commandArgs"></param>
/// <returns></returns>
public async Task TaskOptimizeDatabase(CommandArgs commandArgs)
{
int WAIT_TIME_INTERVAL = 120000; // 2 minute intervals between checks.
Profiler profiler = new Profiler();
MDTrace.WriteLine(LogLevel.DEBUG, $"Starting...");
List<String> monthlyUpdateProcessList = new List<String> { "UPDATEFINANCIALSTATEMENTS", "UPDATEFUNDAMENTALS", "UPDATEHISTORICAL", "CALCSTICKER" };
while (Utility.IsProcessRunning("mk", monthlyUpdateProcessList))
{
MDTrace.WriteLine(LogLevel.DEBUG, $"OPTIMIZEDB is waiting for items in the process completion list {Utility.ListToString(monthlyUpdateProcessList)}");
try { Thread.Sleep(WAIT_TIME_INTERVAL); } catch (Exception) {; }
}
MaintenanceDA.OptimizeDatabase();
MDTrace.WriteLine(LogLevel.DEBUG, $"Total took {profiler.End()} (ms)");
await Task.FromResult(true);
}
public async Task TaskMGSHSession(CommandArgs commandArgs)
{
@@ -431,7 +456,7 @@ namespace MarketData.Services
Trace.Listeners.Remove("Default");
Console.WriteLine($"Adding Trace Listener :{currentWorkingDirectory+logFolder+"/"+strLogFile}");
Trace.Listeners.Add(new TextWriterTraceListener(currentWorkingDirectory+logFolder+"/"+strLogFile));
MDTrace.WriteLine($"Trace Listener added.");
MDTrace.WriteLine(LogLevel.DEBUG,$"Trace Listener added.");
Utility.ShowLogs(currentWorkingDirectory + logFolder);
return true;
}
@@ -651,7 +676,7 @@ namespace MarketData.Services
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine("Started.");
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
PriceIndices priceIndices=MarketDataHelper.GetConsumerPriceIndices();
if(null==priceIndices)
{
@@ -697,7 +722,7 @@ namespace MarketData.Services
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine("Started.");
MDTrace.WriteLine(LogLevel.DEBUG,"Started.");
EconomicIndicators economicIndicators=MarketDataHelper.GetGDPPerCapita();
if(null==economicIndicators||0==economicIndicators.Count)
{
@@ -857,7 +882,7 @@ namespace MarketData.Services
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine("Updating splits.");
MDTrace.WriteLine(LogLevel.DEBUG,"Updating splits.");
Splits splits=MarketDataHelper.GetSplits();
if(null==splits||0==splits.Count)return;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} split records.",splits.Count));
@@ -880,7 +905,7 @@ namespace MarketData.Services
Profiler profiler = new Profiler();
try
{
MDTrace.WriteLine("Processing all splits");
MDTrace.WriteLine(LogLevel.DEBUG,"Processing all splits");
Splits splits=SplitsDA.GetUnappliedSplits();
if(null==splits||0==splits.Count)return;
SplitHelper.ProcessSplits(splits);

69
MarketData/MarketDataLib/DataAccess/MaintenanceDA.cs Normal file
View File

@@ -0,0 +1,69 @@
using System.Text;
using MySql.Data.MySqlClient;
using MarketData.MarketDataModel;
using MarketData.Utils;
namespace MarketData.DataAccess
{
public class MaintenanceDA
{
private MaintenanceDA()
{
}
public static bool OptimizeDatabase()
{
MySqlConnection sqlConnection = null;
String strQuery = null;
int COMMAND_TIMEOUT = 600; // 10 minutes
List<String> tasks = new List<String>
{
"PURGE BINARY LOGS BEFORE NOW();",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE analystpricetarget;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE analystratings;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE balancesheet;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE incomestatement;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE cashflowstatement;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE dividendhistory;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE etfholdings;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE fundamentals;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE historical;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE insidertransaction;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE options;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE prices;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE secfilings;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE valuations;",
"USE market_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE yieldcurve;",
"USE portfolio_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE accounts;",
"USE portfolio_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE cash;",
"USE portfolio_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE trades;",
"USE portfolio_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE watchlist;",
"USE portfolio_data;OPTIMIZE NO_WRITE_TO_BINLOG TABLE watchlists;"
};
try
{
sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
foreach (String task in tasks)
{
strQuery = task;
MDTrace.WriteLine(LogLevel.DEBUG, $"Executing {task}");
using MySqlCommand sqlCommand = new MySqlCommand(task, sqlConnection);
sqlCommand.CommandTimeout = COMMAND_TIMEOUT;
sqlCommand.ExecuteNonQuery();
}
return true;
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG, exception);
MDTrace.WriteLine(LogLevel.DEBUG, "Query was '" + strQuery + "'");
return false;
}
finally
{
if (null != sqlConnection) sqlConnection.Close();
}
}
}
}

1
README.md
View File

@@ -105,6 +105,7 @@ CRON_DIR_IPMONITOR=/opt/MarketData/IPMonitor
0 0 15 * * cd $CRON_DIR ; /opt/MarketData/MarketData/mk UPDATEFUNDAMENTALS > /dev/null 2>&1
0 0 15 * * cd $CRON_DIR ; /opt/MarketData/MarketData/mk UPDATEHISTORICAL > /dev/null 2>&1
15 0 15 * * cd $CRON_DIR ; /opt/MarketData/MarketData/mk CALCSTICKER /WAITFORCOMPLETION:UPDATEFINANCIALSTATEMENTS,UPDATEFUNDAMENTALS,UPDATEHISTORICAL > /dev/null 2>&1
0 18 * * Sun cd $CRON_DIR ; /opt/MarketData/MarketData/mk OPTIMIZEDB > /dev/null 2>&1
0 * * * * cd $CRON_DIR_IPMONITOR ; /opt/MarketData/IPMonitor/ipmonitor IPMONITOR /FORCE:true > /dev/null 2>&1
10,20,30,40,50 * * * * cd $CRON_DIR_IPMONITOR ; /opt/MarketData/IPMonitor/ipmonitor > /dev/null 2>&1