From b0075feee9ba539ee7755478048d8d7bc63c2282 Mon Sep 17 00:00:00 2001 From: Sean Date: Fri, 11 Apr 2025 17:07:48 -0400 Subject: [PATCH] Add profiling to selected DAs --- .../DataAccess/CompanyProfileDA.cs | 12 +++++++ .../MarketDataLib/DataAccess/FundamentalDA.cs | 26 ++++++++++++++ .../DataAccess/IncomeStatementDA.cs | 20 +++++++++++ MarketData/MarketDataLib/Utility/SQLUtils.cs | 35 +++++-------------- 4 files changed, 67 insertions(+), 26 deletions(-) diff --git a/MarketData/MarketDataLib/DataAccess/CompanyProfileDA.cs b/MarketData/MarketDataLib/DataAccess/CompanyProfileDA.cs index d502cce..86f6da7 100755 --- a/MarketData/MarketDataLib/DataAccess/CompanyProfileDA.cs +++ b/MarketData/MarketDataLib/DataAccess/CompanyProfileDA.cs @@ -15,6 +15,7 @@ namespace MarketData.DataAccess // *********************************************************************************************************** public static bool UpdateCompanyProfile(CompanyProfile companyProfile) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlTransaction sqlTransaction = null; MySqlCommand sqlCommand=null; @@ -66,11 +67,13 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlTransaction) sqlTransaction.Dispose(); if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static CompanyProfiles GetCompanyProfiles() { + Profiler profiler = new Profiler(); CompanyProfiles companyProfiles=new CompanyProfiles(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; @@ -112,11 +115,13 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static CompanyProfile GetCompanyProfile(String symbol) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand=null; @@ -155,12 +160,14 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } // This handles the actual company profile table where we store additional company related informtion private static bool CompanyProfileDetailsExists(CompanyProfile companyProfile,MySqlConnection sqlConnection,MySqlTransaction sqlTransaction) { + Profiler profiler = new Profiler(); MySqlDataReader sqlDataReader=null; MySqlCommand sqlCommand=null; String strQuery = null; @@ -186,11 +193,13 @@ namespace MarketData.DataAccess { if(null!=sqlCommand)sqlCommand.Dispose(); if(null!=sqlDataReader){sqlDataReader.Close();sqlDataReader.Dispose();} + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } private static bool InsertCompanyProfileDescription(CompanyProfile companyProfile,MySqlConnection sqlConnection,MySqlTransaction sqlTransaction) { + Profiler profiler = new Profiler(); MySqlCommand sqlCommand=null; String strQuery = null; @@ -216,11 +225,13 @@ namespace MarketData.DataAccess finally { if(null!=sqlCommand)sqlCommand.Dispose(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } private static bool UpdateCompanyProfileDescription(CompanyProfile companyProfile,MySqlConnection sqlConnection,MySqlTransaction sqlTransaction) { + Profiler profiler = new Profiler(); MySqlCommand sqlCommand=null; String strQuery = null; @@ -244,6 +255,7 @@ namespace MarketData.DataAccess finally { if(null!=sqlCommand)sqlCommand.Dispose(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } } diff --git a/MarketData/MarketDataLib/DataAccess/FundamentalDA.cs b/MarketData/MarketDataLib/DataAccess/FundamentalDA.cs index 1f04051..2ed714e 100755 --- a/MarketData/MarketDataLib/DataAccess/FundamentalDA.cs +++ b/MarketData/MarketDataLib/DataAccess/FundamentalDA.cs @@ -12,6 +12,7 @@ namespace MarketData.DataAccess } public static bool CheckFundamentalModifiedOn(String symbol,DateTime modified) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand =null; @@ -43,10 +44,12 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static DateTime GetLatestDate(String symbol) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand=null; @@ -79,10 +82,12 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if(null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static TimeSeriesCollection GetTotalCashMils(String symbol) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand =null; @@ -122,10 +127,12 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static TimeSeriesCollection GetPERatio(String symbol) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand =null; @@ -165,10 +172,12 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static TimeSeriesCollection GetEPS(String symbol,DateTime? maxDate=null) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand = null; @@ -209,10 +218,12 @@ namespace MarketData.DataAccess if (null != sqlCommand) sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static Fundamental GetFundamental(String symbol) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand =null; @@ -275,10 +286,12 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static Fundamental GetFundamental(String symbol, DateTime asof) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand=null; @@ -344,11 +357,13 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } // get the maximum date on record on or before asof public static DateTime? GetMaxDateFromFundamental(String symbol,DateTime asof) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand=null; @@ -382,10 +397,12 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static Fundamental GetFundamentalMaxDate(String symbol, DateTime asof) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand=null; @@ -455,11 +472,13 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static Fundamentals GetFundamentalMaxDateTop(String symbol, DateTime asof, int top) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand=null; @@ -535,11 +554,13 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static bool InsertFundamentals(Fundamentals fundamentals) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlTransaction sqlTransaction = null; String strQuery = null; @@ -630,11 +651,13 @@ namespace MarketData.DataAccess { if(null!=sqlTransaction)sqlTransaction.Dispose(); if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static bool InsertFundamental(Fundamental fundamental) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlTransaction sqlTransaction = null; String strQuery = null; @@ -721,6 +744,7 @@ namespace MarketData.DataAccess { if(null!=sqlTransaction)sqlTransaction.Dispose(); if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } private static bool DeleteFundamentals(Fundamentals fundamentals, MySqlConnection sqlConnection, MySqlTransaction sqlTransaction) @@ -733,6 +757,7 @@ namespace MarketData.DataAccess } private static bool DeleteFundamental(Fundamental fundamental, MySqlConnection sqlConnection, MySqlTransaction sqlTransaction) { + Profiler profiler = new Profiler(); StringBuilder sb = new StringBuilder(); String strQuery = null; @@ -758,6 +783,7 @@ namespace MarketData.DataAccess } finally { + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } } diff --git a/MarketData/MarketDataLib/DataAccess/IncomeStatementDA.cs b/MarketData/MarketDataLib/DataAccess/IncomeStatementDA.cs index 4fdaa23..fc83957 100755 --- a/MarketData/MarketDataLib/DataAccess/IncomeStatementDA.cs +++ b/MarketData/MarketDataLib/DataAccess/IncomeStatementDA.cs @@ -10,6 +10,7 @@ namespace MarketData.DataAccess // ********************************************* I N C O M E S T A T E M E N T ************************************** public static bool CheckIncomeStatementModifiedOn(String symbol, DateTime modified,IncomeStatement.PeriodType periodType) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand=null; @@ -42,10 +43,12 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static List GetIncomeStatementDates(String symbol,IncomeStatement.PeriodType periodType) { + Profiler profiler = new Profiler(); List incomeStatementDates = new List(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; @@ -80,10 +83,12 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static DateTime? GetLatestIncomeStatementDate(String symbol,IncomeStatement.PeriodType periodType) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand=null; @@ -118,10 +123,12 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static IncomeStatement GetIncomeStatement(String symbol,IncomeStatement.PeriodType periodType) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand=null; @@ -170,10 +177,12 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static TimeSeriesCollection GetRevenue(String symbol,IncomeStatement.PeriodType period) { + Profiler profiler = new Profiler(); TimeSeriesCollection timeSeriesCollection = new TimeSeriesCollection(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; @@ -212,11 +221,13 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } // Profit Margin is calculated as a percentage. public static TimeSeriesCollection GetProfitMargin(String symbol,IncomeStatement.PeriodType period=IncomeStatement.PeriodType.Annual) { + Profiler profiler = new Profiler(); TimeSeriesCollection timeSeriesCollection = new TimeSeriesCollection(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; @@ -260,11 +271,13 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } // Profit Margin is calculated as a percentage. public static TimeSeriesCollection GetProfitMarginMaxAsOf(String symbol,DateTime maxDate, IncomeStatement.PeriodType period = IncomeStatement.PeriodType.Annual) { + Profiler profiler = new Profiler(); TimeSeriesCollection timeSeriesCollection = new TimeSeriesCollection(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; @@ -309,10 +322,12 @@ namespace MarketData.DataAccess if (null != sqlCommand) sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static IncomeStatement GetIncomeStatement(String symbol,DateTime asof,IncomeStatement.PeriodType periodType) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand=null; @@ -359,10 +374,12 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static IncomeStatement GetIncomeStatementMaxAsOf(String symbol,DateTime asof,IncomeStatement.PeriodType periodType) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlDataReader sqlDataReader = null; MySqlCommand sqlCommand=null; @@ -410,10 +427,12 @@ namespace MarketData.DataAccess if(null!=sqlCommand)sqlCommand.Dispose(); if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();} if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } public static bool InsertIncomeStatements(List incomeStatements) { + Profiler profiler = new Profiler(); MySqlConnection sqlConnection = null; MySqlTransaction sqlTransaction = null; String strQuery = null; @@ -476,6 +495,7 @@ namespace MarketData.DataAccess { if (null != sqlTransaction) sqlTransaction.Dispose(); if (null != sqlConnection) sqlConnection.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } private static bool DeleteIncomeStatements(List incomeStatements, MySqlConnection sqlConnection, MySqlTransaction sqlTransaction) diff --git a/MarketData/MarketDataLib/Utility/SQLUtils.cs b/MarketData/MarketDataLib/Utility/SQLUtils.cs index 4444663..4359dbc 100755 --- a/MarketData/MarketDataLib/Utility/SQLUtils.cs +++ b/MarketData/MarketDataLib/Utility/SQLUtils.cs @@ -12,10 +12,10 @@ namespace MarketData.Utils /// SQlUtils - SQL utility class public class SqlUtils { - public static int COMMAND_TIMEOUT=300; - public static int MIN_POOL_SIZE=10; - public static int MAX_POOL_SIZE=50; - public static string COLLATION="utf8mb4_unicode_ci"; + private static readonly int MIN_POOL_SIZE=10; + private static readonly int MAX_POOL_SIZE=100; + public static readonly int COMMAND_TIMEOUT=300; // seconds + private static readonly int CONNECTION_TIMEOUT=300; // seconds /// CreateMySqlConnection - Creates an SQL connection. /// datasource. @@ -40,13 +40,12 @@ namespace MarketData.Utils sb.Append("MinimumPoolSize=").Append(MIN_POOL_SIZE).Append(";"); sb.Append("MaximumPoolSize=").Append(MAX_POOL_SIZE).Append(";"); sb.Append("allow user variables=").Append("true").Append("; "); - sb.Append("default command timeout=").Append("300").Append("; "); - sb.Append("connect timeout=").Append("300").Append(";"); + sb.Append("default command timeout=").Append(COMMAND_TIMEOUT).Append("; "); + sb.Append("connect timeout=").Append(CONNECTION_TIMEOUT).Append(";"); sb.Append("charset=").Append("utf8mb4").Append(" "); connectionString=sb.ToString(); MySqlConnection connection=new MySqlConnection(connectionString); connection.Open(); - //SetConnectionCollation(connection); return connection; } catch(Exception exception) @@ -74,8 +73,8 @@ namespace MarketData.Utils sb.Append("SslMode=").Append("None").Append("; "); sb.Append("pooling=").Append("true").Append(";"); sb.Append("allow user variables=").Append("true").Append("; "); - sb.Append("default command timeout=").Append("300").Append("; "); - sb.Append("connect timeout=").Append("300").Append(";"); + sb.Append("default command timeout=").Append(COMMAND_TIMEOUT).Append("; "); + sb.Append("connect timeout=").Append(CONNECTION_TIMEOUT).Append(";"); sb.Append("MinimumPoolSize=").Append(MIN_POOL_SIZE).Append(";"); sb.Append("MaximumPoolSize=").Append(MAX_POOL_SIZE).Append(";"); sb.Append("charset=").Append("utf8mb4").Append(" "); @@ -193,7 +192,7 @@ namespace MarketData.Utils sb.Append(")"); return sb.ToString(); } -// + public static String CreateInClause(List list) { StringBuilder sb = new StringBuilder(); @@ -243,21 +242,5 @@ namespace MarketData.Utils { return SqlString(value,true); } - public static string Set(string name,string value) - { - StringBuilder sb=new StringBuilder(); - sb.Append("SET ").Append(name).Append("=").Append(AddQuotes(value)).Append(" ").Append(Collation()).Append(";"); - return sb.ToString(); - } - public static string Set(string name,DateTime date) - { - StringBuilder sb=new StringBuilder(); - sb.Append("SET ").Append(name).Append("=").Append(SqlDate(date,true)).Append(" ").Append(Collation()).Append(";"); - return sb.ToString(); - } - private static string Collation() - { - return "COLLATE "+COLLATION; - } } }