471 lines
19 KiB
C#
Executable File
471 lines
19 KiB
C#
Executable File
using System.Text;
|
|
using MySql.Data.MySqlClient;
|
|
using MarketData.MarketDataModel;
|
|
using MarketData.Utils;
|
|
|
|
namespace MarketData.DataAccess
|
|
{
|
|
public class HeadlinesDA
|
|
{
|
|
private HeadlinesDA()
|
|
{
|
|
}
|
|
public static DateTime GetMaxHeadlineDate()
|
|
{
|
|
MySqlConnection sqlConnection = null;
|
|
MySqlDataReader sqlDataReader = null;
|
|
MySqlCommand sqlCommand = null;
|
|
String strQuery = null;
|
|
DateTime maxDate = Utility.Epoch;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
|
|
sb.Append("select max(asof) from headlines");
|
|
strQuery = sb.ToString(); ;
|
|
sqlCommand = new MySqlCommand(strQuery, sqlConnection);
|
|
sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
|
|
sqlDataReader = sqlCommand.ExecuteReader();
|
|
if (sqlDataReader.Read())
|
|
{
|
|
maxDate=sqlDataReader.GetDateTime(0);
|
|
}
|
|
return maxDate;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, exception);
|
|
return Utility.Epoch;
|
|
}
|
|
finally
|
|
{
|
|
if (null != sqlCommand) sqlCommand.Dispose();
|
|
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
|
|
if (null != sqlConnection) sqlConnection.Close();
|
|
}
|
|
}
|
|
public static List<String> GetHeadlineDates()
|
|
{
|
|
MySqlConnection sqlConnection = null;
|
|
MySqlDataReader sqlDataReader = null;
|
|
MySqlCommand sqlCommand=null;
|
|
String strQuery = null;
|
|
List<String> headlineDates=new List<String>();
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
|
|
sb.Append("select distinct asof from headlines order by 1 desc");
|
|
strQuery = sb.ToString(); ;
|
|
sqlCommand = new MySqlCommand(strQuery, sqlConnection);
|
|
sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
|
|
sqlDataReader = sqlCommand.ExecuteReader();
|
|
while (sqlDataReader.Read())
|
|
{
|
|
headlineDates.Add(sqlDataReader.GetDateTime(0).ToShortDateString());
|
|
}
|
|
return headlineDates;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return headlineDates;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=sqlCommand)sqlCommand.Dispose();
|
|
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
|
|
if (null != sqlConnection) sqlConnection.Close();
|
|
}
|
|
}
|
|
public static Headlines GetHeadlines(String symbol)
|
|
{
|
|
MySqlConnection sqlConnection = null;
|
|
MySqlDataReader sqlDataReader = null;
|
|
MySqlCommand sqlCommand=null;
|
|
String strQuery = null;
|
|
Headlines headlines=new Headlines();
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
|
|
sb.Append("select h.symbol, h.asof, h.headline, h.source, h.modified,sm.company from headlines h left outer join securitymaster sm on h.symbol=sm.symbol where h.symbol=").Append(SqlUtils.AddQuotes(symbol));
|
|
sb.Append(" order by h.asof desc, h.modified desc");
|
|
strQuery = sb.ToString(); ;
|
|
sqlCommand = new MySqlCommand(strQuery, sqlConnection);
|
|
sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
|
|
sqlDataReader = sqlCommand.ExecuteReader();
|
|
while (sqlDataReader.Read())
|
|
{
|
|
Headline headline=new Headline();
|
|
headline.Symbol=symbol;
|
|
headline.Date=sqlDataReader.GetDateTime(1);
|
|
headline.Entry=sqlDataReader.GetString(2);
|
|
headline.Source=sqlDataReader.GetString(3);
|
|
headline.Modified=sqlDataReader.GetDateTime(4);
|
|
headline.CompanyName=sqlDataReader.GetString(5);
|
|
headlines.Add(headline);
|
|
}
|
|
return headlines;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return headlines;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=sqlCommand)sqlCommand.Dispose();
|
|
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
|
|
if (null != sqlConnection) sqlConnection.Close();
|
|
}
|
|
}
|
|
public static Headlines GetHeadlines()
|
|
{
|
|
MySqlConnection sqlConnection = null;
|
|
MySqlDataReader sqlDataReader = null;
|
|
MySqlCommand sqlCommand=null;
|
|
String strQuery = null;
|
|
Headlines headlines=new Headlines();
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
|
|
sb.Append("select h.symbol, h.asof, h.headline, h.source, h.modified, sm.company from headlines h left outer join securitymaster sm on h.symbol=sm.symbol");
|
|
sb.Append(" order by h.asof desc,h.symbol,h.modified desc;");
|
|
strQuery = sb.ToString(); ;
|
|
sqlCommand = new MySqlCommand(strQuery, sqlConnection);
|
|
sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
|
|
sqlDataReader = sqlCommand.ExecuteReader();
|
|
while (sqlDataReader.Read())
|
|
{
|
|
Headline headline=new Headline();
|
|
headline.Symbol=sqlDataReader.GetString(0);
|
|
headline.Date=sqlDataReader.GetDateTime(1);
|
|
headline.Entry=sqlDataReader.GetString(2);
|
|
headline.Source=sqlDataReader.GetString(3);
|
|
headline.Modified=sqlDataReader.GetDateTime(4);
|
|
headline.CompanyName=sqlDataReader.GetString(5);
|
|
headlines.Add(headline);
|
|
}
|
|
return headlines;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return headlines;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=sqlCommand)sqlCommand.Dispose();
|
|
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
|
|
if (null != sqlConnection) sqlConnection.Close();
|
|
}
|
|
}
|
|
// This was authored for mobile app. It wants the sorting to match the WPF app.
|
|
public static Headlines GetLatestHeadlines()
|
|
{
|
|
MySqlConnection sqlConnection = null;
|
|
MySqlDataReader sqlDataReader = null;
|
|
MySqlCommand sqlCommand = null;
|
|
String strQuery = null;
|
|
Headlines headlines = new Headlines();
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
|
|
sb.Append("select h.symbol, h.asof, h.headline, h.source, h.modified, sm.company from headlines h left outer join securitymaster sm on h.symbol=sm.symbol");
|
|
sb.Append(" where h.asof=(select max(asof) from headlines)");
|
|
sb.Append(" order by h.asof desc,h.modified desc,h.symbol desc;");
|
|
strQuery = sb.ToString(); ;
|
|
sqlCommand = new MySqlCommand(strQuery, sqlConnection);
|
|
sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
|
|
sqlDataReader = sqlCommand.ExecuteReader();
|
|
while (sqlDataReader.Read())
|
|
{
|
|
Headline headline = new Headline();
|
|
headline.Symbol = sqlDataReader.GetString(0);
|
|
headline.Date = sqlDataReader.GetDateTime(1);
|
|
headline.Entry = sqlDataReader.GetString(2);
|
|
headline.Source = sqlDataReader.GetString(3);
|
|
headline.Modified = sqlDataReader.GetDateTime(4);
|
|
headline.CompanyName = sqlDataReader.GetString(5);
|
|
headlines.Add(headline);
|
|
}
|
|
return headlines;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, exception);
|
|
return headlines;
|
|
}
|
|
finally
|
|
{
|
|
if (null != sqlCommand) sqlCommand.Dispose();
|
|
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
|
|
if (null != sqlConnection) sqlConnection.Close();
|
|
}
|
|
}
|
|
|
|
public static Headlines GetHeadlines(String symbol,DateTime dateTime)
|
|
{
|
|
MySqlConnection sqlConnection = null;
|
|
MySqlDataReader sqlDataReader = null;
|
|
MySqlCommand sqlCommand=null;
|
|
String strQuery = null;
|
|
Headlines headlines=new Headlines();
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
|
|
sb.Append("select h.symbol, h.asof, h.headline, h.source, h.modified, sm.company from headlines h left outer join securitymaster sm on h.symbol=sm.symbol where h.asof=").Append(SqlUtils.AddQuotes(SqlUtils.SqlDate(dateTime)));
|
|
sb.Append(" and h.symbol=").Append(SqlUtils.AddQuotes(symbol));
|
|
sb.Append("order by h.asof desc, h.modified desc");
|
|
strQuery = sb.ToString(); ;
|
|
sqlCommand = new MySqlCommand(strQuery, sqlConnection);
|
|
sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
|
|
sqlDataReader = sqlCommand.ExecuteReader();
|
|
while (sqlDataReader.Read())
|
|
{
|
|
Headline headline=new Headline();
|
|
headline.Symbol=sqlDataReader.GetString(0);
|
|
headline.Date=sqlDataReader.GetDateTime(1);
|
|
headline.Entry=sqlDataReader.GetString(2);
|
|
headline.Source=sqlDataReader.GetString(3);
|
|
headline.Modified=sqlDataReader.GetDateTime(4);
|
|
headline.CompanyName=sqlDataReader.GetString(5);
|
|
headlines.Add(headline);
|
|
}
|
|
return headlines;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return headlines;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=sqlCommand)sqlCommand.Dispose();
|
|
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
|
|
if (null != sqlConnection) sqlConnection.Close();
|
|
}
|
|
}
|
|
|
|
public static Headlines GetHeadlines(DateTime dateTime)
|
|
{
|
|
MySqlConnection sqlConnection = null;
|
|
MySqlDataReader sqlDataReader = null;
|
|
MySqlCommand sqlCommand=null;
|
|
String strQuery = null;
|
|
Headlines headlines=new Headlines();
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
|
|
sb.Append("select h.symbol, h.asof, h.headline, h.source, h.modified, sm.company from headlines h left outer join securitymaster sm on h.symbol=sm.symbol where h.asof=").Append(SqlUtils.AddQuotes(SqlUtils.SqlDate(dateTime)));
|
|
sb.Append("order by h.modified desc,h.symbol");
|
|
strQuery = sb.ToString(); ;
|
|
sqlCommand = new MySqlCommand(strQuery, sqlConnection);
|
|
sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
|
|
sqlDataReader = sqlCommand.ExecuteReader();
|
|
while (sqlDataReader.Read())
|
|
{
|
|
Headline headline=new Headline();
|
|
headline.Symbol=sqlDataReader.GetString(0);
|
|
headline.Date=sqlDataReader.GetDateTime(1);
|
|
headline.Entry=sqlDataReader.GetString(2);
|
|
headline.Source=sqlDataReader.GetString(3);
|
|
headline.Modified=sqlDataReader.GetDateTime(4);
|
|
headline.CompanyName=sqlDataReader.GetString(5);
|
|
headlines.Add(headline);
|
|
}
|
|
return headlines;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return headlines;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=sqlCommand)sqlCommand.Dispose();
|
|
if (null != sqlDataReader) {sqlDataReader.Close();sqlDataReader.Dispose();}
|
|
if (null != sqlConnection) sqlConnection.Close();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// InsertHeadlines - Inserts headlines by batching them up into groups.
|
|
/// </summary>
|
|
/// <param name="headlines"></param>
|
|
/// <returns></returns>
|
|
public static bool InsertHeadlines(Headlines headlines)
|
|
{
|
|
const int batchSize = 1000;
|
|
MySqlConnection sqlConnection=null;
|
|
MySqlTransaction sqlTransaction=null;
|
|
|
|
try
|
|
{
|
|
if (null == headlines || 0 == headlines.Count) return false;
|
|
sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
|
|
sqlTransaction = sqlConnection.BeginTransaction(System.Data.IsolationLevel.ReadCommitted);
|
|
headlines=new Headlines(headlines.Distinct(new HeadlinesEqualityComparer()).ToList());
|
|
foreach (IEnumerable<Headline> batch in headlines.Chunk(batchSize))
|
|
{
|
|
SafeInsertBatch(batch.ToList(), sqlConnection, sqlTransaction);
|
|
}
|
|
sqlTransaction.Commit();
|
|
return true;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
sqlTransaction?.Rollback(); // This will rollback without explicityl calling Rollback but we'll do this anyway.
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return false;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=sqlTransaction)sqlTransaction.Dispose();
|
|
if(null!=sqlConnection) sqlConnection.Close();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// SafeInsertBatch - This recursive approach allows us to catch exceptions and recursively call SafeInsertBatch continually dividing the
|
|
/// batch until we find the offensive row.
|
|
/// </summary>
|
|
/// <param name="batch"></param>
|
|
/// <param name="sqlConnection"></param>
|
|
/// <param name="sqlTransaction"></param>
|
|
private static void SafeInsertBatch(List<Headline> batch, MySqlConnection sqlConnection, MySqlTransaction sqlTransaction)
|
|
{
|
|
try
|
|
{
|
|
InsertBatch(batch, sqlConnection, sqlTransaction);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// If only one row, it's the offender
|
|
if (batch.Count == 1)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, ex);
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"Bad row: Symbol={batch[0].Symbol}, Headline={batch[0].Entry}");
|
|
return;
|
|
}
|
|
// Split batch and retry
|
|
int mid = batch.Count / 2;
|
|
List<Headline> firstHalf = batch.Take(mid).ToList();
|
|
List<Headline> secondHalf = batch.Skip(mid).ToList();
|
|
SafeInsertBatch(firstHalf, sqlConnection, sqlTransaction);
|
|
SafeInsertBatch(secondHalf, sqlConnection, sqlTransaction);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Inserts a batch of headlines
|
|
/// </summary>
|
|
/// <param name="batch"></param>
|
|
/// <param name="sqlConnection"></param>
|
|
/// <param name="sqlTransaction"></param>
|
|
private static void InsertBatch(IEnumerable<Headline> batch, MySqlConnection sqlConnection, MySqlTransaction sqlTransaction)
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
using MySqlCommand sqlCommand = new MySqlCommand();
|
|
sqlCommand.Connection = sqlConnection;
|
|
sqlCommand.Transaction = sqlTransaction;
|
|
DateTime now = DateTime.Now;
|
|
|
|
sb.Append("INSERT IGNORE INTO Headlines (symbol, asof, headline, source, modified) VALUES ");
|
|
int index = 0;
|
|
foreach (Headline headline in batch)
|
|
{
|
|
if (index > 0) sb.Append(",");
|
|
sb.Append($"(@symbol{index}, @asof{index}, @headline{index}, @source{index}, @modified{index})");
|
|
sqlCommand.Parameters.AddWithValue($"@symbol{index}", headline.Symbol);
|
|
sqlCommand.Parameters.AddWithValue($"@asof{index}", headline.Date);
|
|
sqlCommand.Parameters.AddWithValue($"@headline{index}", headline.Entry);
|
|
sqlCommand.Parameters.AddWithValue($"@source{index}", headline.Source);
|
|
DateTime modified = Utility.IsEpoch(headline.Modified) ? now : headline.Modified;
|
|
sqlCommand.Parameters.AddWithValue($"@modified{index}", modified);
|
|
index++;
|
|
}
|
|
sqlCommand.CommandText = sb.ToString();
|
|
sqlCommand.ExecuteNonQuery();
|
|
}
|
|
|
|
/// <summary>
|
|
/// InsertHeadline - This is now parameterized. The MySql driver should handle all escaping etc.,
|
|
/// </summary>
|
|
/// <param name="headline"></param>
|
|
/// <param name="sqlConnection"></param>
|
|
/// <param name="sqlTransaction"></param>
|
|
/// <returns></returns>
|
|
private static bool InsertHeadline(Headline headline,MySqlConnection sqlConnection,MySqlTransaction sqlTransaction)
|
|
{
|
|
MySqlCommand sqlCommand=null;
|
|
String strQuery = null;
|
|
|
|
try
|
|
{
|
|
if (null == headline || null == headline.Symbol || null==headline.Entry) return false;
|
|
strQuery = @"INSERT INTO Headlines (symbol, asof, headline, source, modified) VALUES (@symbol, @asof, @headline, @source, @modified)";
|
|
sqlCommand = new MySqlCommand(strQuery, sqlConnection, sqlTransaction);
|
|
sqlCommand.Parameters.AddWithValue("@symbol", headline.Symbol);
|
|
sqlCommand.Parameters.AddWithValue("@asof", headline.Date);
|
|
sqlCommand.Parameters.AddWithValue("@headline", headline.Entry);
|
|
sqlCommand.Parameters.AddWithValue("@source", headline.Source);
|
|
DateTime modified = Utility.IsEpoch(headline.Modified) ? DateTime.Now : headline.Modified;
|
|
sqlCommand.Parameters.AddWithValue("@modified", modified);
|
|
sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
|
|
sqlCommand.ExecuteNonQuery();
|
|
return true;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
SqlUtils.LogCommandParameters(strQuery, sqlCommand);
|
|
return false;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=sqlCommand)sqlCommand.Dispose();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// HeadlineExists - The now uses parameterized arguments now. The driver will handle escaping etc.,
|
|
/// </summary>
|
|
/// <param name="headline"></param>
|
|
/// <param name="sqlConnection"></param>
|
|
/// <param name="sqlTransaction"></param>
|
|
/// <returns></returns>
|
|
private static bool HeadlineExists(Headline headline,MySqlConnection sqlConnection,MySqlTransaction sqlTransaction)
|
|
{
|
|
try
|
|
{
|
|
if (null == headline || null == headline.Symbol) return false;
|
|
string strQuery = @"SELECT count(*) FROM headlines WHERE asof = @asof AND headline = @headline";
|
|
using MySqlCommand sqlCommand = new MySqlCommand(strQuery, sqlConnection, sqlTransaction);
|
|
sqlCommand.Parameters.AddWithValue("@asof", headline.Date);
|
|
sqlCommand.Parameters.AddWithValue("@headline", headline.Entry); // This will handle proper escaping of characters etc.,
|
|
sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
|
|
int result = Convert.ToInt32(sqlCommand.ExecuteScalar());
|
|
return 0!=result;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
}
|