Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

InsiderTransactions refactorings.

Browse Source
This commit is contained in:
Sean Kessler
2024-02-26 19:48:35 -05:00
parent dc73a51156
commit 661bd3d7ec
7 changed files with 422 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
MarketDataLib/DataAccess/InsiderTransactionDA.cs
View File

@@ -173,7 +173,7 @@ namespace MarketData.DataAccess
if(null!=sqlTransaction)sqlTransaction.Dispose();
}
}
private static bool DeleteInsiderTransactions(InsiderTransactions insiderTransactions, MySqlConnection sqlConnection) //, MySqlTransaction sqlTransaction)
private static bool DeleteInsiderTransactions(InsiderTransactions insiderTransactions, MySqlConnection sqlConnection)
{
StringBuilder sb = null;
String strQuery = null;
@@ -210,5 +210,45 @@ namespace MarketData.DataAccess
if(null!=sqlTransaction)sqlTransaction.Dispose();
}
}
/// <summary>
/// Delete InsiderTransaction for given symbol for all years>=given year.
/// </summary>
/// <param name="symbol">The symbol.</param>
/// <param name="yearsGreaterEqual">All years greater then or equal to the year.</param>
public static bool DeleteInsiderTransactionsYearsGreaterEqual(String symbol, int yearsGreaterEqual)
{
MySqlConnection sqlConnection = null;
StringBuilder sb = null;
String strQuery = null;
MySqlTransaction sqlTransaction = null;
try
{
if (null == symbol) return false;
sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
sqlTransaction = sqlConnection.BeginTransaction(System.Data.IsolationLevel.ReadCommitted);
sb = new StringBuilder();
sb.Append("DELETE FROM InsiderTransaction WHERE symbol=");
sb.Append("'").Append(symbol).Append("'");
sb.Append(" AND YEAR(filing_date)>=").Append(yearsGreaterEqual);
strQuery = sb.ToString();
MySqlCommand sqlCommand = new MySqlCommand(strQuery, sqlConnection, sqlTransaction);
sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
sqlCommand.ExecuteNonQuery();
sqlTransaction.Commit();
return true;
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
if (null != strQuery) MDTrace.WriteLine(LogLevel.DEBUG,"Query was " + strQuery);
return false;
}
finally
{
if(null!=sqlTransaction)sqlTransaction.Dispose();
if(null!=sqlConnection)sqlConnection.Close();
}
}
}
}

1
MarketDataLib/Helper/FundamentalMarketDataHelper.cs
View File

@@ -11,7 +11,6 @@ namespace MarketData.Helper
{
public class FundamentalMarketDataHelper : MarketDataHelperBase<String>
{
// private static int MaxThreads = 5; //(int)ThreadHelperEnum.MaxThreads;
private static int MaxThreads = 5; //(int)ThreadHelperEnum.MaxThreads;
private UpdateManager updateManager=new UpdateManager();

81
MarketDataLib/Helper/InsiderTransactionMarketDataHelper.cs
View File

@@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
@@ -9,12 +8,23 @@ using MarketData.Utils;
namespace MarketData.Helper
{
public class InsiderTransactionThreadHelper : ThreadHelper
{
public InsiderTransactionThreadHelper(String symbol, ManualResetEvent resetEvent)
: base(symbol, resetEvent)
{
}
public int Year { get; set; }
}
public class InsiderTransactionMarketDataHelper
{
private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads;
private static int MaxThreads = 10; // 10 threads avoids receiving HTTP Response 429 (Too many requests)
private static int WAIT_TIME_MS=500; // wait between request
private List<String> symbols;
private int currentIndex = 0;
private UpdateManager UpdateManager = new UpdateManager();
public InsiderTransactionMarketDataHelper()
{
@@ -83,6 +93,45 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadInsiderTransactions]End, total took {0}(ms)",profiler.End()));
}
}
public bool LoadInsiderTransactionsYear(List<String> symbols,int year)
{
Profiler profiler=new Profiler();
try
{
this.symbols = symbols;
currentIndex = 0;
UpdateManager.Prepare("load_insider_transactions_year.txt", 7); // use max age 7 days
this.symbols=this.symbols.Except(new List<String>(UpdateManager.Entries)).ToList();
while (true)
{
List<String> queueSymbols = GetQueueSymbols();
if (null == queueSymbols || 0 == queueSymbols.Count) break;
ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
{
resetEvents[eventIndex] = new ManualResetEvent(false);
}
for (int index = 0; index < queueSymbols.Count; index++)
{
InsiderTransactionThreadHelper threadHelper = new InsiderTransactionThreadHelper(queueSymbols[index], resetEvents[index]);
threadHelper.UpdateManager=UpdateManager;
threadHelper.Year=year;
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackYearGreaterEqual, threadHelper);
try { Thread.Sleep(WAIT_TIME_MS); } catch(Exception) { ;} // wait
}
MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);
}
return true;
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadInsiderTransactions]End, total took {0}(ms)",profiler.End()));
}
}
private List<String> GetQueueSymbols()
{
List<String> queueSymbols = new List<String>();
@@ -94,6 +143,7 @@ namespace MarketData.Helper
currentIndex = index;
return queueSymbols;
}
public void ThreadPoolCallback(Object threadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
@@ -102,6 +152,7 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} ended for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
threadHelper.ResetEvent.Set();
}
public static void LoadInsiderTransactionsSymbolEx(String symbol)
{
symbol = symbol.ToUpper();
@@ -116,5 +167,31 @@ namespace MarketData.Helper
InsiderTransactionDA.InsertInsiderTransactions(insiderTransactions);
MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions - Done.");
}
public void ThreadPoolCallbackYearGreaterEqual(Object threadHelperContext)
{
InsiderTransactionThreadHelper threadHelper = (InsiderTransactionThreadHelper)threadHelperContext;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
LoadInsiderTransactionsYearGreaterEqualEx(threadHelper.Symbol,threadHelper.Year);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} ended for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
threadHelper.UpdateManager.Add(threadHelper.Symbol);
threadHelper.ResetEvent.Set();
}
public static void LoadInsiderTransactionsYearGreaterEqualEx(String symbol, int year)
{
symbol = symbol.ToUpper();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load insider transactions for {0} years>={1}", symbol,year));
InsiderTransactions insiderTransactions = MarketDataHelper.GetInsiderTransactionsYear(symbol, year);
if (null == insiderTransactions || 0 == insiderTransactions.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No insider transactions for {0} years>={1}", symbol,year));
return;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions, Saving {0} records for {1} years>={2}", insiderTransactions.Count, symbol, year));
InsiderTransactionDA.DeleteInsiderTransactionsYearsGreaterEqual(symbol, year);
InsiderTransactionDA.InsertInsiderTransactions(insiderTransactions);
MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions - Done.");
}
}
}

298
MarketDataLib/Helper/InsiderTransactionsParser.cs
View File

@@ -6,12 +6,13 @@ using HtmlAgilityPack;
using MarketData.Utils;
using MarketData.MarketDataModel;
using MarketDataLib.Utility;
using System.Collections.Concurrent;
namespace MarketData.Helper
{
public class InsiderTransactionsParser
{
private static Dictionary<String,String> transactionCodes=new Dictionary<String,String>();
private static ConcurrentDictionary<String,String> transactionCodes=new ConcurrentDictionary<String,String>();
private static InsiderTransactionsParser instance=null;
private InsiderTransactionsParser()
{
@@ -26,6 +27,140 @@ namespace MarketData.Helper
return instance;
}
}
/// <summary>
/// Parse the SECFilings. Each SECFiling may contain a list of InsiderTransaction
/// </summary>
/// <param name="secFilings">The SECFilings.</param>
public InsiderTransactions Parse(SECFilings secFilings)
{
try
{
InsiderTransactions insiderTransactions=new InsiderTransactions();
if(null==secFilings || 0==secFilings.Count)return insiderTransactions;
foreach(SECFiling secFiling in secFilings)
{
InsiderTransactions secFilinginsiderTransactions =
Parse(secFiling.FormText, secFiling.Symbol,
secFiling.SECAccessionNumber,
secFiling.Form, secFiling.FilingDate);
if(null!=secFilinginsiderTransactions && secFilinginsiderTransactions.Count>0)
insiderTransactions.AddRange(secFilinginsiderTransactions);
}
return insiderTransactions;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse SECFiling got exception {0}",exception.ToString()));
return new InsiderTransactions();
}
}
public InsiderTransactions Parse(String strHtml,String symbol, String secAccessionNumber, String form, DateTime filingDate)
{
try
{
InsiderTransactions insiderTransactions = new InsiderTransactions();
if(null==strHtml)return null;
List<Dictionary<String,String>> dictionaryList=new List<Dictionary<String,String>>();
String insiderName=null;
byte[] streamBytes = Encoding.ASCII.GetBytes(strHtml);
MemoryStream memoryStream = new MemoryStream(streamBytes);
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.Load(memoryStream);
HtmlNodeCollection tables=htmlDocument.DocumentNode.SelectNodes("//table");
if(null==tables || tables.Count<5)return null;
HtmlNode nameAndAddressTable=FindTable(tables,"1. Name and Address of Reporting Person");
if(null==nameAndAddressTable)return null;
HtmlNodeCollection nameAndAddressRows = nameAndAddressTable.SelectNodes(".//tr");
if(nameAndAddressRows.Count<2)return null;
insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);
for(int index=0;index<tables.Count;index++)
{
HtmlNodeCollection nodeCollection = tables[index].SelectNodes(".//tr");
foreach(HtmlNode node in nodeCollection)
{
HtmlNodeCollection dataCollection = node.SelectNodes(".//td");
if(null==dataCollection || 0==dataCollection.Count)continue;
if(11==dataCollection.Count)
{
Dictionary<String,String> dictionary=new Dictionary<String,String>();
dictionary.Add("Title of Security",dataCollection[0].InnerHtml);
dictionary.Add("Transaction Date",dataCollection[1].InnerHtml);
dictionary.Add("Transaction Code",dataCollection[3].InnerHtml);
dictionary.Add("Securities Acquired or Disposed",dataCollection[5].InnerHtml);
dictionary.Add("Acquired or Disposed",dataCollection[6].InnerHtml);
dictionary.Add("Price",dataCollection[7].InnerHtml);
dictionary.Add("Ownership Form",dataCollection[9].InnerHtml);
dictionaryList.Add(dictionary);
}
}
}
foreach(Dictionary<String,String> dictionary in dictionaryList)
{
String strItem=null;
InsiderTransaction insiderTransaction=new InsiderTransaction();
insiderTransaction.Symbol=symbol;
insiderTransaction.SECAccessionNumber=secAccessionNumber;
insiderTransaction.Form=form;
insiderTransaction.FilingDate=filingDate;
insiderTransaction.InsiderName=insiderName;
insiderTransaction.Securities=GetFirstSection(Sections.GetSections(dictionary["Title of Security"]));
insiderTransaction.OwnershipType=GetOwnershipForm(Sections.GetSections(dictionary["Ownership Form"]));
if(String.IsNullOrEmpty(insiderTransaction.OwnershipType))continue;
strItem=GetFirstSection(Sections.GetSections(dictionary["Transaction Date"]));
if(String.IsNullOrEmpty(strItem))continue;
insiderTransaction.TransactionDate=Utility.ParseDate(strItem);
insiderTransaction.NatureOfTransaction=transactionCodes.ContainsKey(GetFirstSection(Sections.GetSections(dictionary["Transaction Code"])))?transactionCodes[GetFirstSection(Sections.GetSections(dictionary["Transaction Code"]))]:Constants.CONST_QUESTION;
insiderTransaction.NumberOrValueAcquiredDisposed=FeedParser.ParseValue(GetFirstSection(Sections.GetSections(dictionary["Securities Acquired or Disposed"])));
insiderTransaction.FormRowNumber=((decimal)insiderTransaction.NumberOrValueAcquiredDisposed).ToString();
insiderTransaction.Price=FeedParser.ParseValue(CombineSections(Sections.GetSections(dictionary["Price"])));
String acquiredOrDisposed=GetAcquiredOrDisposed(dictionary);
if(null==acquiredOrDisposed)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse - Cannot determine acquied or disposed for accession#{0}",secAccessionNumber));
continue;
}
if(acquiredOrDisposed.Equals("D"))
{
insiderTransaction.NumberOrValueAcquiredDisposed*=-1;
}
insiderTransactions.Add(insiderTransaction);
}
return insiderTransactions;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse - Got exception {0}",exception.ToString()));
return null;
}
}
private HtmlNode FindTable(HtmlNodeCollection tables,String startsWith)
{
if(null==tables)return null;
for(int index=0;index<tables.Count;index++)
{
HtmlNodeCollection collection = tables[index].SelectNodes(".//tr");
for(int itemIndex=0;itemIndex<collection.Count;itemIndex++)
{
String marker=collection[itemIndex].InnerText;
if(marker.StartsWith(startsWith))
{
return tables[index];
}
}
}
return null;
}
private String ApplyNameCase(String name)
{
StringBuilder sb=new StringBuilder();
@@ -40,18 +175,33 @@ namespace MarketData.Helper
}
return sb.ToString();
}
private String GetOwnershipForm(List<String> sections)
{
String item=GetFirstSection(sections);
if(null==item)item="";
item=item.ToUpper();
if(item.Equals("D"))return "Direct Ownership";
else if(item.Equals("I"))return "Indirect Ownership";
return "";
}
private String GetAcquiredOrDisposed(Dictionary<String,String> dictionary)
{
if(!dictionary.ContainsKey("Acquired or Disposed"))return null;
List<String> section=Sections.GetSections(dictionary["Acquired or Disposed"]);
if(null==section||0==section.Count)return null;
String acquiredOrDisposed=section[0];
if(null==acquiredOrDisposed)return null;
return acquiredOrDisposed.Trim().ToUpper();
}
private String GetFirstSection(List<String> sections)
{
if(null==sections || 0==sections.Count)return "";
return sections[0];
}
private String CombineSections(List<String> sections)
{
StringBuilder sb=new StringBuilder();
@@ -62,135 +212,29 @@ namespace MarketData.Helper
}
return sb.ToString();
}
public InsiderTransactions Parse(SECFilings secFilings)
{
lock(instance)
{
try
{
InsiderTransactions insiderTransactions=new InsiderTransactions();
if(null==secFilings || 0==secFilings.Count)return insiderTransactions;
foreach(SECFiling secFiling in secFilings)
{
List<Dictionary<String,String>> dictionaryList=new List<Dictionary<String,String>>();
String insiderName=null;
String relationshipOfReportingPerson=null;
byte[] streamBytes = Encoding.ASCII.GetBytes(secFiling.FormText);
MemoryStream memoryStream = new MemoryStream(streamBytes);
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.Load(memoryStream);
HtmlNodeCollection tables=htmlDocument.DocumentNode.SelectNodes("//table");
if(null==tables || tables.Count<5)continue;
HtmlNode nameAndAddressTable=FindTable(tables,"1. Name and Address of Reporting Person");
if(null==nameAndAddressTable)continue;
HtmlNodeCollection nameAndAddressRows = nameAndAddressTable.SelectNodes(".//tr");
if(nameAndAddressRows.Count<2)continue;
insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);
//HtmlNodeCollection nameAndAddressRows = tables[4].SelectNodes(".//tr");
//if(nameAndAddressRows.Count<10)continue;
//insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);
//relationshipOfReportingPerson=Utility.RemoveHtml(nameAndAddressRows[9].InnerText);
for(int index=0;index<tables.Count;index++)
{
HtmlNodeCollection nodeCollection = tables[index].SelectNodes(".//tr");
foreach(HtmlNode node in nodeCollection)
{
HtmlNodeCollection dataCollection = node.SelectNodes(".//td");
if(null==dataCollection || 0==dataCollection.Count)continue;
if(11==dataCollection.Count)
{
Dictionary<String,String> dictionary=new Dictionary<String,String>();
dictionary.Add("Title of Security",dataCollection[0].InnerHtml);
dictionary.Add("Transaction Date",dataCollection[1].InnerHtml);
dictionary.Add("Transaction Code",dataCollection[3].InnerHtml);
dictionary.Add("Securities Acquired or Disposed",dataCollection[5].InnerHtml);
dictionary.Add("Acquired or Disposed",dataCollection[6].InnerHtml);
dictionary.Add("Price",dataCollection[7].InnerHtml);
dictionary.Add("Ownership Form",dataCollection[9].InnerHtml);
dictionaryList.Add(dictionary);
}
}
}
foreach(Dictionary<String,String> dictionary in dictionaryList)
{
String strItem=null;
InsiderTransaction insiderTransaction = new InsiderTransaction();
insiderTransaction.Symbol=secFiling.Symbol;
insiderTransaction.SECAccessionNumber=secFiling.SECAccessionNumber;
insiderTransaction.FormRowNumber=(secFiling.Sequence+1).ToString();
insiderTransaction.Form=secFiling.Form;
insiderTransaction.FilingDate=secFiling.FilingDate;
insiderTransaction.InsiderName=insiderName;
insiderTransaction.Securities=GetFirstSection(Sections.GetSections(dictionary["Title of Security"]));
insiderTransaction.OwnershipType=GetOwnershipForm(Sections.GetSections(dictionary["Ownership Form"]));
if(String.IsNullOrEmpty(insiderTransaction.OwnershipType))continue;
strItem=GetFirstSection(Sections.GetSections(dictionary["Transaction Date"]));
if(String.IsNullOrEmpty(strItem))continue;
insiderTransaction.TransactionDate=Utility.ParseDate(strItem);
insiderTransaction.NatureOfTransaction=transactionCodes.ContainsKey(GetFirstSection(Sections.GetSections(dictionary["Transaction Code"])))?transactionCodes[GetFirstSection(Sections.GetSections(dictionary["Transaction Code"]))]:Constants.CONST_QUESTION;
insiderTransaction.NumberOrValueAcquiredDisposed=FeedParser.ParseValue(GetFirstSection(Sections.GetSections(dictionary["Securities Acquired or Disposed"])));
insiderTransaction.Price=FeedParser.ParseValue(CombineSections(Sections.GetSections(dictionary["Price"])));
insiderTransactions.Add(insiderTransaction);
}
}
return insiderTransactions;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse SECFiling got exception {0}",exception.ToString()));
return new InsiderTransactions();
}
}
}
private HtmlNode FindTable(HtmlNodeCollection tables,String startsWith)
{
if(null==tables)return null;
for(int index=0;index<tables.Count;index++)
{
HtmlNodeCollection nameAndAddressRows = tables[index].SelectNodes(".//tr");
for(int itemIndex=0;itemIndex<nameAndAddressRows.Count;itemIndex++)
{
String marker=nameAndAddressRows[itemIndex].InnerText;
if(marker.StartsWith(startsWith))
{
return tables[index];
}
}
}
return null;
}
private void BuildTransactionCodes()
{
transactionCodes.Add("P","Open market or private purchase of non-derivative or derivative security");
transactionCodes.Add("S","Open market or private sale of non-derivative or derivative security");
transactionCodes.Add("V","Transaction voluntarily reported earlier than required");
transactionCodes.Add("A","Grant, award or other acquisition pursuant to Rule 16b-3(d)");
transactionCodes.Add("D","Disposition to the issuer of issuer equity securities pursuant to Rule 16b-3(e)");
transactionCodes.Add("F","Payment of exercise price or tax liability by delivering or withholding securities incident to the receipt, exercise or vesting of a security issued in accordance with Rule 16b-3");
transactionCodes.Add("I","Discretionary transaction in accordance with Rule 16b-3(f) resulting in acquisition or disposition of issuer securities");
transactionCodes.Add("M","Exercise or conversion of derivative security exempted pursuant to Rule 16b-3");
transactionCodes.Add("C","Conversion of derivative security");
transactionCodes.Add("E","Expiration of short derivative position");
transactionCodes.Add("H","Expiration (or cancellation) of long derivative position with value received");
transactionCodes.Add("O","Exercise of out-of-the-money derivative security");
transactionCodes.Add("X","Exercise of in-the-money or at-the-money derivative security");
transactionCodes.Add("G","Bona fide gift");
transactionCodes.Add("L","Small acquisition under Rule 16a-6");
transactionCodes.Add("W","Acquisition or disposition by will or the laws of descent and distribution");
transactionCodes.Add("Z","Deposit into or withdrawal from voting trust");
transactionCodes.Add("J","Other acquisition or disposition (describe transaction)");
transactionCodes.Add("K","Transaction in equity swap or instrument with similar characteristics");
transactionCodes.Add("U","Disposition pursuant to a tender of shares in a change of control transaction");
transactionCodes.TryAdd("P","Open market or private purchase of non-derivative or derivative security");
transactionCodes.TryAdd("S","Open market or private sale of non-derivative or derivative security");
transactionCodes.TryAdd("V","Transaction voluntarily reported earlier than required");
transactionCodes.TryAdd("A","Grant, award or other acquisition pursuant to Rule 16b-3(d)");
transactionCodes.TryAdd("D","Disposition to the issuer of issuer equity securities pursuant to Rule 16b-3(e)");
transactionCodes.TryAdd("F","Payment of exercise price or tax liability by delivering or withholding securities incident to the receipt, exercise or vesting of a security issued in accordance with Rule 16b-3");
transactionCodes.TryAdd("I","Discretionary transaction in accordance with Rule 16b-3(f) resulting in acquisition or disposition of issuer securities");
transactionCodes.TryAdd("M","Exercise or conversion of derivative security exempted pursuant to Rule 16b-3");
transactionCodes.TryAdd("C","Conversion of derivative security");
transactionCodes.TryAdd("E","Expiration of short derivative position");
transactionCodes.TryAdd("H","Expiration (or cancellation) of long derivative position with value received");
transactionCodes.TryAdd("O","Exercise of out-of-the-money derivative security");
transactionCodes.TryAdd("X","Exercise of in-the-money or at-the-money derivative security");
transactionCodes.TryAdd("G","Bona fide gift");
transactionCodes.TryAdd("L","Small acquisition under Rule 16a-6");
transactionCodes.TryAdd("W","Acquisition or disposition by will or the laws of descent and distribution");
transactionCodes.TryAdd("Z","Deposit into or withdrawal from voting trust");
transactionCodes.TryAdd("J","Other acquisition or disposition (describe transaction)");
transactionCodes.TryAdd("K","Transaction in equity swap or instrument with similar characteristics");
transactionCodes.TryAdd("U","Disposition pursuant to a tender of shares in a change of control transaction");
}
}
}

126
MarketDataLib/Helper/MarketDataHelper.cs
View File

@@ -3,19 +3,15 @@ using System.IO;
using System.Collections.Generic;
using System.Text;
using System.Linq;
using System.Text.RegularExpressions;
using System.Data;
using System.Threading;
using System.Net;
using System.Collections;
using System.Reflection;
using System.Web;
using System.Xml.Linq;
using Newtonsoft.Json.Linq;
using HtmlAgilityPack;
using MarketData.MarketDataModel;
using MarketData.Utils;
using MarketData.Numerical;
using MarketData.CSVHelper;
using MarketData.DataAccess;
using MarketData.Integration;
@@ -1245,7 +1241,6 @@ namespace MarketData.Helper
// ***************************************************************************************************************************************************************************************
// ************************************ S E C F I L I N G S - F O R M 4 & F O R M 5 - F O R I N S I D E R T R A N S A C T I O N S S E C . G O V ***************************S
// ***************************************************************************************************************************************************************************************
// public static InsiderTransactions GetInsiderTransactions(String symbol,int timePeriodDays=7)
public static InsiderTransactions GetInsiderTransactions(String symbol,int timePeriodDays=30)
{
MemoryStream memoryStream = null;
@@ -1254,6 +1249,7 @@ namespace MarketData.Helper
DateTime minFilingDate=DateTime.Now;
int maxFilings=80;
bool continuationFlag=true;
int TIMEOUT_MS_BETWEEN_REQUESTS=1000;
try
{
@@ -1285,8 +1281,10 @@ namespace MarketData.Helper
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
if (null == tables || tables.Count < 1) return null;
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: symbol:{0} .Fetching {1} SECFilingDocuments ",symbol,rows.Count));
for (int row = 0; row < rows.Count && continuationFlag; row ++)
{
try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;}
HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
if (null != headerColumns && headerColumns.Count > 0) continue;
HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
@@ -1320,13 +1318,114 @@ namespace MarketData.Helper
if(null!=httpNetResponse)httpNetResponse.Dispose();
}
}
/// <summary>
/// Get InsiderTransactions for years greater than or equal to specified year.
/// For example. If 2023 is specified then get all for years>=2023
/// </summary>
/// <param name="symbol">The symbol.</param>
/// <param name="yearGreaterThanEqualTo">the new y-coordinate.</param>
public static InsiderTransactions GetInsiderTransactionsYear(String symbol,int yearGreaterThanEqualTo)
{
MemoryStream memoryStream = null;
HttpNetResponse httpNetResponse=null;
String[] descriptionStartsWith=new String[]{"Form 4","Form 5"};
int maxFilings=120;
bool continuationFlag=true;
int TIMEOUT_MS_BETWEEN_REQUESTS=1000;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}",symbol,yearGreaterThanEqualTo));
DateGenerator dateGenerator=new DateGenerator();
String cik=PricingDA.GetCIKForSymbol(symbol);
if(null==cik)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: No CIK for symbol {0}",symbol));
return null;
}
SECFilings secFilings = new SECFilings();
StringBuilder sb = new StringBuilder();
String strRequest;
sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
strRequest = sb.ToString();
WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return null;
}
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
memoryStream = new MemoryStream(streamBytes);
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.Load(memoryStream);
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
if (null == tables || tables.Count < 1) return null;
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}. Fetching {2} SECFilingDocuments ",symbol,yearGreaterThanEqualTo,rows.Count));
for (int row = 0; row < rows.Count && continuationFlag; row ++)
{
try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;}
HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
if (null != headerColumns && headerColumns.Count > 0) continue;
HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
if (null == dataColumns || 5 != dataColumns.Count) continue;
HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
if (null == nodes || 0 == nodes.Count) continue;
String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++)
{
SECFiling innerFiling = innerCollection[index];
if(innerFiling.FilingDate.Year<yearGreaterThanEqualTo)
{
continuationFlag=false;
break;
}
if(!descriptionStartsWith.Any(x=>innerFiling.Description.StartsWith(x)))continue;
secFilings.Add(innerFiling);
}
}
return InsiderTransactionsParser.GetInstance().Parse(secFilings);
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
return null;
}
finally
{
if (null != memoryStream) memoryStream.Close();
if(null!=httpNetResponse)httpNetResponse.Dispose();
}
}
// ***************************************************************************************************************************************************************************************
// ********************************************************************** S E C F I L I N G S - S E C . G O V ********************************************************************
// ***************************************************************************************************************************************************************************************
public static SECFilings GetSECFilings(String symbol,int maxFilings=80)
{
try
{
String cik=PricingDA.GetCIKForSymbol(symbol);
if(null==cik)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilings: No CIK for symbol {0}",symbol));
return null;
}
return GetSECFilings(symbol,cik,maxFilings);
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
return null; }
}
public static SECFilings GetSECFilings(String symbol,String cik,int maxFilings=80)
{
MemoryStream memoryStream = null;
HttpNetResponse httpNetResponse=null;
int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000;
try
{
SECFilings secFilings = new SECFilings();
@@ -1357,6 +1456,7 @@ namespace MarketData.Helper
HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
if (null == nodes || 0 == nodes.Count) continue;
String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
try { Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS); }catch(Exception) { ;}
SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++)
{
@@ -1384,11 +1484,12 @@ namespace MarketData.Helper
SECFilings secFilings = new SECFilings();
String secFilingDocument = null;
HttpNetResponse httpNetResponse=null;
int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving document at {0}]",secFilingDocumentUrl));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving documents at {0}]",secFilingDocumentUrl));
WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments");
try { Thread.Sleep(1000); }catch(Exception) { ;}
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
@@ -1409,11 +1510,11 @@ namespace MarketData.Helper
String formName = secNodes[0].InnerText;
formName = formName.Replace("\n", "");
formName = formName.Trim();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Examining form {0}]",formName));
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile\"]");
if (null == tables || tables.Count < 1) return null;
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
httpNetResponse.Dispose();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Examining form {0} with {1} entries",formName,rows.Count));
// go through the rows in the table file, searching for the different submission form content
int sequence = 0;
for (int row = 0; row < rows.Count; row++)
@@ -1425,21 +1526,22 @@ namespace MarketData.Helper
String document = dataColumns[2].InnerText.Trim();
String type = dataColumns[3].InnerText.Trim();
if(null!=type && type.Equals("GRAPHIC"))continue;
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("SEQ:{0},DOC:{1},TYPE:{2}",sequence,document,type));
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments: SEQ:{0},DOC:{1},TYPE:{2}",sequence,document,type));
if (document.Contains(".htm"))
{
HtmlNodeCollection nodes = dataColumns[2].SelectNodes(".//a");
if (null == nodes || 0 == nodes.Count) continue;
String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}",secFilingDocumentXmlUrl));
try{Thread.Sleep(1000);}catch(Exception){;}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Requesting {0}",secFilingDocumentXmlUrl));
try{Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS);}catch(Exception){;}
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
continue;
}
secFilingDocument =httpNetResponse.ResponseString;
secFilingDocument = httpNetResponse.ResponseString;
if (null == secFilingDocument) continue;
SECFiling secFiling = new SECFiling();
secFiling.SECAccessionNumber = secAccessionNumber;

4
MarketDataLib/Helper/ThreadHelper.cs
View File

@@ -1,8 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Threading;
using MarketData.Utils;

30
MarketDataLib/Utility/UpdateManager.cs
View File

@@ -16,28 +16,28 @@ namespace MarketData.Utils
public UpdateManager()
{
}
public bool Prepare(String strPathFileName)
public bool Prepare(String strPathFileName, int maxAgeDays=5)
{
try
{
String currentWorkingDirectory=Directory.GetCurrentDirectory();
if(!File.Exists(strPathFileName)||IsExpired(strPathFileName))
if(!File.Exists(strPathFileName)||IsExpired(strPathFileName, maxAgeDays))
{
if(File.Exists(strPathFileName))File.Delete(strPathFileName);
fileStream=new FileStream(strPathFileName,FileMode.Create);
streamWriter=new StreamWriter(fileStream);
Console.WriteLine(String.Format("Creating session file:{0}",strPathFileName));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Creating session file:{0}",strPathFileName));
}
else
{
FileStream fileStream=new FileStream(strPathFileName,FileMode.Open);
FileStream fileStream=new FileStream(strPathFileName,FileMode.Open,FileAccess.ReadWrite,FileShare.Read);
StreamReader streamReader=new StreamReader(fileStream);
String strLine=null;
while(null!=(strLine=streamReader.ReadLine()))
{
if(!entries.ContainsKey(strLine))entries.Add(strLine,strLine);
}
Console.WriteLine(String.Format("Loaded {0} entries from session file:{1}",entries.Count,strPathFileName));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Loaded {0} entries from session file:{1}",entries.Count,strPathFileName));
streamReader.Close();
streamReader.Dispose();
fileStream.Close();
@@ -49,27 +49,28 @@ namespace MarketData.Utils
}
catch(Exception exception)
{
Console.WriteLine(String.Format("Exception:{0}",exception));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception));
return false;
}
}
private bool IsExpired(String strPathFileName)
private bool IsExpired(String strPathFileName, int maxAgeDays)
{
try
{
DateGenerator dateGenerator=new DateGenerator();
DateTime creationTime=File.GetCreationTime(strPathFileName);
int daysElapsed=Math.Abs(dateGenerator.DaysBetweenActual(creationTime,DateTime.Now));
if(daysElapsed>5)
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} is {1} days(s) old.",strPathFileName,daysElapsed));
if(daysElapsed>maxAgeDays)
{
Console.WriteLine(String.Format("{0} is expired. {1} days old.",strPathFileName,daysElapsed));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} is expired. {1} days old.",strPathFileName,daysElapsed));
return true;
}
return false;
}
catch(Exception exception)
{
Console.WriteLine(String.Format("Exception:{0}",exception));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception));
return true;
}
}
@@ -79,9 +80,12 @@ namespace MarketData.Utils
}
public void Add(String entry)
{
if(null==streamWriter)return;
streamWriter.WriteLine(entry);
streamWriter.Flush();
lock(this)
{
if(null==streamWriter)return;
streamWriter.WriteLine(entry);
streamWriter.Flush();
}
}
public void Dispose()
{